180 行
6.4 KiB
Python
180 行
6.4 KiB
Python
|
|
"""Bounded HTTP helpers for provider-controlled remote responses."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import ipaddress
|
||
|
|
import socket
|
||
|
|
import urllib.request
|
||
|
|
from collections.abc import Iterable
|
||
|
|
from typing import Any
|
||
|
|
from urllib.parse import urlsplit
|
||
|
|
|
||
|
|
|
||
|
|
class RemoteFetchError(ValueError):
|
||
|
|
"""Raised when a remote URL or response violates the fetch policy."""
|
||
|
|
|
||
|
|
|
||
|
|
def read_http_response_limited(
|
||
|
|
request: str | urllib.request.Request,
|
||
|
|
*,
|
||
|
|
timeout_seconds: int,
|
||
|
|
max_bytes: int,
|
||
|
|
allow_private: bool = False,
|
||
|
|
allowed_private_hosts: Iterable[str] = (),
|
||
|
|
same_origin_redirects_only: bool = False,
|
||
|
|
) -> bytes:
|
||
|
|
"""Fetch an HTTP(S) response while validating redirects and size."""
|
||
|
|
|
||
|
|
if max_bytes <= 0:
|
||
|
|
raise ValueError("max_bytes must be positive")
|
||
|
|
url = request.full_url if isinstance(request, urllib.request.Request) else str(request)
|
||
|
|
allowed_hosts = frozenset(_normalize_host(host) for host in allowed_private_hosts if _normalize_host(host))
|
||
|
|
validate_http_url(url, allow_private=allow_private, allowed_private_hosts=allowed_hosts)
|
||
|
|
opener = urllib.request.build_opener(
|
||
|
|
_ValidatedRedirectHandler(
|
||
|
|
allow_private=allow_private,
|
||
|
|
allowed_private_hosts=allowed_hosts,
|
||
|
|
same_origin_only=same_origin_redirects_only,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
with opener.open(request, timeout=timeout_seconds) as response:
|
||
|
|
declared_size = _content_length(response)
|
||
|
|
if declared_size is not None and declared_size > max_bytes:
|
||
|
|
raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes")
|
||
|
|
body = response.read(max_bytes + 1)
|
||
|
|
if len(body) > max_bytes:
|
||
|
|
raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes")
|
||
|
|
return body
|
||
|
|
|
||
|
|
|
||
|
|
def validate_http_url(
|
||
|
|
url: str,
|
||
|
|
*,
|
||
|
|
allow_private: bool = False,
|
||
|
|
allowed_private_hosts: Iterable[str] = (),
|
||
|
|
) -> None:
|
||
|
|
"""Reject non-HTTP, credential-bearing and unsafe private-network URLs."""
|
||
|
|
|
||
|
|
try:
|
||
|
|
parsed = urlsplit(str(url or "").strip())
|
||
|
|
host = _normalize_host(parsed.hostname)
|
||
|
|
port = parsed.port or (443 if parsed.scheme.lower() == "https" else 80)
|
||
|
|
except ValueError as exc:
|
||
|
|
raise RemoteFetchError("remote URL is malformed") from exc
|
||
|
|
if parsed.scheme.lower() not in {"http", "https"}:
|
||
|
|
raise RemoteFetchError("remote URL must use http or https")
|
||
|
|
if parsed.username is not None or parsed.password is not None:
|
||
|
|
raise RemoteFetchError("remote URL must not include credentials")
|
||
|
|
if not host:
|
||
|
|
raise RemoteFetchError("remote URL must include a hostname")
|
||
|
|
allowed_hosts = {_normalize_host(item) for item in allowed_private_hosts}
|
||
|
|
if allow_private or host in allowed_hosts:
|
||
|
|
return
|
||
|
|
for address in _resolve_addresses(host, port):
|
||
|
|
if _is_unsafe_address(address):
|
||
|
|
raise RemoteFetchError("remote URL resolves to a private or local network address")
|
||
|
|
|
||
|
|
|
||
|
|
def url_hostname(url: str | None) -> str | None:
|
||
|
|
"""Return a normalized hostname from an HTTP(S) URL when present."""
|
||
|
|
|
||
|
|
try:
|
||
|
|
parsed = urlsplit(str(url or "").strip())
|
||
|
|
except ValueError:
|
||
|
|
return None
|
||
|
|
if parsed.scheme.lower() not in {"http", "https"}:
|
||
|
|
return None
|
||
|
|
return _normalize_host(parsed.hostname) or None
|
||
|
|
|
||
|
|
|
||
|
|
class _ValidatedRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
*,
|
||
|
|
allow_private: bool,
|
||
|
|
allowed_private_hosts: frozenset[str],
|
||
|
|
same_origin_only: bool,
|
||
|
|
):
|
||
|
|
super().__init__()
|
||
|
|
self.allow_private = allow_private
|
||
|
|
self.allowed_private_hosts = allowed_private_hosts
|
||
|
|
self.same_origin_only = same_origin_only
|
||
|
|
|
||
|
|
def redirect_request(
|
||
|
|
self,
|
||
|
|
req: urllib.request.Request,
|
||
|
|
fp: Any,
|
||
|
|
code: int,
|
||
|
|
msg: str,
|
||
|
|
headers: Any,
|
||
|
|
newurl: str,
|
||
|
|
) -> urllib.request.Request | None:
|
||
|
|
validate_http_url(
|
||
|
|
newurl,
|
||
|
|
allow_private=self.allow_private,
|
||
|
|
allowed_private_hosts=self.allowed_private_hosts,
|
||
|
|
)
|
||
|
|
old_origin = _origin(req.full_url)
|
||
|
|
new_origin = _origin(newurl)
|
||
|
|
if old_origin[0] == "https" and new_origin[0] != "https":
|
||
|
|
raise RemoteFetchError("HTTPS redirects must not downgrade to HTTP")
|
||
|
|
if self.same_origin_only and old_origin != new_origin:
|
||
|
|
raise RemoteFetchError("provider endpoint redirected to a different origin")
|
||
|
|
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
|
||
|
|
if redirected is not None and old_origin != new_origin:
|
||
|
|
redirected.remove_header("Authorization")
|
||
|
|
return redirected
|
||
|
|
|
||
|
|
|
||
|
|
def _resolve_addresses(host: str, port: int) -> set[ipaddress.IPv4Address | ipaddress.IPv6Address]:
|
||
|
|
try:
|
||
|
|
literal = ipaddress.ip_address(host)
|
||
|
|
except ValueError:
|
||
|
|
literal = None
|
||
|
|
if literal is not None:
|
||
|
|
return {literal}
|
||
|
|
try:
|
||
|
|
rows = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||
|
|
except OSError as exc:
|
||
|
|
raise RemoteFetchError("remote hostname could not be resolved") from exc
|
||
|
|
addresses: set[ipaddress.IPv4Address | ipaddress.IPv6Address] = set()
|
||
|
|
for row in rows:
|
||
|
|
try:
|
||
|
|
addresses.add(ipaddress.ip_address(row[4][0].split("%", 1)[0]))
|
||
|
|
except (ValueError, IndexError):
|
||
|
|
continue
|
||
|
|
if not addresses:
|
||
|
|
raise RemoteFetchError("remote hostname did not resolve to an IP address")
|
||
|
|
return addresses
|
||
|
|
|
||
|
|
|
||
|
|
def _is_unsafe_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||
|
|
# is_global also excludes shared/CGNAT (100.64.0.0/10), documentation,
|
||
|
|
# loopback, link-local, reserved and unspecified ranges.
|
||
|
|
return not address.is_global or address.is_multicast
|
||
|
|
|
||
|
|
|
||
|
|
def _content_length(response: Any) -> int | None:
|
||
|
|
value = response.headers.get("Content-Length") if getattr(response, "headers", None) else None
|
||
|
|
if value in (None, ""):
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
return int(value)
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_host(value: str | None) -> str:
|
||
|
|
return str(value or "").strip().rstrip(".").lower()
|
||
|
|
|
||
|
|
|
||
|
|
def _origin(url: str) -> tuple[str, str, int]:
|
||
|
|
try:
|
||
|
|
parsed = urlsplit(url)
|
||
|
|
scheme = parsed.scheme.lower()
|
||
|
|
host = _normalize_host(parsed.hostname)
|
||
|
|
port = parsed.port or (443 if scheme == "https" else 80)
|
||
|
|
except ValueError as exc:
|
||
|
|
raise RemoteFetchError("redirect URL is malformed") from exc
|
||
|
|
return scheme, host, port
|