"""Bounded HTTP helpers for provider-controlled remote responses.""" from __future__ import annotations import ipaddress import socket import urllib.request from collections.abc import Iterable from typing import Any from urllib.parse import urlsplit class RemoteFetchError(ValueError): """Raised when a remote URL or response violates the fetch policy.""" def read_http_response_limited( request: str | urllib.request.Request, *, timeout_seconds: int, max_bytes: int, allow_private: bool = False, allowed_private_hosts: Iterable[str] = (), same_origin_redirects_only: bool = False, ) -> bytes: """Fetch an HTTP(S) response while validating redirects and size.""" if max_bytes <= 0: raise ValueError("max_bytes must be positive") url = request.full_url if isinstance(request, urllib.request.Request) else str(request) allowed_hosts = frozenset(_normalize_host(host) for host in allowed_private_hosts if _normalize_host(host)) validate_http_url(url, allow_private=allow_private, allowed_private_hosts=allowed_hosts) opener = urllib.request.build_opener( _ValidatedRedirectHandler( allow_private=allow_private, allowed_private_hosts=allowed_hosts, same_origin_only=same_origin_redirects_only, ) ) with opener.open(request, timeout=timeout_seconds) as response: declared_size = _content_length(response) if declared_size is not None and declared_size > max_bytes: raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes") body = response.read(max_bytes + 1) if len(body) > max_bytes: raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes") return body def validate_http_url( url: str, *, allow_private: bool = False, allowed_private_hosts: Iterable[str] = (), ) -> None: """Reject non-HTTP, credential-bearing and unsafe private-network URLs.""" try: parsed = urlsplit(str(url or "").strip()) host = _normalize_host(parsed.hostname) port = parsed.port or (443 if parsed.scheme.lower() == "https" else 80) except ValueError as exc: raise RemoteFetchError("remote URL is malformed") from exc if parsed.scheme.lower() not in {"http", "https"}: raise RemoteFetchError("remote URL must use http or https") if parsed.username is not None or parsed.password is not None: raise RemoteFetchError("remote URL must not include credentials") if not host: raise RemoteFetchError("remote URL must include a hostname") allowed_hosts = {_normalize_host(item) for item in allowed_private_hosts} if allow_private or host in allowed_hosts: return for address in _resolve_addresses(host, port): if _is_unsafe_address(address): raise RemoteFetchError("remote URL resolves to a private or local network address") def url_hostname(url: str | None) -> str | None: """Return a normalized hostname from an HTTP(S) URL when present.""" try: parsed = urlsplit(str(url or "").strip()) except ValueError: return None if parsed.scheme.lower() not in {"http", "https"}: return None return _normalize_host(parsed.hostname) or None class _ValidatedRedirectHandler(urllib.request.HTTPRedirectHandler): def __init__( self, *, allow_private: bool, allowed_private_hosts: frozenset[str], same_origin_only: bool, ): super().__init__() self.allow_private = allow_private self.allowed_private_hosts = allowed_private_hosts self.same_origin_only = same_origin_only def redirect_request( self, req: urllib.request.Request, fp: Any, code: int, msg: str, headers: Any, newurl: str, ) -> urllib.request.Request | None: validate_http_url( newurl, allow_private=self.allow_private, allowed_private_hosts=self.allowed_private_hosts, ) old_origin = _origin(req.full_url) new_origin = _origin(newurl) if old_origin[0] == "https" and new_origin[0] != "https": raise RemoteFetchError("HTTPS redirects must not downgrade to HTTP") if self.same_origin_only and old_origin != new_origin: raise RemoteFetchError("provider endpoint redirected to a different origin") redirected = super().redirect_request(req, fp, code, msg, headers, newurl) if redirected is not None and old_origin != new_origin: redirected.remove_header("Authorization") return redirected def _resolve_addresses(host: str, port: int) -> set[ipaddress.IPv4Address | ipaddress.IPv6Address]: try: literal = ipaddress.ip_address(host) except ValueError: literal = None if literal is not None: return {literal} try: rows = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) except OSError as exc: raise RemoteFetchError("remote hostname could not be resolved") from exc addresses: set[ipaddress.IPv4Address | ipaddress.IPv6Address] = set() for row in rows: try: addresses.add(ipaddress.ip_address(row[4][0].split("%", 1)[0])) except (ValueError, IndexError): continue if not addresses: raise RemoteFetchError("remote hostname did not resolve to an IP address") return addresses def _is_unsafe_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: # is_global also excludes shared/CGNAT (100.64.0.0/10), documentation, # loopback, link-local, reserved and unspecified ranges. return not address.is_global or address.is_multicast def _content_length(response: Any) -> int | None: value = response.headers.get("Content-Length") if getattr(response, "headers", None) else None if value in (None, ""): return None try: return int(value) except (TypeError, ValueError): return None def _normalize_host(value: str | None) -> str: return str(value or "").strip().rstrip(".").lower() def _origin(url: str) -> tuple[str, str, int]: try: parsed = urlsplit(url) scheme = parsed.scheme.lower() host = _normalize_host(parsed.hostname) port = parsed.port or (443 if scheme == "https" else 80) except ValueError as exc: raise RemoteFetchError("redirect URL is malformed") from exc return scheme, host, port