astrbot_plugin_imagestudio/core/remote_fetch.py

180 行
6.4 KiB
Python

"""Bounded HTTP helpers for provider-controlled remote responses."""
from __future__ import annotations
import ipaddress
import socket
import urllib.request
from collections.abc import Iterable
from typing import Any
from urllib.parse import urlsplit
class RemoteFetchError(ValueError):
"""Raised when a remote URL or response violates the fetch policy."""
def read_http_response_limited(
request: str | urllib.request.Request,
*,
timeout_seconds: int,
max_bytes: int,
allow_private: bool = False,
allowed_private_hosts: Iterable[str] = (),
same_origin_redirects_only: bool = False,
) -> bytes:
"""Fetch an HTTP(S) response while validating redirects and size."""
if max_bytes <= 0:
raise ValueError("max_bytes must be positive")
url = request.full_url if isinstance(request, urllib.request.Request) else str(request)
allowed_hosts = frozenset(_normalize_host(host) for host in allowed_private_hosts if _normalize_host(host))
validate_http_url(url, allow_private=allow_private, allowed_private_hosts=allowed_hosts)
opener = urllib.request.build_opener(
_ValidatedRedirectHandler(
allow_private=allow_private,
allowed_private_hosts=allowed_hosts,
same_origin_only=same_origin_redirects_only,
)
)
with opener.open(request, timeout=timeout_seconds) as response:
declared_size = _content_length(response)
if declared_size is not None and declared_size > max_bytes:
raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes")
body = response.read(max_bytes + 1)
if len(body) > max_bytes:
raise RemoteFetchError(f"remote response exceeds {max_bytes} bytes")
return body
def validate_http_url(
url: str,
*,
allow_private: bool = False,
allowed_private_hosts: Iterable[str] = (),
) -> None:
"""Reject non-HTTP, credential-bearing and unsafe private-network URLs."""
try:
parsed = urlsplit(str(url or "").strip())
host = _normalize_host(parsed.hostname)
port = parsed.port or (443 if parsed.scheme.lower() == "https" else 80)
except ValueError as exc:
raise RemoteFetchError("remote URL is malformed") from exc
if parsed.scheme.lower() not in {"http", "https"}:
raise RemoteFetchError("remote URL must use http or https")
if parsed.username is not None or parsed.password is not None:
raise RemoteFetchError("remote URL must not include credentials")
if not host:
raise RemoteFetchError("remote URL must include a hostname")
allowed_hosts = {_normalize_host(item) for item in allowed_private_hosts}
if allow_private or host in allowed_hosts:
return
for address in _resolve_addresses(host, port):
if _is_unsafe_address(address):
raise RemoteFetchError("remote URL resolves to a private or local network address")
def url_hostname(url: str | None) -> str | None:
"""Return a normalized hostname from an HTTP(S) URL when present."""
try:
parsed = urlsplit(str(url or "").strip())
except ValueError:
return None
if parsed.scheme.lower() not in {"http", "https"}:
return None
return _normalize_host(parsed.hostname) or None
class _ValidatedRedirectHandler(urllib.request.HTTPRedirectHandler):
def __init__(
self,
*,
allow_private: bool,
allowed_private_hosts: frozenset[str],
same_origin_only: bool,
):
super().__init__()
self.allow_private = allow_private
self.allowed_private_hosts = allowed_private_hosts
self.same_origin_only = same_origin_only
def redirect_request(
self,
req: urllib.request.Request,
fp: Any,
code: int,
msg: str,
headers: Any,
newurl: str,
) -> urllib.request.Request | None:
validate_http_url(
newurl,
allow_private=self.allow_private,
allowed_private_hosts=self.allowed_private_hosts,
)
old_origin = _origin(req.full_url)
new_origin = _origin(newurl)
if old_origin[0] == "https" and new_origin[0] != "https":
raise RemoteFetchError("HTTPS redirects must not downgrade to HTTP")
if self.same_origin_only and old_origin != new_origin:
raise RemoteFetchError("provider endpoint redirected to a different origin")
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
if redirected is not None and old_origin != new_origin:
redirected.remove_header("Authorization")
return redirected
def _resolve_addresses(host: str, port: int) -> set[ipaddress.IPv4Address | ipaddress.IPv6Address]:
try:
literal = ipaddress.ip_address(host)
except ValueError:
literal = None
if literal is not None:
return {literal}
try:
rows = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise RemoteFetchError("remote hostname could not be resolved") from exc
addresses: set[ipaddress.IPv4Address | ipaddress.IPv6Address] = set()
for row in rows:
try:
addresses.add(ipaddress.ip_address(row[4][0].split("%", 1)[0]))
except (ValueError, IndexError):
continue
if not addresses:
raise RemoteFetchError("remote hostname did not resolve to an IP address")
return addresses
def _is_unsafe_address(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
# is_global also excludes shared/CGNAT (100.64.0.0/10), documentation,
# loopback, link-local, reserved and unspecified ranges.
return not address.is_global or address.is_multicast
def _content_length(response: Any) -> int | None:
value = response.headers.get("Content-Length") if getattr(response, "headers", None) else None
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _normalize_host(value: str | None) -> str:
return str(value or "").strip().rstrip(".").lower()
def _origin(url: str) -> tuple[str, str, int]:
try:
parsed = urlsplit(url)
scheme = parsed.scheme.lower()
host = _normalize_host(parsed.hostname)
port = parsed.port or (443 if scheme == "https" else 80)
except ValueError as exc:
raise RemoteFetchError("redirect URL is malformed") from exc
return scheme, host, port