2026-07-06 03:37:17 +08:00
|
|
|
"""Vision provider adapter interfaces for material recognition."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import json
|
|
|
|
|
import urllib.error
|
|
|
|
|
import urllib.request
|
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
from typing import Any, Callable, Protocol
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
from core.remote_fetch import RemoteFetchError, read_http_response_limited, validate_http_url
|
|
|
|
|
|
2026-07-06 03:37:17 +08:00
|
|
|
|
|
|
|
|
JsonDict = dict[str, Any]
|
|
|
|
|
HttpTransport = Callable[[str, JsonDict, dict[str, str], int], JsonDict]
|
2026-07-11 21:50:34 +08:00
|
|
|
MAX_PROVIDER_JSON_BYTES = 4 * 1024 * 1024
|
|
|
|
|
MAX_PROVIDER_ERROR_BYTES = 64 * 1024
|
2026-07-06 03:37:17 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
class VisionProvider(Protocol):
|
|
|
|
|
provider_id: str
|
|
|
|
|
|
|
|
|
|
def recognize_material(self, request: "VisionRecognitionRequest") -> "VisionRecognitionResult":
|
|
|
|
|
...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class VisionProviderConfig:
|
|
|
|
|
provider_id: str
|
|
|
|
|
base_url: str
|
|
|
|
|
api_key: str = ""
|
|
|
|
|
vision_model: str = ""
|
|
|
|
|
vision_endpoint: str = "/v1/chat/completions"
|
|
|
|
|
provider_type: str = "openai_compatible"
|
|
|
|
|
timeout_seconds: int = 180
|
|
|
|
|
extra_headers: dict[str, str] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
def endpoint_url(self) -> str:
|
|
|
|
|
base = self.base_url.rstrip("/")
|
|
|
|
|
endpoint = self.vision_endpoint if self.vision_endpoint.startswith("/") else f"/{self.vision_endpoint}"
|
|
|
|
|
return f"{base}{endpoint}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class VisionRecognitionRequest:
|
|
|
|
|
display_name: str
|
|
|
|
|
image_uri: str
|
|
|
|
|
thumbnail_uri: str | None = None
|
|
|
|
|
prompt: str | None = None
|
|
|
|
|
negative_prompt: str | None = None
|
|
|
|
|
tags: tuple[str, ...] = ()
|
|
|
|
|
model: str | None = None
|
|
|
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class VisionRecognitionResult:
|
|
|
|
|
provider_id: str
|
|
|
|
|
model: str
|
|
|
|
|
prompt: str
|
|
|
|
|
negative_prompt: str | None = None
|
|
|
|
|
tags: list[str] = field(default_factory=list)
|
|
|
|
|
confidence: float | None = None
|
|
|
|
|
description: str | None = None
|
|
|
|
|
raw_metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class VisionProviderError(RuntimeError):
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
message: str,
|
|
|
|
|
*,
|
|
|
|
|
provider_id: str,
|
|
|
|
|
status_code: int | None = None,
|
|
|
|
|
error_type: str = "provider_error",
|
|
|
|
|
details: dict[str, Any] | None = None,
|
|
|
|
|
):
|
|
|
|
|
super().__init__(message)
|
|
|
|
|
self.provider_id = provider_id
|
|
|
|
|
self.status_code = status_code
|
|
|
|
|
self.error_type = error_type
|
|
|
|
|
self.details = details or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class OpenAIVisionProvider:
|
|
|
|
|
def __init__(self, config: VisionProviderConfig, transport: HttpTransport | None = None):
|
|
|
|
|
self.config = config
|
|
|
|
|
self.provider_id = config.provider_id
|
|
|
|
|
self._transport = transport or _post_json
|
|
|
|
|
|
|
|
|
|
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
|
|
|
|
|
model = request.model or self.config.vision_model
|
|
|
|
|
if not model:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision model is required",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="configuration_error",
|
|
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
if not (_remote_image_url(request.image_uri) or _remote_image_url(request.thumbnail_uri)):
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision image must be supplied as an HTTP(S) URL or data:image URL",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="invalid_request",
|
|
|
|
|
)
|
2026-07-06 03:37:17 +08:00
|
|
|
payload = {
|
|
|
|
|
"model": model,
|
|
|
|
|
"messages": [
|
|
|
|
|
{
|
|
|
|
|
"role": "system",
|
|
|
|
|
"content": (
|
|
|
|
|
"Return compact JSON for a reusable outfit material with keys: "
|
|
|
|
|
"prompt, negative_prompt, tags, confidence, description."
|
|
|
|
|
),
|
|
|
|
|
},
|
|
|
|
|
{"role": "user", "content": _user_content(request)},
|
|
|
|
|
],
|
|
|
|
|
"temperature": 0.1,
|
|
|
|
|
"response_format": {"type": "json_object"},
|
|
|
|
|
}
|
|
|
|
|
headers = {
|
|
|
|
|
"Content-Type": "application/json",
|
2026-07-11 21:50:34 +08:00
|
|
|
**({"Authorization": f"Bearer {self.config.api_key}"} if self.config.api_key else {}),
|
2026-07-06 03:37:17 +08:00
|
|
|
**self.config.extra_headers,
|
|
|
|
|
}
|
|
|
|
|
try:
|
|
|
|
|
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
|
|
|
|
|
except VisionProviderError as exc:
|
|
|
|
|
if exc.provider_id != "unknown":
|
|
|
|
|
raise
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
str(exc),
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
status_code=exc.status_code,
|
|
|
|
|
error_type=exc.error_type,
|
|
|
|
|
details=exc.details,
|
|
|
|
|
) from exc
|
|
|
|
|
return _parse_vision_response(self.provider_id, model, response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MockVisionProvider:
|
|
|
|
|
provider_id = "mock"
|
|
|
|
|
|
|
|
|
|
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
|
|
|
|
|
text = f"{request.display_name} {request.prompt or ''}".lower()
|
|
|
|
|
tags = ["material"]
|
|
|
|
|
for keyword in ("white", "blue", "black", "red", "ribbon", "shirt", "blouse", "dress", "skirt", "shoe"):
|
|
|
|
|
if keyword in text:
|
|
|
|
|
tags.append(keyword)
|
|
|
|
|
base = request.display_name.strip().lower().replace("_", " ") or "uploaded material"
|
|
|
|
|
return VisionRecognitionResult(
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
model=request.model or "mock-vlm",
|
|
|
|
|
prompt=f"{base}, clean product reference, transparent background",
|
|
|
|
|
negative_prompt="body, face, background, watermark",
|
|
|
|
|
tags=tags,
|
|
|
|
|
confidence=0.72,
|
|
|
|
|
raw_metadata={"mock": True},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]:
|
|
|
|
|
content: list[dict[str, Any]] = [
|
|
|
|
|
{
|
|
|
|
|
"type": "text",
|
|
|
|
|
"text": (
|
|
|
|
|
f"Material name: {request.display_name}\n"
|
|
|
|
|
f"Known prompt: {request.prompt or ''}\n"
|
|
|
|
|
f"Known negative prompt: {request.negative_prompt or ''}\n"
|
|
|
|
|
f"Known tags: {', '.join(request.tags)}"
|
|
|
|
|
),
|
|
|
|
|
}
|
|
|
|
|
]
|
2026-07-11 21:50:34 +08:00
|
|
|
image_url = _remote_image_url(request.image_uri) or _remote_image_url(request.thumbnail_uri)
|
2026-07-06 03:37:17 +08:00
|
|
|
if image_url:
|
|
|
|
|
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
|
|
|
|
return content
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _remote_image_url(value: str | None) -> str | None:
|
|
|
|
|
text = str(value or "").strip()
|
|
|
|
|
if text.startswith(("http://", "https://", "data:image/")):
|
|
|
|
|
return text
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _post_json(url: str, payload: JsonDict, headers: dict[str, str], timeout_seconds: int) -> JsonDict:
|
2026-07-11 21:50:34 +08:00
|
|
|
try:
|
|
|
|
|
validate_http_url(url, allow_private=True)
|
|
|
|
|
except RemoteFetchError as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider endpoint is invalid",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
error_type="configuration_error",
|
|
|
|
|
details={"reason": str(exc)},
|
|
|
|
|
) from exc
|
2026-07-06 03:37:17 +08:00
|
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
|
|
|
request = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
|
|
|
|
try:
|
2026-07-11 21:50:34 +08:00
|
|
|
body = read_http_response_limited(
|
|
|
|
|
request,
|
|
|
|
|
timeout_seconds=timeout_seconds,
|
|
|
|
|
max_bytes=MAX_PROVIDER_JSON_BYTES,
|
|
|
|
|
allow_private=True,
|
|
|
|
|
same_origin_redirects_only=True,
|
|
|
|
|
).decode("utf-8-sig")
|
2026-07-06 03:37:17 +08:00
|
|
|
except urllib.error.HTTPError as exc:
|
2026-07-11 21:50:34 +08:00
|
|
|
body = exc.read(MAX_PROVIDER_ERROR_BYTES + 1).decode("utf-8", errors="replace")
|
2026-07-06 03:37:17 +08:00
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider returned an HTTP error",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
status_code=exc.code,
|
|
|
|
|
error_type="http_error",
|
2026-07-11 21:50:34 +08:00
|
|
|
details={"body_preview": body[:500], "body_truncated": len(body) > MAX_PROVIDER_ERROR_BYTES},
|
2026-07-06 03:37:17 +08:00
|
|
|
) from exc
|
|
|
|
|
except urllib.error.URLError as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider network error",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
error_type="network_error",
|
|
|
|
|
details={"reason": str(exc.reason)},
|
|
|
|
|
) from exc
|
2026-07-11 21:50:34 +08:00
|
|
|
except (RemoteFetchError, UnicodeDecodeError) as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider response exceeded safety limits",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"reason": str(exc)},
|
|
|
|
|
) from exc
|
2026-07-06 03:37:17 +08:00
|
|
|
try:
|
|
|
|
|
parsed = json.loads(body)
|
|
|
|
|
except json.JSONDecodeError as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider returned invalid JSON",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"body_preview": body[:500]},
|
|
|
|
|
) from exc
|
|
|
|
|
if not isinstance(parsed, dict):
|
|
|
|
|
raise VisionProviderError("vision provider response must be an object", provider_id="unknown")
|
|
|
|
|
return parsed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _parse_vision_response(provider_id: str, model: str, response: JsonDict) -> VisionRecognitionResult:
|
|
|
|
|
content = _response_content(response)
|
|
|
|
|
parsed = _json_from_text(content)
|
|
|
|
|
prompt = _optional_text(parsed.get("prompt") or parsed.get("positive_prompt") or parsed.get("description"))
|
|
|
|
|
if not prompt:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider response must include prompt or description",
|
|
|
|
|
provider_id=provider_id,
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"content_preview": content[:500]},
|
|
|
|
|
)
|
|
|
|
|
return VisionRecognitionResult(
|
|
|
|
|
provider_id=provider_id,
|
|
|
|
|
model=model,
|
|
|
|
|
prompt=prompt,
|
|
|
|
|
negative_prompt=_optional_text(parsed.get("negative_prompt")),
|
|
|
|
|
tags=_string_list(parsed.get("tags")),
|
|
|
|
|
confidence=_float_or_none(parsed.get("confidence")),
|
|
|
|
|
description=_optional_text(parsed.get("description")),
|
|
|
|
|
raw_metadata={"response_keys": sorted(str(key) for key in response.keys())},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _response_content(response: JsonDict) -> str:
|
|
|
|
|
output_text = _optional_text(response.get("output_text"))
|
|
|
|
|
if output_text:
|
|
|
|
|
return output_text
|
|
|
|
|
choices = response.get("choices")
|
|
|
|
|
if isinstance(choices, list) and choices:
|
|
|
|
|
first = choices[0] if isinstance(choices[0], dict) else {}
|
|
|
|
|
message = first.get("message") if isinstance(first.get("message"), dict) else {}
|
|
|
|
|
return _content_to_text(message.get("content"))
|
|
|
|
|
output = response.get("output")
|
|
|
|
|
if isinstance(output, list):
|
|
|
|
|
return " ".join(_content_to_text(item.get("content")) for item in output if isinstance(item, dict)).strip()
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _content_to_text(value: Any) -> str:
|
|
|
|
|
if isinstance(value, str):
|
|
|
|
|
return value.strip()
|
|
|
|
|
if isinstance(value, list):
|
|
|
|
|
parts = []
|
|
|
|
|
for item in value:
|
|
|
|
|
if not isinstance(item, dict):
|
|
|
|
|
continue
|
|
|
|
|
text = item.get("text") or item.get("content")
|
|
|
|
|
if isinstance(text, str):
|
|
|
|
|
parts.append(text)
|
|
|
|
|
return " ".join(parts).strip()
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _json_from_text(value: str) -> JsonDict:
|
|
|
|
|
text = value.strip()
|
|
|
|
|
if not text:
|
|
|
|
|
raise VisionProviderError("vision provider response was empty", provider_id="unknown", error_type="invalid_response")
|
|
|
|
|
try:
|
|
|
|
|
parsed = json.loads(text)
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
start = text.find("{")
|
|
|
|
|
end = text.rfind("}")
|
|
|
|
|
if start < 0 or end <= start:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"vision provider response was not JSON",
|
|
|
|
|
provider_id="unknown",
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"content_preview": text[:500]},
|
|
|
|
|
) from None
|
|
|
|
|
parsed = json.loads(text[start : end + 1])
|
|
|
|
|
if not isinstance(parsed, dict):
|
|
|
|
|
raise VisionProviderError("vision provider JSON must be an object", provider_id="unknown", error_type="invalid_response")
|
|
|
|
|
return parsed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _string_list(value: Any) -> list[str]:
|
|
|
|
|
if isinstance(value, str):
|
|
|
|
|
values = value.split(",")
|
|
|
|
|
elif isinstance(value, list):
|
|
|
|
|
values = value
|
|
|
|
|
else:
|
|
|
|
|
values = []
|
|
|
|
|
return sorted({str(item).strip() for item in values if str(item).strip()})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _float_or_none(value: Any) -> float | None:
|
|
|
|
|
try:
|
|
|
|
|
number = float(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return None
|
|
|
|
|
return min(max(number, 0.0), 1.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
text = str(value or "").strip()
|
|
|
|
|
return text or None
|