astrbot_plugin_imagestudio/engines/vision_provider.py

380 行
14 KiB
Python

"""Vision provider adapter interfaces for material recognition."""
from __future__ import annotations
import json
import re
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any, Callable, Protocol
from core.remote_fetch import RemoteFetchError, read_http_response_limited, validate_http_url
JsonDict = dict[str, Any]
HttpTransport = Callable[[str, JsonDict, dict[str, str], int], JsonDict]
MAX_PROVIDER_JSON_BYTES = 4 * 1024 * 1024
MAX_PROVIDER_ERROR_BYTES = 64 * 1024
class VisionProvider(Protocol):
provider_id: str
def recognize_material(self, request: "VisionRecognitionRequest") -> "VisionRecognitionResult":
...
@dataclass(frozen=True)
class VisionProviderConfig:
provider_id: str
base_url: str
api_key: str = ""
vision_model: str = ""
vision_endpoint: str = "/v1/chat/completions"
provider_type: str = "openai_compatible"
timeout_seconds: int = 180
extra_headers: dict[str, str] = field(default_factory=dict)
def endpoint_url(self) -> str:
base = self.base_url.rstrip("/")
endpoint = self.vision_endpoint if self.vision_endpoint.startswith("/") else f"/{self.vision_endpoint}"
return f"{base}{endpoint}"
@dataclass(frozen=True)
class VisionRecognitionRequest:
display_name: str
image_uri: str
thumbnail_uri: str | None = None
prompt: str | None = None
negative_prompt: str | None = None
tags: tuple[str, ...] = ()
model: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class VisionRecognitionResult:
provider_id: str
model: str
prompt: str
negative_prompt: str | None = None
tags: list[str] = field(default_factory=list)
confidence: float | None = None
description: str | None = None
structured_data: dict[str, Any] = field(default_factory=dict)
raw_metadata: dict[str, Any] = field(default_factory=dict)
class VisionProviderError(RuntimeError):
def __init__(
self,
message: str,
*,
provider_id: str,
status_code: int | None = None,
error_type: str = "provider_error",
details: dict[str, Any] | None = None,
):
super().__init__(message)
self.provider_id = provider_id
self.status_code = status_code
self.error_type = error_type
self.details = details or {}
class OpenAIVisionProvider:
def __init__(self, config: VisionProviderConfig, transport: HttpTransport | None = None):
self.config = config
self.provider_id = config.provider_id
self._transport = transport or _post_json
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
model = request.model or self.config.vision_model
if not model:
raise VisionProviderError(
"vision model is required",
provider_id=self.provider_id,
error_type="configuration_error",
)
if not (_remote_image_url(request.image_uri) or _remote_image_url(request.thumbnail_uri)):
raise VisionProviderError(
"vision image must be supplied as an HTTP(S) URL or data:image URL",
provider_id=self.provider_id,
error_type="invalid_request",
)
payload = {
"model": model,
"messages": [
{
"role": "system",
"content": _system_instruction(request),
},
{"role": "user", "content": _user_content(request)},
],
"temperature": 0.1,
"response_format": {"type": "json_object"},
}
headers = {
"Content-Type": "application/json",
**({"Authorization": f"Bearer {self.config.api_key}"} if self.config.api_key else {}),
**self.config.extra_headers,
}
try:
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
except VisionProviderError as exc:
if exc.provider_id != "unknown":
raise
raise VisionProviderError(
str(exc),
provider_id=self.provider_id,
status_code=exc.status_code,
error_type=exc.error_type,
details=exc.details,
) from exc
return _parse_vision_response(self.provider_id, model, response)
class MockVisionProvider:
provider_id = "mock"
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
text = f"{request.display_name} {request.prompt or ''}".lower()
tags = ["material"]
for keyword in ("white", "blue", "black", "red", "ribbon", "shirt", "blouse", "dress", "skirt", "shoe"):
if keyword in text:
tags.append(keyword)
base = request.display_name.strip().lower().replace("_", " ") or "uploaded material"
return VisionRecognitionResult(
provider_id=self.provider_id,
model=request.model or "mock-vlm",
prompt=f"{base}, clean product reference, transparent background",
negative_prompt="body, face, background, watermark",
tags=tags,
confidence=0.72,
structured_data={
"raw_analysis": {
"attributes": {
"design": {
"value": base,
"confidence": 0.72,
"source_state": "observed",
}
}
}
},
raw_metadata={"mock": True},
)
def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]:
content: list[dict[str, Any]] = [
{
"type": "text",
"text": (
f"Material name: {request.display_name}\n"
f"Known prompt: {request.prompt or ''}\n"
f"Known negative prompt: {request.negative_prompt or ''}\n"
f"Known tags: {', '.join(request.tags)}"
),
}
]
image_url = _remote_image_url(request.image_uri) or _remote_image_url(request.thumbnail_uri)
if image_url:
content.append({"type": "image_url", "image_url": {"url": image_url}})
return content
def _system_instruction(request: VisionRecognitionRequest) -> str:
configured = str(request.metadata.get("recognition_instruction") or "").strip()
fallback = (
"Analyze the visible material as exhaustively as possible. First identify its category, then extract only "
"visible facts including object type, colors, materials, texture, construction, silhouette, patterns, trims, "
"accessories, condition, viewpoint, lighting and reusable design features. Distinguish certainty from guesses. "
"Produce a detailed image-generation prompt that preserves the recognized design, plus a useful negative prompt. "
"Return one JSON object. Keep compatibility keys prompt, negative_prompt, tags, confidence and description. "
"Also include raw_analysis.attributes where every attribute has value, confidence (0-1), and source_state "
"(observed, inferred, unknown, occluded, or low_visibility); normalized_asset_profile with category, role, "
"body_slots, wear_layer, must_preserve and forbidden_changes; and optional generation_capsules. "
"Never present guesses as observed facts. Tags must be concise and searchable."
)
return configured or fallback
def _remote_image_url(value: str | None) -> str | None:
text = str(value or "").strip()
if text.startswith(("http://", "https://", "data:image/")):
return text
return None
def _post_json(url: str, payload: JsonDict, headers: dict[str, str], timeout_seconds: int) -> JsonDict:
try:
validate_http_url(url, allow_private=True)
except RemoteFetchError as exc:
raise VisionProviderError(
"vision provider endpoint is invalid",
provider_id="unknown",
error_type="configuration_error",
details={"reason": str(exc)},
) from exc
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(url, data=data, headers=headers, method="POST")
try:
body = read_http_response_limited(
request,
timeout_seconds=timeout_seconds,
max_bytes=MAX_PROVIDER_JSON_BYTES,
allow_private=True,
same_origin_redirects_only=True,
).decode("utf-8-sig")
except urllib.error.HTTPError as exc:
body = exc.read(MAX_PROVIDER_ERROR_BYTES + 1).decode("utf-8", errors="replace")
raise VisionProviderError(
"vision provider returned an HTTP error",
provider_id="unknown",
status_code=exc.code,
error_type="http_error",
details={"body_preview": body[:500], "body_truncated": len(body) > MAX_PROVIDER_ERROR_BYTES},
) from exc
except urllib.error.URLError as exc:
raise VisionProviderError(
"vision provider network error",
provider_id="unknown",
error_type="network_error",
details={"reason": str(exc.reason)},
) from exc
except (RemoteFetchError, UnicodeDecodeError) as exc:
raise VisionProviderError(
"vision provider response exceeded safety limits",
provider_id="unknown",
error_type="invalid_response",
details={"reason": str(exc)},
) from exc
try:
parsed = json.loads(body)
except json.JSONDecodeError as exc:
raise VisionProviderError(
"vision provider returned invalid JSON",
provider_id="unknown",
error_type="invalid_response",
details={"body_preview": body[:500]},
) from exc
if not isinstance(parsed, dict):
raise VisionProviderError("vision provider response must be an object", provider_id="unknown")
return parsed
def _parse_vision_response(provider_id: str, model: str, response: JsonDict) -> VisionRecognitionResult:
content = _response_content(response)
parsed = _json_from_text(content)
prompt = _optional_text(parsed.get("prompt") or parsed.get("positive_prompt") or parsed.get("description"))
if not prompt:
raise VisionProviderError(
"vision provider response must include prompt or description",
provider_id=provider_id,
error_type="invalid_response",
details={"content_preview": content[:500]},
)
return VisionRecognitionResult(
provider_id=provider_id,
model=model,
prompt=prompt,
negative_prompt=_optional_text(parsed.get("negative_prompt")),
tags=_string_list(parsed.get("tags")),
confidence=_float_or_none(parsed.get("confidence")),
description=_optional_text(parsed.get("description")),
structured_data=parsed,
raw_metadata={
"response_keys": sorted(str(key) for key in response.keys()),
"structured_keys": sorted(str(key) for key in parsed.keys()),
},
)
def _response_content(response: JsonDict) -> str:
output_text = _optional_text(response.get("output_text"))
if output_text:
return output_text
choices = response.get("choices")
if isinstance(choices, list) and choices:
first = choices[0] if isinstance(choices[0], dict) else {}
message = first.get("message") if isinstance(first.get("message"), dict) else {}
return _content_to_text(message.get("content"))
output = response.get("output")
if isinstance(output, list):
return " ".join(_content_to_text(item.get("content")) for item in output if isinstance(item, dict)).strip()
return ""
def _content_to_text(value: Any) -> str:
if isinstance(value, str):
return value.strip()
if isinstance(value, list):
parts = []
for item in value:
if not isinstance(item, dict):
continue
text = item.get("text") or item.get("content")
if isinstance(text, str):
parts.append(text)
return " ".join(parts).strip()
return ""
def _json_from_text(value: str) -> JsonDict:
text = value.strip().lstrip("\ufeff")
if not text:
raise VisionProviderError("vision provider response was empty", provider_id="unknown", error_type="invalid_response")
candidates = [text]
start = text.find("{")
end = text.rfind("}")
if start >= 0 and end > start:
extracted = text[start : end + 1]
if extracted != text:
candidates.append(extracted)
# A deliberately small repair set: fenced JSON and trailing commas only.
# It does not invent fields or values when a provider response is malformed.
candidates.extend(re.sub(r",\s*([}\]])", r"\1", item) for item in list(candidates))
parsed: Any = None
last_error: json.JSONDecodeError | None = None
for candidate in candidates[:4]:
try:
parsed = json.loads(candidate)
break
except json.JSONDecodeError as exc:
last_error = exc
if parsed is None:
raise VisionProviderError(
"vision provider response was not valid JSON",
provider_id="unknown",
error_type="invalid_response",
details={"content_preview": text[:500], "parse_error": str(last_error or "unknown")},
) from None
if not isinstance(parsed, dict):
raise VisionProviderError("vision provider JSON must be an object", provider_id="unknown", error_type="invalid_response")
return parsed
def _string_list(value: Any) -> list[str]:
if isinstance(value, str):
values = value.split(",")
elif isinstance(value, list):
values = value
else:
values = []
return sorted({str(item).strip() for item in values if str(item).strip()})
def _float_or_none(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return min(max(number, 0.0), 1.0)
def _optional_text(value: Any) -> str | None:
text = str(value or "").strip()
return text or None