S100 VLM适配器
这个提交存在于:
父节点
36a3c1708c
当前提交
0df3546c6d
共有 8 个文件被更改,包括 711 次插入 和 6 次删除
|
|
@ -9,11 +9,14 @@ from backend.dto.material_dto import Material, MaterialListResult
|
|||
from backend.routes.common import handle_request
|
||||
from backend.services.material_service import MaterialService
|
||||
from backend.services.media_service import MediaService
|
||||
from backend.services.model_provider_runtime import ModelProviderRuntime
|
||||
from backend.services.model_provider_service import ModelProviderService
|
||||
from core.audit_service import AuditService
|
||||
from core.identity import RequestContext
|
||||
from core.security import require_authenticated, require_permission
|
||||
from database.db_helper import Database, load_database_settings
|
||||
from database.repositories.material_repository import MaterialRepository
|
||||
from database.repositories.model_provider_repository import ModelProviderRepository
|
||||
|
||||
|
||||
def list_material_categories(
|
||||
|
|
@ -227,9 +230,12 @@ def confirm_material(
|
|||
|
||||
def _create_material_service() -> MaterialService:
|
||||
database = Database(load_database_settings())
|
||||
model_provider_service = ModelProviderService(ModelProviderRepository(database))
|
||||
model_provider_runtime = ModelProviderRuntime(model_provider_service)
|
||||
return MaterialService(
|
||||
repository=MaterialRepository(database),
|
||||
audit_service=AuditService(database),
|
||||
vision_provider_factory=model_provider_runtime.vision_provider_for_route,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import PurePosixPath
|
||||
from typing import Any
|
||||
from typing import Any, Callable
|
||||
from uuid import uuid4
|
||||
|
||||
from backend.dto.common import ApiError, ErrorCode
|
||||
|
|
@ -11,6 +11,13 @@ from backend.dto.material_dto import Material, MaterialCategory, MaterialListRes
|
|||
from core.audit_service import AuditService
|
||||
from core.identity import RequestContext
|
||||
from database.repositories.material_repository import MaterialRepository
|
||||
from engines.vision_provider import (
|
||||
MockVisionProvider,
|
||||
VisionProvider,
|
||||
VisionProviderError,
|
||||
VisionRecognitionRequest,
|
||||
VisionRecognitionResult,
|
||||
)
|
||||
|
||||
|
||||
MATERIAL_STATUSES = {"pending_upload", "pending_recognition", "pending_confirm", "confirmed", "archived"}
|
||||
|
|
@ -25,9 +32,18 @@ UPLOAD_ALLOWED_CONTENT_TYPES = {
|
|||
|
||||
|
||||
class MaterialService:
|
||||
def __init__(self, repository: MaterialRepository, audit_service: AuditService):
|
||||
def __init__(
|
||||
self,
|
||||
repository: MaterialRepository,
|
||||
audit_service: AuditService,
|
||||
*,
|
||||
vision_provider_factory: Callable[[dict[str, Any]], VisionProvider | None] | None = None,
|
||||
default_recognition_route: str = "material_vlm",
|
||||
):
|
||||
self.repository = repository
|
||||
self.audit_service = audit_service
|
||||
self.vision_provider_factory = vision_provider_factory
|
||||
self.default_recognition_route = default_recognition_route
|
||||
|
||||
def create_category(self, payload: dict[str, Any], ctx: RequestContext) -> MaterialCategory:
|
||||
name = str(payload.get("name") or "").strip()
|
||||
|
|
@ -189,6 +205,9 @@ class MaterialService:
|
|||
content_type = _normalize_upload_content_type(payload.get("content_type"))
|
||||
declared_size = _normalize_upload_size(payload)
|
||||
upload_metadata = _normalize_upload_metadata(payload.get("metadata"))
|
||||
recognition_route = _optional_text(payload.get("recognition_model_route"))
|
||||
if recognition_route:
|
||||
upload_metadata["recognition_model_route"] = recognition_route
|
||||
extension = _validate_upload_file_type(filename=filename, image_uri=image_uri, content_type=content_type)
|
||||
display_name = str(payload.get("display_name") or _display_name_from_filename(filename) or "Uploaded material").strip()
|
||||
thumbnail_uri = str(payload.get("thumbnail_uri") or image_uri).strip()
|
||||
|
|
@ -225,16 +244,25 @@ class MaterialService:
|
|||
return self._recognize_material(material, ctx)
|
||||
|
||||
def _recognize_material(self, material: Material, ctx: RequestContext) -> Material:
|
||||
generated_tags = _mock_tags(material)
|
||||
prompt = material.prompt or _mock_prompt(material)
|
||||
recognition, provider_error, fallback_used, route_name = self._recognize_with_adapter(material)
|
||||
generated_tags = recognition.tags
|
||||
prompt = material.prompt or recognition.prompt or _mock_prompt(material)
|
||||
after = self.update_material(
|
||||
material.material_id,
|
||||
{
|
||||
"prompt": prompt,
|
||||
"negative_prompt": material.negative_prompt or "body, face, background, watermark",
|
||||
"negative_prompt": material.negative_prompt or recognition.negative_prompt or "body, face, background, watermark",
|
||||
"tags": sorted(set(material.tags + generated_tags)),
|
||||
"recognition_status": "pending_confirm",
|
||||
"metadata": {**material.metadata, "recognition": {"engine": "mock_vlm", "confidence": 0.72}},
|
||||
"metadata": {
|
||||
**material.metadata,
|
||||
"recognition": _recognition_metadata(
|
||||
recognition,
|
||||
route_name=route_name,
|
||||
fallback_used=fallback_used,
|
||||
provider_error=provider_error,
|
||||
),
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
|
|
@ -248,6 +276,30 @@ class MaterialService:
|
|||
)
|
||||
return after
|
||||
|
||||
def _recognize_with_adapter(
|
||||
self,
|
||||
material: Material,
|
||||
) -> tuple[VisionRecognitionResult, dict[str, Any] | None, bool, str]:
|
||||
route_name = _recognition_route(material.metadata, self.default_recognition_route)
|
||||
request = _vision_request(material)
|
||||
provider = None
|
||||
if self.vision_provider_factory:
|
||||
provider = self.vision_provider_factory(
|
||||
{
|
||||
"route_name": route_name,
|
||||
"modality": "vision_recognition",
|
||||
"material_id": material.material_id,
|
||||
"image_uri": material.image_uri,
|
||||
"thumbnail_uri": material.thumbnail_uri,
|
||||
}
|
||||
)
|
||||
if provider is None:
|
||||
return _mock_vision_recognition(material), None, False, route_name
|
||||
try:
|
||||
return provider.recognize_material(request), None, False, route_name
|
||||
except VisionProviderError as exc:
|
||||
return _mock_vision_recognition(material), _vision_provider_error(exc), True, route_name
|
||||
|
||||
def confirm_material(self, material_id: str, payload: dict[str, Any], ctx: RequestContext) -> Material:
|
||||
material = self.get_material(material_id)
|
||||
update_payload = {
|
||||
|
|
@ -310,6 +362,65 @@ def _mock_tags(material: Material) -> list[str]:
|
|||
return tags
|
||||
|
||||
|
||||
def _vision_request(material: Material) -> VisionRecognitionRequest:
|
||||
return VisionRecognitionRequest(
|
||||
display_name=material.display_name,
|
||||
image_uri=str(material.image_uri or ""),
|
||||
thumbnail_uri=material.thumbnail_uri,
|
||||
prompt=material.prompt,
|
||||
negative_prompt=material.negative_prompt,
|
||||
tags=tuple(material.tags),
|
||||
metadata=material.metadata,
|
||||
)
|
||||
|
||||
|
||||
def _mock_vision_recognition(material: Material) -> VisionRecognitionResult:
|
||||
return MockVisionProvider().recognize_material(_vision_request(material))
|
||||
|
||||
|
||||
def _recognition_route(metadata: dict[str, Any], default_route: str) -> str:
|
||||
upload = metadata.get("upload") if isinstance(metadata.get("upload"), dict) else {}
|
||||
return (
|
||||
_optional_text(metadata.get("recognition_model_route"))
|
||||
or _optional_text(metadata.get("recognition_route"))
|
||||
or _optional_text(upload.get("recognition_model_route"))
|
||||
or default_route
|
||||
)
|
||||
|
||||
|
||||
def _recognition_metadata(
|
||||
recognition: VisionRecognitionResult,
|
||||
*,
|
||||
route_name: str,
|
||||
fallback_used: bool,
|
||||
provider_error: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
metadata = {
|
||||
"engine": "mock_vlm" if recognition.provider_id == "mock" else "vlm_adapter",
|
||||
"provider_id": recognition.provider_id,
|
||||
"model": recognition.model,
|
||||
"confidence": recognition.confidence,
|
||||
"route_name": route_name,
|
||||
"fallback_used": fallback_used,
|
||||
"description": recognition.description,
|
||||
"raw_metadata": recognition.raw_metadata,
|
||||
}
|
||||
if provider_error:
|
||||
metadata["provider_error"] = provider_error
|
||||
return metadata
|
||||
|
||||
|
||||
def _vision_provider_error(exc: VisionProviderError) -> dict[str, Any]:
|
||||
return {
|
||||
"source": "vision_provider",
|
||||
"provider_id": exc.provider_id,
|
||||
"status_code": exc.status_code,
|
||||
"error_type": exc.error_type,
|
||||
"message": str(exc),
|
||||
"details": exc.details,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_upload_filename(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
|
@ -406,3 +517,8 @@ def _display_name_from_filename(filename: str | None) -> str | None:
|
|||
if not filename:
|
||||
return None
|
||||
return PurePosixPath(filename).stem.replace("_", " ").strip() or None
|
||||
|
||||
|
||||
def _optional_text(value: Any) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
return text or None
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import Any
|
|||
from backend.services.model_provider_service import ModelProviderService
|
||||
from core.secret_service import SecretStore, resolve_secret_value
|
||||
from engines.image_provider import ImageProvider, ImageProviderConfig, OpenAIImageProvider
|
||||
from engines.vision_provider import OpenAIVisionProvider, VisionProvider, VisionProviderConfig
|
||||
|
||||
|
||||
class ModelProviderRuntime:
|
||||
|
|
@ -48,3 +49,39 @@ class ModelProviderRuntime:
|
|||
extra_headers=dict(provider.get("extra_headers") or {}),
|
||||
)
|
||||
)
|
||||
|
||||
def vision_provider_for_route(self, route_context: dict[str, Any]) -> VisionProvider | None:
|
||||
route_name = str(route_context.get("route_name") or "").strip()
|
||||
if not route_name:
|
||||
return None
|
||||
resolved = self.model_provider_service.resolve_route(route_name, modality="vision_recognition")
|
||||
if not resolved:
|
||||
return None
|
||||
provider = resolved.get("provider") or {}
|
||||
if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible":
|
||||
return None
|
||||
secret = resolve_secret_value(
|
||||
env_var=provider.get("api_key_env_var"),
|
||||
secret_ref=provider.get("api_key_secret_ref"),
|
||||
secret_store=self.secret_store,
|
||||
)
|
||||
if not secret.value:
|
||||
return None
|
||||
settings = dict(resolved.get("settings") or {})
|
||||
capabilities = dict(provider.get("capabilities") or {})
|
||||
return OpenAIVisionProvider(
|
||||
VisionProviderConfig(
|
||||
provider_id=str(resolved["provider_id"]),
|
||||
base_url=str(provider.get("base_url") or ""),
|
||||
api_key=secret.value,
|
||||
vision_model=str(resolved["model_id"]),
|
||||
vision_endpoint=str(
|
||||
settings.get("vision_endpoint")
|
||||
or capabilities.get("vision_endpoint")
|
||||
or "/v1/chat/completions"
|
||||
),
|
||||
provider_type=str(provider.get("provider_type") or "openai_compatible"),
|
||||
timeout_seconds=int(provider.get("timeout_seconds") or 180),
|
||||
extra_headers=dict(provider.get("extra_headers") or {}),
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,14 @@ from engines.image_provider import (
|
|||
MockImageProvider,
|
||||
OpenAIImageProvider,
|
||||
)
|
||||
from engines.vision_provider import (
|
||||
MockVisionProvider,
|
||||
OpenAIVisionProvider,
|
||||
VisionProviderConfig,
|
||||
VisionProviderError,
|
||||
VisionRecognitionRequest,
|
||||
VisionRecognitionResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GeneratedImagePayload",
|
||||
|
|
@ -17,5 +25,11 @@ __all__ = [
|
|||
"ImageProviderConfig",
|
||||
"ImageProviderError",
|
||||
"MockImageProvider",
|
||||
"MockVisionProvider",
|
||||
"OpenAIImageProvider",
|
||||
"OpenAIVisionProvider",
|
||||
"VisionProviderConfig",
|
||||
"VisionProviderError",
|
||||
"VisionRecognitionRequest",
|
||||
"VisionRecognitionResult",
|
||||
]
|
||||
|
|
|
|||
309
engines/vision_provider.py
普通文件
309
engines/vision_provider.py
普通文件
|
|
@ -0,0 +1,309 @@
|
|||
"""Vision provider adapter interfaces for material recognition."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Protocol
|
||||
|
||||
|
||||
JsonDict = dict[str, Any]
|
||||
HttpTransport = Callable[[str, JsonDict, dict[str, str], int], JsonDict]
|
||||
|
||||
|
||||
class VisionProvider(Protocol):
|
||||
provider_id: str
|
||||
|
||||
def recognize_material(self, request: "VisionRecognitionRequest") -> "VisionRecognitionResult":
|
||||
...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VisionProviderConfig:
|
||||
provider_id: str
|
||||
base_url: str
|
||||
api_key: str = ""
|
||||
vision_model: str = ""
|
||||
vision_endpoint: str = "/v1/chat/completions"
|
||||
provider_type: str = "openai_compatible"
|
||||
timeout_seconds: int = 180
|
||||
extra_headers: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def endpoint_url(self) -> str:
|
||||
base = self.base_url.rstrip("/")
|
||||
endpoint = self.vision_endpoint if self.vision_endpoint.startswith("/") else f"/{self.vision_endpoint}"
|
||||
return f"{base}{endpoint}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VisionRecognitionRequest:
|
||||
display_name: str
|
||||
image_uri: str
|
||||
thumbnail_uri: str | None = None
|
||||
prompt: str | None = None
|
||||
negative_prompt: str | None = None
|
||||
tags: tuple[str, ...] = ()
|
||||
model: str | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VisionRecognitionResult:
|
||||
provider_id: str
|
||||
model: str
|
||||
prompt: str
|
||||
negative_prompt: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
confidence: float | None = None
|
||||
description: str | None = None
|
||||
raw_metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class VisionProviderError(RuntimeError):
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
provider_id: str,
|
||||
status_code: int | None = None,
|
||||
error_type: str = "provider_error",
|
||||
details: dict[str, Any] | None = None,
|
||||
):
|
||||
super().__init__(message)
|
||||
self.provider_id = provider_id
|
||||
self.status_code = status_code
|
||||
self.error_type = error_type
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
class OpenAIVisionProvider:
|
||||
def __init__(self, config: VisionProviderConfig, transport: HttpTransport | None = None):
|
||||
self.config = config
|
||||
self.provider_id = config.provider_id
|
||||
self._transport = transport or _post_json
|
||||
|
||||
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
|
||||
model = request.model or self.config.vision_model
|
||||
if not model:
|
||||
raise VisionProviderError(
|
||||
"vision model is required",
|
||||
provider_id=self.provider_id,
|
||||
error_type="configuration_error",
|
||||
)
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"Return compact JSON for a reusable outfit material with keys: "
|
||||
"prompt, negative_prompt, tags, confidence, description."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": _user_content(request)},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.config.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
**self.config.extra_headers,
|
||||
}
|
||||
try:
|
||||
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
|
||||
except VisionProviderError as exc:
|
||||
if exc.provider_id != "unknown":
|
||||
raise
|
||||
raise VisionProviderError(
|
||||
str(exc),
|
||||
provider_id=self.provider_id,
|
||||
status_code=exc.status_code,
|
||||
error_type=exc.error_type,
|
||||
details=exc.details,
|
||||
) from exc
|
||||
return _parse_vision_response(self.provider_id, model, response)
|
||||
|
||||
|
||||
class MockVisionProvider:
|
||||
provider_id = "mock"
|
||||
|
||||
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
|
||||
text = f"{request.display_name} {request.prompt or ''}".lower()
|
||||
tags = ["material"]
|
||||
for keyword in ("white", "blue", "black", "red", "ribbon", "shirt", "blouse", "dress", "skirt", "shoe"):
|
||||
if keyword in text:
|
||||
tags.append(keyword)
|
||||
base = request.display_name.strip().lower().replace("_", " ") or "uploaded material"
|
||||
return VisionRecognitionResult(
|
||||
provider_id=self.provider_id,
|
||||
model=request.model or "mock-vlm",
|
||||
prompt=f"{base}, clean product reference, transparent background",
|
||||
negative_prompt="body, face, background, watermark",
|
||||
tags=tags,
|
||||
confidence=0.72,
|
||||
raw_metadata={"mock": True},
|
||||
)
|
||||
|
||||
|
||||
def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]:
|
||||
content: list[dict[str, Any]] = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"Material name: {request.display_name}\n"
|
||||
f"Known prompt: {request.prompt or ''}\n"
|
||||
f"Known negative prompt: {request.negative_prompt or ''}\n"
|
||||
f"Known tags: {', '.join(request.tags)}"
|
||||
),
|
||||
}
|
||||
]
|
||||
image_url = _remote_image_url(request.thumbnail_uri) or _remote_image_url(request.image_uri)
|
||||
if image_url:
|
||||
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
||||
else:
|
||||
content[0]["text"] += f"\nStorage image URI: {request.image_uri}"
|
||||
return content
|
||||
|
||||
|
||||
def _remote_image_url(value: str | None) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
if text.startswith(("http://", "https://", "data:image/")):
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def _post_json(url: str, payload: JsonDict, headers: dict[str, str], timeout_seconds: int) -> JsonDict:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8-sig")
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise VisionProviderError(
|
||||
"vision provider returned an HTTP error",
|
||||
provider_id="unknown",
|
||||
status_code=exc.code,
|
||||
error_type="http_error",
|
||||
details={"body_preview": body[:500]},
|
||||
) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise VisionProviderError(
|
||||
"vision provider network error",
|
||||
provider_id="unknown",
|
||||
error_type="network_error",
|
||||
details={"reason": str(exc.reason)},
|
||||
) from exc
|
||||
try:
|
||||
parsed = json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise VisionProviderError(
|
||||
"vision provider returned invalid JSON",
|
||||
provider_id="unknown",
|
||||
error_type="invalid_response",
|
||||
details={"body_preview": body[:500]},
|
||||
) from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise VisionProviderError("vision provider response must be an object", provider_id="unknown")
|
||||
return parsed
|
||||
|
||||
|
||||
def _parse_vision_response(provider_id: str, model: str, response: JsonDict) -> VisionRecognitionResult:
|
||||
content = _response_content(response)
|
||||
parsed = _json_from_text(content)
|
||||
prompt = _optional_text(parsed.get("prompt") or parsed.get("positive_prompt") or parsed.get("description"))
|
||||
if not prompt:
|
||||
raise VisionProviderError(
|
||||
"vision provider response must include prompt or description",
|
||||
provider_id=provider_id,
|
||||
error_type="invalid_response",
|
||||
details={"content_preview": content[:500]},
|
||||
)
|
||||
return VisionRecognitionResult(
|
||||
provider_id=provider_id,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
negative_prompt=_optional_text(parsed.get("negative_prompt")),
|
||||
tags=_string_list(parsed.get("tags")),
|
||||
confidence=_float_or_none(parsed.get("confidence")),
|
||||
description=_optional_text(parsed.get("description")),
|
||||
raw_metadata={"response_keys": sorted(str(key) for key in response.keys())},
|
||||
)
|
||||
|
||||
|
||||
def _response_content(response: JsonDict) -> str:
|
||||
output_text = _optional_text(response.get("output_text"))
|
||||
if output_text:
|
||||
return output_text
|
||||
choices = response.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
first = choices[0] if isinstance(choices[0], dict) else {}
|
||||
message = first.get("message") if isinstance(first.get("message"), dict) else {}
|
||||
return _content_to_text(message.get("content"))
|
||||
output = response.get("output")
|
||||
if isinstance(output, list):
|
||||
return " ".join(_content_to_text(item.get("content")) for item in output if isinstance(item, dict)).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _content_to_text(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, list):
|
||||
parts = []
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
text = item.get("text") or item.get("content")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
return " ".join(parts).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _json_from_text(value: str) -> JsonDict:
|
||||
text = value.strip()
|
||||
if not text:
|
||||
raise VisionProviderError("vision provider response was empty", provider_id="unknown", error_type="invalid_response")
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
raise VisionProviderError(
|
||||
"vision provider response was not JSON",
|
||||
provider_id="unknown",
|
||||
error_type="invalid_response",
|
||||
details={"content_preview": text[:500]},
|
||||
) from None
|
||||
parsed = json.loads(text[start : end + 1])
|
||||
if not isinstance(parsed, dict):
|
||||
raise VisionProviderError("vision provider JSON must be an object", provider_id="unknown", error_type="invalid_response")
|
||||
return parsed
|
||||
|
||||
|
||||
def _string_list(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
values = value.split(",")
|
||||
elif isinstance(value, list):
|
||||
values = value
|
||||
else:
|
||||
values = []
|
||||
return sorted({str(item).strip() for item in values if str(item).strip()})
|
||||
|
||||
|
||||
def _float_or_none(value: Any) -> float | None:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return min(max(number, 0.0), 1.0)
|
||||
|
||||
|
||||
def _optional_text(value: Any) -> str | None:
|
||||
text = str(value or "").strip()
|
||||
return text or None
|
||||
|
|
@ -25,6 +25,7 @@ from core.config_service import ConfigService
|
|||
from core.identity import RequestContext
|
||||
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
||||
from database.repositories.material_repository import MaterialRepository
|
||||
from engines.vision_provider import VisionProviderError, VisionRecognitionResult
|
||||
|
||||
|
||||
class MaterialServiceTests(unittest.TestCase):
|
||||
|
|
@ -153,6 +154,8 @@ storage:
|
|||
self.assertEqual(recognized.recognition_status, "pending_confirm")
|
||||
self.assertIn("white", recognized.tags)
|
||||
self.assertIn("clean product reference", recognized.prompt)
|
||||
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock")
|
||||
self.assertFalse(recognized.metadata["recognition"]["fallback_used"])
|
||||
|
||||
confirmed = self.material_service.confirm_material(
|
||||
uploaded.material_id,
|
||||
|
|
@ -170,6 +173,89 @@ storage:
|
|||
self.assertEqual(re_recognized.recognition_status, "pending_confirm")
|
||||
self.assertIn("white", re_recognized.tags)
|
||||
|
||||
def test_recognize_material_uses_injected_vlm_adapter(self):
|
||||
class FakeVisionProvider:
|
||||
provider_id = "fake_vlm"
|
||||
|
||||
def __init__(self):
|
||||
self.requests = []
|
||||
|
||||
def recognize_material(self, request):
|
||||
self.requests.append(request)
|
||||
return VisionRecognitionResult(
|
||||
provider_id=self.provider_id,
|
||||
model="vision-test",
|
||||
prompt="white lace blouse, product reference",
|
||||
negative_prompt="body",
|
||||
tags=["white", "lace", "upper"],
|
||||
confidence=0.93,
|
||||
description="white lace blouse",
|
||||
raw_metadata={"test": True},
|
||||
)
|
||||
|
||||
provider = FakeVisionProvider()
|
||||
service = MaterialService(
|
||||
MaterialRepository(self.database),
|
||||
self.audit_service,
|
||||
vision_provider_factory=lambda route: provider,
|
||||
)
|
||||
category = service.create_category({"name": "Upper"}, self.ctx)
|
||||
uploaded = service.upload_material(
|
||||
{
|
||||
"category_id": category.category_id,
|
||||
"filename": "white_blouse.png",
|
||||
"image_uri": "storage://materials/white_blouse.png",
|
||||
"recognition_model_route": "material_vlm",
|
||||
},
|
||||
self.ctx,
|
||||
)
|
||||
|
||||
recognized = service.recognize_material(uploaded.material_id, self.ctx)
|
||||
|
||||
self.assertEqual(provider.requests[0].display_name, "white blouse")
|
||||
self.assertEqual(recognized.prompt, "white lace blouse, product reference")
|
||||
self.assertEqual(recognized.negative_prompt, "body")
|
||||
self.assertIn("lace", recognized.tags)
|
||||
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "fake_vlm")
|
||||
self.assertEqual(recognized.metadata["recognition"]["model"], "vision-test")
|
||||
self.assertEqual(recognized.metadata["recognition"]["route_name"], "material_vlm")
|
||||
self.assertFalse(recognized.metadata["recognition"]["fallback_used"])
|
||||
|
||||
def test_recognize_material_falls_back_when_vlm_adapter_fails(self):
|
||||
class FailingVisionProvider:
|
||||
provider_id = "broken_vlm"
|
||||
|
||||
def recognize_material(self, request):
|
||||
raise VisionProviderError(
|
||||
"vision endpoint timeout",
|
||||
provider_id=self.provider_id,
|
||||
error_type="timeout",
|
||||
details={"retryable": True},
|
||||
)
|
||||
|
||||
service = MaterialService(
|
||||
MaterialRepository(self.database),
|
||||
self.audit_service,
|
||||
vision_provider_factory=lambda route: FailingVisionProvider(),
|
||||
)
|
||||
category = service.create_category({"name": "Upper"}, self.ctx)
|
||||
uploaded = service.upload_material(
|
||||
{
|
||||
"category_id": category.category_id,
|
||||
"filename": "white_blouse.png",
|
||||
"image_uri": "storage://materials/white_blouse.png",
|
||||
},
|
||||
self.ctx,
|
||||
)
|
||||
|
||||
recognized = service.recognize_material(uploaded.material_id, self.ctx)
|
||||
|
||||
self.assertIn("white", recognized.tags)
|
||||
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock")
|
||||
self.assertTrue(recognized.metadata["recognition"]["fallback_used"])
|
||||
self.assertEqual(recognized.metadata["recognition"]["provider_error"]["provider_id"], "broken_vlm")
|
||||
self.assertEqual(recognized.metadata["recognition"]["provider_error"]["error_type"], "timeout")
|
||||
|
||||
def test_upload_rejects_unsafe_filename(self):
|
||||
category = self.material_service.create_category({"name": "Upper"}, self.ctx)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,40 @@ class ModelProviderRuntimeTests(unittest.TestCase):
|
|||
self.assertEqual(provider.config.image_model, "image2")
|
||||
self.assertEqual(provider.config.api_key, "secret-value")
|
||||
|
||||
def test_runtime_builds_openai_vision_provider_from_route_settings(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
database = Database(DatabaseSettings(sqlite=SqliteSettings(path=str(Path(tmp) / "imagestudio.db"))))
|
||||
database.initialize()
|
||||
service = ModelProviderService(ModelProviderRepository(database))
|
||||
service.upsert_provider(
|
||||
{
|
||||
"provider_id": "site_vlm",
|
||||
"base_url": "https://vision.example",
|
||||
"api_key_env_var": "IMAGESTUDIO_TEST_VLM",
|
||||
"timeout_seconds": 90,
|
||||
}
|
||||
)
|
||||
service.upsert_route(
|
||||
{
|
||||
"route_name": "material_vlm",
|
||||
"provider_id": "site_vlm",
|
||||
"model_id": "vision-model",
|
||||
"modality": "vision_recognition",
|
||||
"settings": {"vision_endpoint": "/v1/chat/completions"},
|
||||
}
|
||||
)
|
||||
os.environ["IMAGESTUDIO_TEST_VLM"] = "vision-secret"
|
||||
try:
|
||||
provider = ModelProviderRuntime(service).vision_provider_for_route({"route_name": "material_vlm"})
|
||||
finally:
|
||||
os.environ.pop("IMAGESTUDIO_TEST_VLM", None)
|
||||
|
||||
self.assertIsNotNone(provider)
|
||||
self.assertEqual(provider.config.provider_id, "site_vlm")
|
||||
self.assertEqual(provider.config.vision_model, "vision-model")
|
||||
self.assertEqual(provider.config.vision_endpoint, "/v1/chat/completions")
|
||||
self.assertEqual(provider.config.api_key, "vision-secret")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
103
tests/test_vision_provider.py
普通文件
103
tests/test_vision_provider.py
普通文件
|
|
@ -0,0 +1,103 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from engines.vision_provider import (
|
||||
OpenAIVisionProvider,
|
||||
VisionProviderConfig,
|
||||
VisionProviderError,
|
||||
VisionRecognitionRequest,
|
||||
)
|
||||
|
||||
|
||||
class VisionProviderTests(unittest.TestCase):
|
||||
def test_openai_vision_provider_normalizes_recognition_response(self):
|
||||
calls = []
|
||||
|
||||
def transport(url, payload, headers, timeout):
|
||||
calls.append((url, payload, headers, timeout))
|
||||
return {
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": (
|
||||
'{"prompt":"white blouse, clean product reference",'
|
||||
'"negative_prompt":"body, watermark",'
|
||||
'"tags":["white","blouse"],'
|
||||
'"confidence":0.91,'
|
||||
'"description":"white blouse"}'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
provider = OpenAIVisionProvider(
|
||||
VisionProviderConfig(
|
||||
provider_id="site_vlm",
|
||||
base_url="https://example.test",
|
||||
api_key="secret",
|
||||
vision_model="vision-model",
|
||||
vision_endpoint="/v1/chat/completions",
|
||||
timeout_seconds=120,
|
||||
),
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
result = provider.recognize_material(
|
||||
VisionRecognitionRequest(
|
||||
display_name="White blouse",
|
||||
image_uri="https://example.test/white.png",
|
||||
tags=("upper",),
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(result.provider_id, "site_vlm")
|
||||
self.assertEqual(result.model, "vision-model")
|
||||
self.assertEqual(result.prompt, "white blouse, clean product reference")
|
||||
self.assertEqual(result.negative_prompt, "body, watermark")
|
||||
self.assertEqual(result.tags, ["blouse", "white"])
|
||||
self.assertEqual(result.confidence, 0.91)
|
||||
self.assertEqual(calls[0][0], "https://example.test/v1/chat/completions")
|
||||
self.assertEqual(calls[0][1]["model"], "vision-model")
|
||||
self.assertEqual(calls[0][1]["messages"][1]["content"][1]["image_url"]["url"], "https://example.test/white.png")
|
||||
self.assertEqual(calls[0][3], 120)
|
||||
|
||||
def test_provider_requires_model(self):
|
||||
provider = OpenAIVisionProvider(VisionProviderConfig(provider_id="missing", base_url="https://example.test"))
|
||||
|
||||
with self.assertRaises(VisionProviderError) as raised:
|
||||
provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png"))
|
||||
|
||||
self.assertEqual(raised.exception.error_type, "configuration_error")
|
||||
|
||||
def test_provider_relabels_transport_errors_with_configured_provider(self):
|
||||
def transport(url, payload, headers, timeout):
|
||||
raise VisionProviderError(
|
||||
"remote unauthorized",
|
||||
provider_id="unknown",
|
||||
status_code=401,
|
||||
error_type="http_error",
|
||||
details={"body_preview": "invalid credentials"},
|
||||
)
|
||||
|
||||
provider = OpenAIVisionProvider(
|
||||
VisionProviderConfig(
|
||||
provider_id="site_vlm",
|
||||
base_url="https://example.test",
|
||||
api_key="secret",
|
||||
vision_model="vision-model",
|
||||
),
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
with self.assertRaises(VisionProviderError) as raised:
|
||||
provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png"))
|
||||
|
||||
self.assertEqual(raised.exception.provider_id, "site_vlm")
|
||||
self.assertEqual(raised.exception.status_code, 401)
|
||||
self.assertEqual(raised.exception.error_type, "http_error")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
正在加载…
添加表格
在新工单中引用