From 0df3546c6d87cce422b64c91eda39ea633a59de5 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Mon, 6 Jul 2026 03:37:17 +0800 Subject: [PATCH] =?UTF-8?q?S100=20VLM=E9=80=82=E9=85=8D=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/material_routes.py | 6 + backend/services/material_service.py | 128 ++++++++- backend/services/model_provider_runtime.py | 37 +++ engines/__init__.py | 14 + engines/vision_provider.py | 309 +++++++++++++++++++++ tests/test_material_service.py | 86 ++++++ tests/test_model_provider_runtime.py | 34 +++ tests/test_vision_provider.py | 103 +++++++ 8 files changed, 711 insertions(+), 6 deletions(-) create mode 100644 engines/vision_provider.py create mode 100644 tests/test_vision_provider.py diff --git a/backend/routes/material_routes.py b/backend/routes/material_routes.py index 0d94f84..760914b 100644 --- a/backend/routes/material_routes.py +++ b/backend/routes/material_routes.py @@ -9,11 +9,14 @@ from backend.dto.material_dto import Material, MaterialListResult from backend.routes.common import handle_request from backend.services.material_service import MaterialService from backend.services.media_service import MediaService +from backend.services.model_provider_runtime import ModelProviderRuntime +from backend.services.model_provider_service import ModelProviderService from core.audit_service import AuditService from core.identity import RequestContext from core.security import require_authenticated, require_permission from database.db_helper import Database, load_database_settings from database.repositories.material_repository import MaterialRepository +from database.repositories.model_provider_repository import ModelProviderRepository def list_material_categories( @@ -227,9 +230,12 @@ def confirm_material( def _create_material_service() -> MaterialService: database = Database(load_database_settings()) + model_provider_service = ModelProviderService(ModelProviderRepository(database)) + model_provider_runtime = ModelProviderRuntime(model_provider_service) return MaterialService( repository=MaterialRepository(database), audit_service=AuditService(database), + vision_provider_factory=model_provider_runtime.vision_provider_for_route, ) diff --git a/backend/services/material_service.py b/backend/services/material_service.py index 7f60a58..2ba89ad 100644 --- a/backend/services/material_service.py +++ b/backend/services/material_service.py @@ -3,7 +3,7 @@ from __future__ import annotations from pathlib import PurePosixPath -from typing import Any +from typing import Any, Callable from uuid import uuid4 from backend.dto.common import ApiError, ErrorCode @@ -11,6 +11,13 @@ from backend.dto.material_dto import Material, MaterialCategory, MaterialListRes from core.audit_service import AuditService from core.identity import RequestContext from database.repositories.material_repository import MaterialRepository +from engines.vision_provider import ( + MockVisionProvider, + VisionProvider, + VisionProviderError, + VisionRecognitionRequest, + VisionRecognitionResult, +) MATERIAL_STATUSES = {"pending_upload", "pending_recognition", "pending_confirm", "confirmed", "archived"} @@ -25,9 +32,18 @@ UPLOAD_ALLOWED_CONTENT_TYPES = { class MaterialService: - def __init__(self, repository: MaterialRepository, audit_service: AuditService): + def __init__( + self, + repository: MaterialRepository, + audit_service: AuditService, + *, + vision_provider_factory: Callable[[dict[str, Any]], VisionProvider | None] | None = None, + default_recognition_route: str = "material_vlm", + ): self.repository = repository self.audit_service = audit_service + self.vision_provider_factory = vision_provider_factory + self.default_recognition_route = default_recognition_route def create_category(self, payload: dict[str, Any], ctx: RequestContext) -> MaterialCategory: name = str(payload.get("name") or "").strip() @@ -189,6 +205,9 @@ class MaterialService: content_type = _normalize_upload_content_type(payload.get("content_type")) declared_size = _normalize_upload_size(payload) upload_metadata = _normalize_upload_metadata(payload.get("metadata")) + recognition_route = _optional_text(payload.get("recognition_model_route")) + if recognition_route: + upload_metadata["recognition_model_route"] = recognition_route extension = _validate_upload_file_type(filename=filename, image_uri=image_uri, content_type=content_type) display_name = str(payload.get("display_name") or _display_name_from_filename(filename) or "Uploaded material").strip() thumbnail_uri = str(payload.get("thumbnail_uri") or image_uri).strip() @@ -225,16 +244,25 @@ class MaterialService: return self._recognize_material(material, ctx) def _recognize_material(self, material: Material, ctx: RequestContext) -> Material: - generated_tags = _mock_tags(material) - prompt = material.prompt or _mock_prompt(material) + recognition, provider_error, fallback_used, route_name = self._recognize_with_adapter(material) + generated_tags = recognition.tags + prompt = material.prompt or recognition.prompt or _mock_prompt(material) after = self.update_material( material.material_id, { "prompt": prompt, - "negative_prompt": material.negative_prompt or "body, face, background, watermark", + "negative_prompt": material.negative_prompt or recognition.negative_prompt or "body, face, background, watermark", "tags": sorted(set(material.tags + generated_tags)), "recognition_status": "pending_confirm", - "metadata": {**material.metadata, "recognition": {"engine": "mock_vlm", "confidence": 0.72}}, + "metadata": { + **material.metadata, + "recognition": _recognition_metadata( + recognition, + route_name=route_name, + fallback_used=fallback_used, + provider_error=provider_error, + ), + }, }, ctx, ) @@ -248,6 +276,30 @@ class MaterialService: ) return after + def _recognize_with_adapter( + self, + material: Material, + ) -> tuple[VisionRecognitionResult, dict[str, Any] | None, bool, str]: + route_name = _recognition_route(material.metadata, self.default_recognition_route) + request = _vision_request(material) + provider = None + if self.vision_provider_factory: + provider = self.vision_provider_factory( + { + "route_name": route_name, + "modality": "vision_recognition", + "material_id": material.material_id, + "image_uri": material.image_uri, + "thumbnail_uri": material.thumbnail_uri, + } + ) + if provider is None: + return _mock_vision_recognition(material), None, False, route_name + try: + return provider.recognize_material(request), None, False, route_name + except VisionProviderError as exc: + return _mock_vision_recognition(material), _vision_provider_error(exc), True, route_name + def confirm_material(self, material_id: str, payload: dict[str, Any], ctx: RequestContext) -> Material: material = self.get_material(material_id) update_payload = { @@ -310,6 +362,65 @@ def _mock_tags(material: Material) -> list[str]: return tags +def _vision_request(material: Material) -> VisionRecognitionRequest: + return VisionRecognitionRequest( + display_name=material.display_name, + image_uri=str(material.image_uri or ""), + thumbnail_uri=material.thumbnail_uri, + prompt=material.prompt, + negative_prompt=material.negative_prompt, + tags=tuple(material.tags), + metadata=material.metadata, + ) + + +def _mock_vision_recognition(material: Material) -> VisionRecognitionResult: + return MockVisionProvider().recognize_material(_vision_request(material)) + + +def _recognition_route(metadata: dict[str, Any], default_route: str) -> str: + upload = metadata.get("upload") if isinstance(metadata.get("upload"), dict) else {} + return ( + _optional_text(metadata.get("recognition_model_route")) + or _optional_text(metadata.get("recognition_route")) + or _optional_text(upload.get("recognition_model_route")) + or default_route + ) + + +def _recognition_metadata( + recognition: VisionRecognitionResult, + *, + route_name: str, + fallback_used: bool, + provider_error: dict[str, Any] | None, +) -> dict[str, Any]: + metadata = { + "engine": "mock_vlm" if recognition.provider_id == "mock" else "vlm_adapter", + "provider_id": recognition.provider_id, + "model": recognition.model, + "confidence": recognition.confidence, + "route_name": route_name, + "fallback_used": fallback_used, + "description": recognition.description, + "raw_metadata": recognition.raw_metadata, + } + if provider_error: + metadata["provider_error"] = provider_error + return metadata + + +def _vision_provider_error(exc: VisionProviderError) -> dict[str, Any]: + return { + "source": "vision_provider", + "provider_id": exc.provider_id, + "status_code": exc.status_code, + "error_type": exc.error_type, + "message": str(exc), + "details": exc.details, + } + + def _normalize_upload_filename(value: Any) -> str | None: if value is None: return None @@ -406,3 +517,8 @@ def _display_name_from_filename(filename: str | None) -> str | None: if not filename: return None return PurePosixPath(filename).stem.replace("_", " ").strip() or None + + +def _optional_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None diff --git a/backend/services/model_provider_runtime.py b/backend/services/model_provider_runtime.py index 6f27da8..f22d3de 100644 --- a/backend/services/model_provider_runtime.py +++ b/backend/services/model_provider_runtime.py @@ -7,6 +7,7 @@ from typing import Any from backend.services.model_provider_service import ModelProviderService from core.secret_service import SecretStore, resolve_secret_value from engines.image_provider import ImageProvider, ImageProviderConfig, OpenAIImageProvider +from engines.vision_provider import OpenAIVisionProvider, VisionProvider, VisionProviderConfig class ModelProviderRuntime: @@ -48,3 +49,39 @@ class ModelProviderRuntime: extra_headers=dict(provider.get("extra_headers") or {}), ) ) + + def vision_provider_for_route(self, route_context: dict[str, Any]) -> VisionProvider | None: + route_name = str(route_context.get("route_name") or "").strip() + if not route_name: + return None + resolved = self.model_provider_service.resolve_route(route_name, modality="vision_recognition") + if not resolved: + return None + provider = resolved.get("provider") or {} + if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible": + return None + secret = resolve_secret_value( + env_var=provider.get("api_key_env_var"), + secret_ref=provider.get("api_key_secret_ref"), + secret_store=self.secret_store, + ) + if not secret.value: + return None + settings = dict(resolved.get("settings") or {}) + capabilities = dict(provider.get("capabilities") or {}) + return OpenAIVisionProvider( + VisionProviderConfig( + provider_id=str(resolved["provider_id"]), + base_url=str(provider.get("base_url") or ""), + api_key=secret.value, + vision_model=str(resolved["model_id"]), + vision_endpoint=str( + settings.get("vision_endpoint") + or capabilities.get("vision_endpoint") + or "/v1/chat/completions" + ), + provider_type=str(provider.get("provider_type") or "openai_compatible"), + timeout_seconds=int(provider.get("timeout_seconds") or 180), + extra_headers=dict(provider.get("extra_headers") or {}), + ) + ) diff --git a/engines/__init__.py b/engines/__init__.py index 039a023..cee7347 100644 --- a/engines/__init__.py +++ b/engines/__init__.py @@ -9,6 +9,14 @@ from engines.image_provider import ( MockImageProvider, OpenAIImageProvider, ) +from engines.vision_provider import ( + MockVisionProvider, + OpenAIVisionProvider, + VisionProviderConfig, + VisionProviderError, + VisionRecognitionRequest, + VisionRecognitionResult, +) __all__ = [ "GeneratedImagePayload", @@ -17,5 +25,11 @@ __all__ = [ "ImageProviderConfig", "ImageProviderError", "MockImageProvider", + "MockVisionProvider", "OpenAIImageProvider", + "OpenAIVisionProvider", + "VisionProviderConfig", + "VisionProviderError", + "VisionRecognitionRequest", + "VisionRecognitionResult", ] diff --git a/engines/vision_provider.py b/engines/vision_provider.py new file mode 100644 index 0000000..c2c2df7 --- /dev/null +++ b/engines/vision_provider.py @@ -0,0 +1,309 @@ +"""Vision provider adapter interfaces for material recognition.""" + +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Callable, Protocol + + +JsonDict = dict[str, Any] +HttpTransport = Callable[[str, JsonDict, dict[str, str], int], JsonDict] + + +class VisionProvider(Protocol): + provider_id: str + + def recognize_material(self, request: "VisionRecognitionRequest") -> "VisionRecognitionResult": + ... + + +@dataclass(frozen=True) +class VisionProviderConfig: + provider_id: str + base_url: str + api_key: str = "" + vision_model: str = "" + vision_endpoint: str = "/v1/chat/completions" + provider_type: str = "openai_compatible" + timeout_seconds: int = 180 + extra_headers: dict[str, str] = field(default_factory=dict) + + def endpoint_url(self) -> str: + base = self.base_url.rstrip("/") + endpoint = self.vision_endpoint if self.vision_endpoint.startswith("/") else f"/{self.vision_endpoint}" + return f"{base}{endpoint}" + + +@dataclass(frozen=True) +class VisionRecognitionRequest: + display_name: str + image_uri: str + thumbnail_uri: str | None = None + prompt: str | None = None + negative_prompt: str | None = None + tags: tuple[str, ...] = () + model: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class VisionRecognitionResult: + provider_id: str + model: str + prompt: str + negative_prompt: str | None = None + tags: list[str] = field(default_factory=list) + confidence: float | None = None + description: str | None = None + raw_metadata: dict[str, Any] = field(default_factory=dict) + + +class VisionProviderError(RuntimeError): + def __init__( + self, + message: str, + *, + provider_id: str, + status_code: int | None = None, + error_type: str = "provider_error", + details: dict[str, Any] | None = None, + ): + super().__init__(message) + self.provider_id = provider_id + self.status_code = status_code + self.error_type = error_type + self.details = details or {} + + +class OpenAIVisionProvider: + def __init__(self, config: VisionProviderConfig, transport: HttpTransport | None = None): + self.config = config + self.provider_id = config.provider_id + self._transport = transport or _post_json + + def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult: + model = request.model or self.config.vision_model + if not model: + raise VisionProviderError( + "vision model is required", + provider_id=self.provider_id, + error_type="configuration_error", + ) + payload = { + "model": model, + "messages": [ + { + "role": "system", + "content": ( + "Return compact JSON for a reusable outfit material with keys: " + "prompt, negative_prompt, tags, confidence, description." + ), + }, + {"role": "user", "content": _user_content(request)}, + ], + "temperature": 0.1, + "response_format": {"type": "json_object"}, + } + headers = { + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + **self.config.extra_headers, + } + try: + response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds) + except VisionProviderError as exc: + if exc.provider_id != "unknown": + raise + raise VisionProviderError( + str(exc), + provider_id=self.provider_id, + status_code=exc.status_code, + error_type=exc.error_type, + details=exc.details, + ) from exc + return _parse_vision_response(self.provider_id, model, response) + + +class MockVisionProvider: + provider_id = "mock" + + def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult: + text = f"{request.display_name} {request.prompt or ''}".lower() + tags = ["material"] + for keyword in ("white", "blue", "black", "red", "ribbon", "shirt", "blouse", "dress", "skirt", "shoe"): + if keyword in text: + tags.append(keyword) + base = request.display_name.strip().lower().replace("_", " ") or "uploaded material" + return VisionRecognitionResult( + provider_id=self.provider_id, + model=request.model or "mock-vlm", + prompt=f"{base}, clean product reference, transparent background", + negative_prompt="body, face, background, watermark", + tags=tags, + confidence=0.72, + raw_metadata={"mock": True}, + ) + + +def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]: + content: list[dict[str, Any]] = [ + { + "type": "text", + "text": ( + f"Material name: {request.display_name}\n" + f"Known prompt: {request.prompt or ''}\n" + f"Known negative prompt: {request.negative_prompt or ''}\n" + f"Known tags: {', '.join(request.tags)}" + ), + } + ] + image_url = _remote_image_url(request.thumbnail_uri) or _remote_image_url(request.image_uri) + if image_url: + content.append({"type": "image_url", "image_url": {"url": image_url}}) + else: + content[0]["text"] += f"\nStorage image URI: {request.image_uri}" + return content + + +def _remote_image_url(value: str | None) -> str | None: + text = str(value or "").strip() + if text.startswith(("http://", "https://", "data:image/")): + return text + return None + + +def _post_json(url: str, payload: JsonDict, headers: dict[str, str], timeout_seconds: int) -> JsonDict: + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + request = urllib.request.Request(url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + body = response.read().decode("utf-8-sig") + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise VisionProviderError( + "vision provider returned an HTTP error", + provider_id="unknown", + status_code=exc.code, + error_type="http_error", + details={"body_preview": body[:500]}, + ) from exc + except urllib.error.URLError as exc: + raise VisionProviderError( + "vision provider network error", + provider_id="unknown", + error_type="network_error", + details={"reason": str(exc.reason)}, + ) from exc + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + raise VisionProviderError( + "vision provider returned invalid JSON", + provider_id="unknown", + error_type="invalid_response", + details={"body_preview": body[:500]}, + ) from exc + if not isinstance(parsed, dict): + raise VisionProviderError("vision provider response must be an object", provider_id="unknown") + return parsed + + +def _parse_vision_response(provider_id: str, model: str, response: JsonDict) -> VisionRecognitionResult: + content = _response_content(response) + parsed = _json_from_text(content) + prompt = _optional_text(parsed.get("prompt") or parsed.get("positive_prompt") or parsed.get("description")) + if not prompt: + raise VisionProviderError( + "vision provider response must include prompt or description", + provider_id=provider_id, + error_type="invalid_response", + details={"content_preview": content[:500]}, + ) + return VisionRecognitionResult( + provider_id=provider_id, + model=model, + prompt=prompt, + negative_prompt=_optional_text(parsed.get("negative_prompt")), + tags=_string_list(parsed.get("tags")), + confidence=_float_or_none(parsed.get("confidence")), + description=_optional_text(parsed.get("description")), + raw_metadata={"response_keys": sorted(str(key) for key in response.keys())}, + ) + + +def _response_content(response: JsonDict) -> str: + output_text = _optional_text(response.get("output_text")) + if output_text: + return output_text + choices = response.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] if isinstance(choices[0], dict) else {} + message = first.get("message") if isinstance(first.get("message"), dict) else {} + return _content_to_text(message.get("content")) + output = response.get("output") + if isinstance(output, list): + return " ".join(_content_to_text(item.get("content")) for item in output if isinstance(item, dict)).strip() + return "" + + +def _content_to_text(value: Any) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts = [] + for item in value: + if not isinstance(item, dict): + continue + text = item.get("text") or item.get("content") + if isinstance(text, str): + parts.append(text) + return " ".join(parts).strip() + return "" + + +def _json_from_text(value: str) -> JsonDict: + text = value.strip() + if not text: + raise VisionProviderError("vision provider response was empty", provider_id="unknown", error_type="invalid_response") + try: + parsed = json.loads(text) + except json.JSONDecodeError: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + raise VisionProviderError( + "vision provider response was not JSON", + provider_id="unknown", + error_type="invalid_response", + details={"content_preview": text[:500]}, + ) from None + parsed = json.loads(text[start : end + 1]) + if not isinstance(parsed, dict): + raise VisionProviderError("vision provider JSON must be an object", provider_id="unknown", error_type="invalid_response") + return parsed + + +def _string_list(value: Any) -> list[str]: + if isinstance(value, str): + values = value.split(",") + elif isinstance(value, list): + values = value + else: + values = [] + return sorted({str(item).strip() for item in values if str(item).strip()}) + + +def _float_or_none(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return min(max(number, 0.0), 1.0) + + +def _optional_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None diff --git a/tests/test_material_service.py b/tests/test_material_service.py index 828f562..1249a65 100644 --- a/tests/test_material_service.py +++ b/tests/test_material_service.py @@ -25,6 +25,7 @@ from core.config_service import ConfigService from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.material_repository import MaterialRepository +from engines.vision_provider import VisionProviderError, VisionRecognitionResult class MaterialServiceTests(unittest.TestCase): @@ -153,6 +154,8 @@ storage: self.assertEqual(recognized.recognition_status, "pending_confirm") self.assertIn("white", recognized.tags) self.assertIn("clean product reference", recognized.prompt) + self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock") + self.assertFalse(recognized.metadata["recognition"]["fallback_used"]) confirmed = self.material_service.confirm_material( uploaded.material_id, @@ -170,6 +173,89 @@ storage: self.assertEqual(re_recognized.recognition_status, "pending_confirm") self.assertIn("white", re_recognized.tags) + def test_recognize_material_uses_injected_vlm_adapter(self): + class FakeVisionProvider: + provider_id = "fake_vlm" + + def __init__(self): + self.requests = [] + + def recognize_material(self, request): + self.requests.append(request) + return VisionRecognitionResult( + provider_id=self.provider_id, + model="vision-test", + prompt="white lace blouse, product reference", + negative_prompt="body", + tags=["white", "lace", "upper"], + confidence=0.93, + description="white lace blouse", + raw_metadata={"test": True}, + ) + + provider = FakeVisionProvider() + service = MaterialService( + MaterialRepository(self.database), + self.audit_service, + vision_provider_factory=lambda route: provider, + ) + category = service.create_category({"name": "Upper"}, self.ctx) + uploaded = service.upload_material( + { + "category_id": category.category_id, + "filename": "white_blouse.png", + "image_uri": "storage://materials/white_blouse.png", + "recognition_model_route": "material_vlm", + }, + self.ctx, + ) + + recognized = service.recognize_material(uploaded.material_id, self.ctx) + + self.assertEqual(provider.requests[0].display_name, "white blouse") + self.assertEqual(recognized.prompt, "white lace blouse, product reference") + self.assertEqual(recognized.negative_prompt, "body") + self.assertIn("lace", recognized.tags) + self.assertEqual(recognized.metadata["recognition"]["provider_id"], "fake_vlm") + self.assertEqual(recognized.metadata["recognition"]["model"], "vision-test") + self.assertEqual(recognized.metadata["recognition"]["route_name"], "material_vlm") + self.assertFalse(recognized.metadata["recognition"]["fallback_used"]) + + def test_recognize_material_falls_back_when_vlm_adapter_fails(self): + class FailingVisionProvider: + provider_id = "broken_vlm" + + def recognize_material(self, request): + raise VisionProviderError( + "vision endpoint timeout", + provider_id=self.provider_id, + error_type="timeout", + details={"retryable": True}, + ) + + service = MaterialService( + MaterialRepository(self.database), + self.audit_service, + vision_provider_factory=lambda route: FailingVisionProvider(), + ) + category = service.create_category({"name": "Upper"}, self.ctx) + uploaded = service.upload_material( + { + "category_id": category.category_id, + "filename": "white_blouse.png", + "image_uri": "storage://materials/white_blouse.png", + }, + self.ctx, + ) + + recognized = service.recognize_material(uploaded.material_id, self.ctx) + + self.assertIn("white", recognized.tags) + self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock") + self.assertTrue(recognized.metadata["recognition"]["fallback_used"]) + self.assertEqual(recognized.metadata["recognition"]["provider_error"]["provider_id"], "broken_vlm") + self.assertEqual(recognized.metadata["recognition"]["provider_error"]["error_type"], "timeout") + def test_upload_rejects_unsafe_filename(self): category = self.material_service.create_category({"name": "Upper"}, self.ctx) diff --git a/tests/test_model_provider_runtime.py b/tests/test_model_provider_runtime.py index 8968a71..7671f2d 100644 --- a/tests/test_model_provider_runtime.py +++ b/tests/test_model_provider_runtime.py @@ -38,6 +38,40 @@ class ModelProviderRuntimeTests(unittest.TestCase): self.assertEqual(provider.config.image_model, "image2") self.assertEqual(provider.config.api_key, "secret-value") + def test_runtime_builds_openai_vision_provider_from_route_settings(self): + with tempfile.TemporaryDirectory() as tmp: + database = Database(DatabaseSettings(sqlite=SqliteSettings(path=str(Path(tmp) / "imagestudio.db")))) + database.initialize() + service = ModelProviderService(ModelProviderRepository(database)) + service.upsert_provider( + { + "provider_id": "site_vlm", + "base_url": "https://vision.example", + "api_key_env_var": "IMAGESTUDIO_TEST_VLM", + "timeout_seconds": 90, + } + ) + service.upsert_route( + { + "route_name": "material_vlm", + "provider_id": "site_vlm", + "model_id": "vision-model", + "modality": "vision_recognition", + "settings": {"vision_endpoint": "/v1/chat/completions"}, + } + ) + os.environ["IMAGESTUDIO_TEST_VLM"] = "vision-secret" + try: + provider = ModelProviderRuntime(service).vision_provider_for_route({"route_name": "material_vlm"}) + finally: + os.environ.pop("IMAGESTUDIO_TEST_VLM", None) + + self.assertIsNotNone(provider) + self.assertEqual(provider.config.provider_id, "site_vlm") + self.assertEqual(provider.config.vision_model, "vision-model") + self.assertEqual(provider.config.vision_endpoint, "/v1/chat/completions") + self.assertEqual(provider.config.api_key, "vision-secret") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_vision_provider.py b/tests/test_vision_provider.py new file mode 100644 index 0000000..881c6a6 --- /dev/null +++ b/tests/test_vision_provider.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import unittest + +from engines.vision_provider import ( + OpenAIVisionProvider, + VisionProviderConfig, + VisionProviderError, + VisionRecognitionRequest, +) + + +class VisionProviderTests(unittest.TestCase): + def test_openai_vision_provider_normalizes_recognition_response(self): + calls = [] + + def transport(url, payload, headers, timeout): + calls.append((url, payload, headers, timeout)) + return { + "choices": [ + { + "message": { + "content": ( + '{"prompt":"white blouse, clean product reference",' + '"negative_prompt":"body, watermark",' + '"tags":["white","blouse"],' + '"confidence":0.91,' + '"description":"white blouse"}' + ) + } + } + ] + } + + provider = OpenAIVisionProvider( + VisionProviderConfig( + provider_id="site_vlm", + base_url="https://example.test", + api_key="secret", + vision_model="vision-model", + vision_endpoint="/v1/chat/completions", + timeout_seconds=120, + ), + transport=transport, + ) + + result = provider.recognize_material( + VisionRecognitionRequest( + display_name="White blouse", + image_uri="https://example.test/white.png", + tags=("upper",), + ) + ) + + self.assertEqual(result.provider_id, "site_vlm") + self.assertEqual(result.model, "vision-model") + self.assertEqual(result.prompt, "white blouse, clean product reference") + self.assertEqual(result.negative_prompt, "body, watermark") + self.assertEqual(result.tags, ["blouse", "white"]) + self.assertEqual(result.confidence, 0.91) + self.assertEqual(calls[0][0], "https://example.test/v1/chat/completions") + self.assertEqual(calls[0][1]["model"], "vision-model") + self.assertEqual(calls[0][1]["messages"][1]["content"][1]["image_url"]["url"], "https://example.test/white.png") + self.assertEqual(calls[0][3], 120) + + def test_provider_requires_model(self): + provider = OpenAIVisionProvider(VisionProviderConfig(provider_id="missing", base_url="https://example.test")) + + with self.assertRaises(VisionProviderError) as raised: + provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png")) + + self.assertEqual(raised.exception.error_type, "configuration_error") + + def test_provider_relabels_transport_errors_with_configured_provider(self): + def transport(url, payload, headers, timeout): + raise VisionProviderError( + "remote unauthorized", + provider_id="unknown", + status_code=401, + error_type="http_error", + details={"body_preview": "invalid credentials"}, + ) + + provider = OpenAIVisionProvider( + VisionProviderConfig( + provider_id="site_vlm", + base_url="https://example.test", + api_key="secret", + vision_model="vision-model", + ), + transport=transport, + ) + + with self.assertRaises(VisionProviderError) as raised: + provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png")) + + self.assertEqual(raised.exception.provider_id, "site_vlm") + self.assertEqual(raised.exception.status_code, 401) + self.assertEqual(raised.exception.error_type, "http_error") + + +if __name__ == "__main__": + unittest.main()