S100 VLM适配器

这个提交存在于:
YokiiroBW 2026-07-06 03:37:17 +08:00
父节点 36a3c1708c
当前提交 0df3546c6d
共有 8 个文件被更改,包括 711 次插入 和 6 次删除

查看文件

@ -9,11 +9,14 @@ from backend.dto.material_dto import Material, MaterialListResult
from backend.routes.common import handle_request from backend.routes.common import handle_request
from backend.services.material_service import MaterialService from backend.services.material_service import MaterialService
from backend.services.media_service import MediaService from backend.services.media_service import MediaService
from backend.services.model_provider_runtime import ModelProviderRuntime
from backend.services.model_provider_service import ModelProviderService
from core.audit_service import AuditService from core.audit_service import AuditService
from core.identity import RequestContext from core.identity import RequestContext
from core.security import require_authenticated, require_permission from core.security import require_authenticated, require_permission
from database.db_helper import Database, load_database_settings from database.db_helper import Database, load_database_settings
from database.repositories.material_repository import MaterialRepository from database.repositories.material_repository import MaterialRepository
from database.repositories.model_provider_repository import ModelProviderRepository
def list_material_categories( def list_material_categories(
@ -227,9 +230,12 @@ def confirm_material(
def _create_material_service() -> MaterialService: def _create_material_service() -> MaterialService:
database = Database(load_database_settings()) database = Database(load_database_settings())
model_provider_service = ModelProviderService(ModelProviderRepository(database))
model_provider_runtime = ModelProviderRuntime(model_provider_service)
return MaterialService( return MaterialService(
repository=MaterialRepository(database), repository=MaterialRepository(database),
audit_service=AuditService(database), audit_service=AuditService(database),
vision_provider_factory=model_provider_runtime.vision_provider_for_route,
) )

查看文件

@ -3,7 +3,7 @@
from __future__ import annotations from __future__ import annotations
from pathlib import PurePosixPath from pathlib import PurePosixPath
from typing import Any from typing import Any, Callable
from uuid import uuid4 from uuid import uuid4
from backend.dto.common import ApiError, ErrorCode from backend.dto.common import ApiError, ErrorCode
@ -11,6 +11,13 @@ from backend.dto.material_dto import Material, MaterialCategory, MaterialListRes
from core.audit_service import AuditService from core.audit_service import AuditService
from core.identity import RequestContext from core.identity import RequestContext
from database.repositories.material_repository import MaterialRepository from database.repositories.material_repository import MaterialRepository
from engines.vision_provider import (
MockVisionProvider,
VisionProvider,
VisionProviderError,
VisionRecognitionRequest,
VisionRecognitionResult,
)
MATERIAL_STATUSES = {"pending_upload", "pending_recognition", "pending_confirm", "confirmed", "archived"} MATERIAL_STATUSES = {"pending_upload", "pending_recognition", "pending_confirm", "confirmed", "archived"}
@ -25,9 +32,18 @@ UPLOAD_ALLOWED_CONTENT_TYPES = {
class MaterialService: class MaterialService:
def __init__(self, repository: MaterialRepository, audit_service: AuditService): def __init__(
self,
repository: MaterialRepository,
audit_service: AuditService,
*,
vision_provider_factory: Callable[[dict[str, Any]], VisionProvider | None] | None = None,
default_recognition_route: str = "material_vlm",
):
self.repository = repository self.repository = repository
self.audit_service = audit_service self.audit_service = audit_service
self.vision_provider_factory = vision_provider_factory
self.default_recognition_route = default_recognition_route
def create_category(self, payload: dict[str, Any], ctx: RequestContext) -> MaterialCategory: def create_category(self, payload: dict[str, Any], ctx: RequestContext) -> MaterialCategory:
name = str(payload.get("name") or "").strip() name = str(payload.get("name") or "").strip()
@ -189,6 +205,9 @@ class MaterialService:
content_type = _normalize_upload_content_type(payload.get("content_type")) content_type = _normalize_upload_content_type(payload.get("content_type"))
declared_size = _normalize_upload_size(payload) declared_size = _normalize_upload_size(payload)
upload_metadata = _normalize_upload_metadata(payload.get("metadata")) upload_metadata = _normalize_upload_metadata(payload.get("metadata"))
recognition_route = _optional_text(payload.get("recognition_model_route"))
if recognition_route:
upload_metadata["recognition_model_route"] = recognition_route
extension = _validate_upload_file_type(filename=filename, image_uri=image_uri, content_type=content_type) extension = _validate_upload_file_type(filename=filename, image_uri=image_uri, content_type=content_type)
display_name = str(payload.get("display_name") or _display_name_from_filename(filename) or "Uploaded material").strip() display_name = str(payload.get("display_name") or _display_name_from_filename(filename) or "Uploaded material").strip()
thumbnail_uri = str(payload.get("thumbnail_uri") or image_uri).strip() thumbnail_uri = str(payload.get("thumbnail_uri") or image_uri).strip()
@ -225,16 +244,25 @@ class MaterialService:
return self._recognize_material(material, ctx) return self._recognize_material(material, ctx)
def _recognize_material(self, material: Material, ctx: RequestContext) -> Material: def _recognize_material(self, material: Material, ctx: RequestContext) -> Material:
generated_tags = _mock_tags(material) recognition, provider_error, fallback_used, route_name = self._recognize_with_adapter(material)
prompt = material.prompt or _mock_prompt(material) generated_tags = recognition.tags
prompt = material.prompt or recognition.prompt or _mock_prompt(material)
after = self.update_material( after = self.update_material(
material.material_id, material.material_id,
{ {
"prompt": prompt, "prompt": prompt,
"negative_prompt": material.negative_prompt or "body, face, background, watermark", "negative_prompt": material.negative_prompt or recognition.negative_prompt or "body, face, background, watermark",
"tags": sorted(set(material.tags + generated_tags)), "tags": sorted(set(material.tags + generated_tags)),
"recognition_status": "pending_confirm", "recognition_status": "pending_confirm",
"metadata": {**material.metadata, "recognition": {"engine": "mock_vlm", "confidence": 0.72}}, "metadata": {
**material.metadata,
"recognition": _recognition_metadata(
recognition,
route_name=route_name,
fallback_used=fallback_used,
provider_error=provider_error,
),
},
}, },
ctx, ctx,
) )
@ -248,6 +276,30 @@ class MaterialService:
) )
return after return after
def _recognize_with_adapter(
self,
material: Material,
) -> tuple[VisionRecognitionResult, dict[str, Any] | None, bool, str]:
route_name = _recognition_route(material.metadata, self.default_recognition_route)
request = _vision_request(material)
provider = None
if self.vision_provider_factory:
provider = self.vision_provider_factory(
{
"route_name": route_name,
"modality": "vision_recognition",
"material_id": material.material_id,
"image_uri": material.image_uri,
"thumbnail_uri": material.thumbnail_uri,
}
)
if provider is None:
return _mock_vision_recognition(material), None, False, route_name
try:
return provider.recognize_material(request), None, False, route_name
except VisionProviderError as exc:
return _mock_vision_recognition(material), _vision_provider_error(exc), True, route_name
def confirm_material(self, material_id: str, payload: dict[str, Any], ctx: RequestContext) -> Material: def confirm_material(self, material_id: str, payload: dict[str, Any], ctx: RequestContext) -> Material:
material = self.get_material(material_id) material = self.get_material(material_id)
update_payload = { update_payload = {
@ -310,6 +362,65 @@ def _mock_tags(material: Material) -> list[str]:
return tags return tags
def _vision_request(material: Material) -> VisionRecognitionRequest:
return VisionRecognitionRequest(
display_name=material.display_name,
image_uri=str(material.image_uri or ""),
thumbnail_uri=material.thumbnail_uri,
prompt=material.prompt,
negative_prompt=material.negative_prompt,
tags=tuple(material.tags),
metadata=material.metadata,
)
def _mock_vision_recognition(material: Material) -> VisionRecognitionResult:
return MockVisionProvider().recognize_material(_vision_request(material))
def _recognition_route(metadata: dict[str, Any], default_route: str) -> str:
upload = metadata.get("upload") if isinstance(metadata.get("upload"), dict) else {}
return (
_optional_text(metadata.get("recognition_model_route"))
or _optional_text(metadata.get("recognition_route"))
or _optional_text(upload.get("recognition_model_route"))
or default_route
)
def _recognition_metadata(
recognition: VisionRecognitionResult,
*,
route_name: str,
fallback_used: bool,
provider_error: dict[str, Any] | None,
) -> dict[str, Any]:
metadata = {
"engine": "mock_vlm" if recognition.provider_id == "mock" else "vlm_adapter",
"provider_id": recognition.provider_id,
"model": recognition.model,
"confidence": recognition.confidence,
"route_name": route_name,
"fallback_used": fallback_used,
"description": recognition.description,
"raw_metadata": recognition.raw_metadata,
}
if provider_error:
metadata["provider_error"] = provider_error
return metadata
def _vision_provider_error(exc: VisionProviderError) -> dict[str, Any]:
return {
"source": "vision_provider",
"provider_id": exc.provider_id,
"status_code": exc.status_code,
"error_type": exc.error_type,
"message": str(exc),
"details": exc.details,
}
def _normalize_upload_filename(value: Any) -> str | None: def _normalize_upload_filename(value: Any) -> str | None:
if value is None: if value is None:
return None return None
@ -406,3 +517,8 @@ def _display_name_from_filename(filename: str | None) -> str | None:
if not filename: if not filename:
return None return None
return PurePosixPath(filename).stem.replace("_", " ").strip() or None return PurePosixPath(filename).stem.replace("_", " ").strip() or None
def _optional_text(value: Any) -> str | None:
text = str(value or "").strip()
return text or None

查看文件

@ -7,6 +7,7 @@ from typing import Any
from backend.services.model_provider_service import ModelProviderService from backend.services.model_provider_service import ModelProviderService
from core.secret_service import SecretStore, resolve_secret_value from core.secret_service import SecretStore, resolve_secret_value
from engines.image_provider import ImageProvider, ImageProviderConfig, OpenAIImageProvider from engines.image_provider import ImageProvider, ImageProviderConfig, OpenAIImageProvider
from engines.vision_provider import OpenAIVisionProvider, VisionProvider, VisionProviderConfig
class ModelProviderRuntime: class ModelProviderRuntime:
@ -48,3 +49,39 @@ class ModelProviderRuntime:
extra_headers=dict(provider.get("extra_headers") or {}), extra_headers=dict(provider.get("extra_headers") or {}),
) )
) )
def vision_provider_for_route(self, route_context: dict[str, Any]) -> VisionProvider | None:
route_name = str(route_context.get("route_name") or "").strip()
if not route_name:
return None
resolved = self.model_provider_service.resolve_route(route_name, modality="vision_recognition")
if not resolved:
return None
provider = resolved.get("provider") or {}
if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible":
return None
secret = resolve_secret_value(
env_var=provider.get("api_key_env_var"),
secret_ref=provider.get("api_key_secret_ref"),
secret_store=self.secret_store,
)
if not secret.value:
return None
settings = dict(resolved.get("settings") or {})
capabilities = dict(provider.get("capabilities") or {})
return OpenAIVisionProvider(
VisionProviderConfig(
provider_id=str(resolved["provider_id"]),
base_url=str(provider.get("base_url") or ""),
api_key=secret.value,
vision_model=str(resolved["model_id"]),
vision_endpoint=str(
settings.get("vision_endpoint")
or capabilities.get("vision_endpoint")
or "/v1/chat/completions"
),
provider_type=str(provider.get("provider_type") or "openai_compatible"),
timeout_seconds=int(provider.get("timeout_seconds") or 180),
extra_headers=dict(provider.get("extra_headers") or {}),
)
)

查看文件

@ -9,6 +9,14 @@ from engines.image_provider import (
MockImageProvider, MockImageProvider,
OpenAIImageProvider, OpenAIImageProvider,
) )
from engines.vision_provider import (
MockVisionProvider,
OpenAIVisionProvider,
VisionProviderConfig,
VisionProviderError,
VisionRecognitionRequest,
VisionRecognitionResult,
)
__all__ = [ __all__ = [
"GeneratedImagePayload", "GeneratedImagePayload",
@ -17,5 +25,11 @@ __all__ = [
"ImageProviderConfig", "ImageProviderConfig",
"ImageProviderError", "ImageProviderError",
"MockImageProvider", "MockImageProvider",
"MockVisionProvider",
"OpenAIImageProvider", "OpenAIImageProvider",
"OpenAIVisionProvider",
"VisionProviderConfig",
"VisionProviderError",
"VisionRecognitionRequest",
"VisionRecognitionResult",
] ]

309
engines/vision_provider.py 普通文件
查看文件

@ -0,0 +1,309 @@
"""Vision provider adapter interfaces for material recognition."""
from __future__ import annotations
import json
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any, Callable, Protocol
JsonDict = dict[str, Any]
HttpTransport = Callable[[str, JsonDict, dict[str, str], int], JsonDict]
class VisionProvider(Protocol):
provider_id: str
def recognize_material(self, request: "VisionRecognitionRequest") -> "VisionRecognitionResult":
...
@dataclass(frozen=True)
class VisionProviderConfig:
provider_id: str
base_url: str
api_key: str = ""
vision_model: str = ""
vision_endpoint: str = "/v1/chat/completions"
provider_type: str = "openai_compatible"
timeout_seconds: int = 180
extra_headers: dict[str, str] = field(default_factory=dict)
def endpoint_url(self) -> str:
base = self.base_url.rstrip("/")
endpoint = self.vision_endpoint if self.vision_endpoint.startswith("/") else f"/{self.vision_endpoint}"
return f"{base}{endpoint}"
@dataclass(frozen=True)
class VisionRecognitionRequest:
display_name: str
image_uri: str
thumbnail_uri: str | None = None
prompt: str | None = None
negative_prompt: str | None = None
tags: tuple[str, ...] = ()
model: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass(frozen=True)
class VisionRecognitionResult:
provider_id: str
model: str
prompt: str
negative_prompt: str | None = None
tags: list[str] = field(default_factory=list)
confidence: float | None = None
description: str | None = None
raw_metadata: dict[str, Any] = field(default_factory=dict)
class VisionProviderError(RuntimeError):
def __init__(
self,
message: str,
*,
provider_id: str,
status_code: int | None = None,
error_type: str = "provider_error",
details: dict[str, Any] | None = None,
):
super().__init__(message)
self.provider_id = provider_id
self.status_code = status_code
self.error_type = error_type
self.details = details or {}
class OpenAIVisionProvider:
def __init__(self, config: VisionProviderConfig, transport: HttpTransport | None = None):
self.config = config
self.provider_id = config.provider_id
self._transport = transport or _post_json
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
model = request.model or self.config.vision_model
if not model:
raise VisionProviderError(
"vision model is required",
provider_id=self.provider_id,
error_type="configuration_error",
)
payload = {
"model": model,
"messages": [
{
"role": "system",
"content": (
"Return compact JSON for a reusable outfit material with keys: "
"prompt, negative_prompt, tags, confidence, description."
),
},
{"role": "user", "content": _user_content(request)},
],
"temperature": 0.1,
"response_format": {"type": "json_object"},
}
headers = {
"Authorization": f"Bearer {self.config.api_key}",
"Content-Type": "application/json",
**self.config.extra_headers,
}
try:
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
except VisionProviderError as exc:
if exc.provider_id != "unknown":
raise
raise VisionProviderError(
str(exc),
provider_id=self.provider_id,
status_code=exc.status_code,
error_type=exc.error_type,
details=exc.details,
) from exc
return _parse_vision_response(self.provider_id, model, response)
class MockVisionProvider:
provider_id = "mock"
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
text = f"{request.display_name} {request.prompt or ''}".lower()
tags = ["material"]
for keyword in ("white", "blue", "black", "red", "ribbon", "shirt", "blouse", "dress", "skirt", "shoe"):
if keyword in text:
tags.append(keyword)
base = request.display_name.strip().lower().replace("_", " ") or "uploaded material"
return VisionRecognitionResult(
provider_id=self.provider_id,
model=request.model or "mock-vlm",
prompt=f"{base}, clean product reference, transparent background",
negative_prompt="body, face, background, watermark",
tags=tags,
confidence=0.72,
raw_metadata={"mock": True},
)
def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]:
content: list[dict[str, Any]] = [
{
"type": "text",
"text": (
f"Material name: {request.display_name}\n"
f"Known prompt: {request.prompt or ''}\n"
f"Known negative prompt: {request.negative_prompt or ''}\n"
f"Known tags: {', '.join(request.tags)}"
),
}
]
image_url = _remote_image_url(request.thumbnail_uri) or _remote_image_url(request.image_uri)
if image_url:
content.append({"type": "image_url", "image_url": {"url": image_url}})
else:
content[0]["text"] += f"\nStorage image URI: {request.image_uri}"
return content
def _remote_image_url(value: str | None) -> str | None:
text = str(value or "").strip()
if text.startswith(("http://", "https://", "data:image/")):
return text
return None
def _post_json(url: str, payload: JsonDict, headers: dict[str, str], timeout_seconds: int) -> JsonDict:
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(url, data=data, headers=headers, method="POST")
try:
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
body = response.read().decode("utf-8-sig")
except urllib.error.HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace")
raise VisionProviderError(
"vision provider returned an HTTP error",
provider_id="unknown",
status_code=exc.code,
error_type="http_error",
details={"body_preview": body[:500]},
) from exc
except urllib.error.URLError as exc:
raise VisionProviderError(
"vision provider network error",
provider_id="unknown",
error_type="network_error",
details={"reason": str(exc.reason)},
) from exc
try:
parsed = json.loads(body)
except json.JSONDecodeError as exc:
raise VisionProviderError(
"vision provider returned invalid JSON",
provider_id="unknown",
error_type="invalid_response",
details={"body_preview": body[:500]},
) from exc
if not isinstance(parsed, dict):
raise VisionProviderError("vision provider response must be an object", provider_id="unknown")
return parsed
def _parse_vision_response(provider_id: str, model: str, response: JsonDict) -> VisionRecognitionResult:
content = _response_content(response)
parsed = _json_from_text(content)
prompt = _optional_text(parsed.get("prompt") or parsed.get("positive_prompt") or parsed.get("description"))
if not prompt:
raise VisionProviderError(
"vision provider response must include prompt or description",
provider_id=provider_id,
error_type="invalid_response",
details={"content_preview": content[:500]},
)
return VisionRecognitionResult(
provider_id=provider_id,
model=model,
prompt=prompt,
negative_prompt=_optional_text(parsed.get("negative_prompt")),
tags=_string_list(parsed.get("tags")),
confidence=_float_or_none(parsed.get("confidence")),
description=_optional_text(parsed.get("description")),
raw_metadata={"response_keys": sorted(str(key) for key in response.keys())},
)
def _response_content(response: JsonDict) -> str:
output_text = _optional_text(response.get("output_text"))
if output_text:
return output_text
choices = response.get("choices")
if isinstance(choices, list) and choices:
first = choices[0] if isinstance(choices[0], dict) else {}
message = first.get("message") if isinstance(first.get("message"), dict) else {}
return _content_to_text(message.get("content"))
output = response.get("output")
if isinstance(output, list):
return " ".join(_content_to_text(item.get("content")) for item in output if isinstance(item, dict)).strip()
return ""
def _content_to_text(value: Any) -> str:
if isinstance(value, str):
return value.strip()
if isinstance(value, list):
parts = []
for item in value:
if not isinstance(item, dict):
continue
text = item.get("text") or item.get("content")
if isinstance(text, str):
parts.append(text)
return " ".join(parts).strip()
return ""
def _json_from_text(value: str) -> JsonDict:
text = value.strip()
if not text:
raise VisionProviderError("vision provider response was empty", provider_id="unknown", error_type="invalid_response")
try:
parsed = json.loads(text)
except json.JSONDecodeError:
start = text.find("{")
end = text.rfind("}")
if start < 0 or end <= start:
raise VisionProviderError(
"vision provider response was not JSON",
provider_id="unknown",
error_type="invalid_response",
details={"content_preview": text[:500]},
) from None
parsed = json.loads(text[start : end + 1])
if not isinstance(parsed, dict):
raise VisionProviderError("vision provider JSON must be an object", provider_id="unknown", error_type="invalid_response")
return parsed
def _string_list(value: Any) -> list[str]:
if isinstance(value, str):
values = value.split(",")
elif isinstance(value, list):
values = value
else:
values = []
return sorted({str(item).strip() for item in values if str(item).strip()})
def _float_or_none(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return min(max(number, 0.0), 1.0)
def _optional_text(value: Any) -> str | None:
text = str(value or "").strip()
return text or None

查看文件

@ -25,6 +25,7 @@ from core.config_service import ConfigService
from core.identity import RequestContext from core.identity import RequestContext
from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.material_repository import MaterialRepository from database.repositories.material_repository import MaterialRepository
from engines.vision_provider import VisionProviderError, VisionRecognitionResult
class MaterialServiceTests(unittest.TestCase): class MaterialServiceTests(unittest.TestCase):
@ -153,6 +154,8 @@ storage:
self.assertEqual(recognized.recognition_status, "pending_confirm") self.assertEqual(recognized.recognition_status, "pending_confirm")
self.assertIn("white", recognized.tags) self.assertIn("white", recognized.tags)
self.assertIn("clean product reference", recognized.prompt) self.assertIn("clean product reference", recognized.prompt)
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock")
self.assertFalse(recognized.metadata["recognition"]["fallback_used"])
confirmed = self.material_service.confirm_material( confirmed = self.material_service.confirm_material(
uploaded.material_id, uploaded.material_id,
@ -170,6 +173,89 @@ storage:
self.assertEqual(re_recognized.recognition_status, "pending_confirm") self.assertEqual(re_recognized.recognition_status, "pending_confirm")
self.assertIn("white", re_recognized.tags) self.assertIn("white", re_recognized.tags)
def test_recognize_material_uses_injected_vlm_adapter(self):
class FakeVisionProvider:
provider_id = "fake_vlm"
def __init__(self):
self.requests = []
def recognize_material(self, request):
self.requests.append(request)
return VisionRecognitionResult(
provider_id=self.provider_id,
model="vision-test",
prompt="white lace blouse, product reference",
negative_prompt="body",
tags=["white", "lace", "upper"],
confidence=0.93,
description="white lace blouse",
raw_metadata={"test": True},
)
provider = FakeVisionProvider()
service = MaterialService(
MaterialRepository(self.database),
self.audit_service,
vision_provider_factory=lambda route: provider,
)
category = service.create_category({"name": "Upper"}, self.ctx)
uploaded = service.upload_material(
{
"category_id": category.category_id,
"filename": "white_blouse.png",
"image_uri": "storage://materials/white_blouse.png",
"recognition_model_route": "material_vlm",
},
self.ctx,
)
recognized = service.recognize_material(uploaded.material_id, self.ctx)
self.assertEqual(provider.requests[0].display_name, "white blouse")
self.assertEqual(recognized.prompt, "white lace blouse, product reference")
self.assertEqual(recognized.negative_prompt, "body")
self.assertIn("lace", recognized.tags)
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "fake_vlm")
self.assertEqual(recognized.metadata["recognition"]["model"], "vision-test")
self.assertEqual(recognized.metadata["recognition"]["route_name"], "material_vlm")
self.assertFalse(recognized.metadata["recognition"]["fallback_used"])
def test_recognize_material_falls_back_when_vlm_adapter_fails(self):
class FailingVisionProvider:
provider_id = "broken_vlm"
def recognize_material(self, request):
raise VisionProviderError(
"vision endpoint timeout",
provider_id=self.provider_id,
error_type="timeout",
details={"retryable": True},
)
service = MaterialService(
MaterialRepository(self.database),
self.audit_service,
vision_provider_factory=lambda route: FailingVisionProvider(),
)
category = service.create_category({"name": "Upper"}, self.ctx)
uploaded = service.upload_material(
{
"category_id": category.category_id,
"filename": "white_blouse.png",
"image_uri": "storage://materials/white_blouse.png",
},
self.ctx,
)
recognized = service.recognize_material(uploaded.material_id, self.ctx)
self.assertIn("white", recognized.tags)
self.assertEqual(recognized.metadata["recognition"]["provider_id"], "mock")
self.assertTrue(recognized.metadata["recognition"]["fallback_used"])
self.assertEqual(recognized.metadata["recognition"]["provider_error"]["provider_id"], "broken_vlm")
self.assertEqual(recognized.metadata["recognition"]["provider_error"]["error_type"], "timeout")
def test_upload_rejects_unsafe_filename(self): def test_upload_rejects_unsafe_filename(self):
category = self.material_service.create_category({"name": "Upper"}, self.ctx) category = self.material_service.create_category({"name": "Upper"}, self.ctx)

查看文件

@ -38,6 +38,40 @@ class ModelProviderRuntimeTests(unittest.TestCase):
self.assertEqual(provider.config.image_model, "image2") self.assertEqual(provider.config.image_model, "image2")
self.assertEqual(provider.config.api_key, "secret-value") self.assertEqual(provider.config.api_key, "secret-value")
def test_runtime_builds_openai_vision_provider_from_route_settings(self):
with tempfile.TemporaryDirectory() as tmp:
database = Database(DatabaseSettings(sqlite=SqliteSettings(path=str(Path(tmp) / "imagestudio.db"))))
database.initialize()
service = ModelProviderService(ModelProviderRepository(database))
service.upsert_provider(
{
"provider_id": "site_vlm",
"base_url": "https://vision.example",
"api_key_env_var": "IMAGESTUDIO_TEST_VLM",
"timeout_seconds": 90,
}
)
service.upsert_route(
{
"route_name": "material_vlm",
"provider_id": "site_vlm",
"model_id": "vision-model",
"modality": "vision_recognition",
"settings": {"vision_endpoint": "/v1/chat/completions"},
}
)
os.environ["IMAGESTUDIO_TEST_VLM"] = "vision-secret"
try:
provider = ModelProviderRuntime(service).vision_provider_for_route({"route_name": "material_vlm"})
finally:
os.environ.pop("IMAGESTUDIO_TEST_VLM", None)
self.assertIsNotNone(provider)
self.assertEqual(provider.config.provider_id, "site_vlm")
self.assertEqual(provider.config.vision_model, "vision-model")
self.assertEqual(provider.config.vision_endpoint, "/v1/chat/completions")
self.assertEqual(provider.config.api_key, "vision-secret")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

查看文件

@ -0,0 +1,103 @@
from __future__ import annotations
import unittest
from engines.vision_provider import (
OpenAIVisionProvider,
VisionProviderConfig,
VisionProviderError,
VisionRecognitionRequest,
)
class VisionProviderTests(unittest.TestCase):
def test_openai_vision_provider_normalizes_recognition_response(self):
calls = []
def transport(url, payload, headers, timeout):
calls.append((url, payload, headers, timeout))
return {
"choices": [
{
"message": {
"content": (
'{"prompt":"white blouse, clean product reference",'
'"negative_prompt":"body, watermark",'
'"tags":["white","blouse"],'
'"confidence":0.91,'
'"description":"white blouse"}'
)
}
}
]
}
provider = OpenAIVisionProvider(
VisionProviderConfig(
provider_id="site_vlm",
base_url="https://example.test",
api_key="secret",
vision_model="vision-model",
vision_endpoint="/v1/chat/completions",
timeout_seconds=120,
),
transport=transport,
)
result = provider.recognize_material(
VisionRecognitionRequest(
display_name="White blouse",
image_uri="https://example.test/white.png",
tags=("upper",),
)
)
self.assertEqual(result.provider_id, "site_vlm")
self.assertEqual(result.model, "vision-model")
self.assertEqual(result.prompt, "white blouse, clean product reference")
self.assertEqual(result.negative_prompt, "body, watermark")
self.assertEqual(result.tags, ["blouse", "white"])
self.assertEqual(result.confidence, 0.91)
self.assertEqual(calls[0][0], "https://example.test/v1/chat/completions")
self.assertEqual(calls[0][1]["model"], "vision-model")
self.assertEqual(calls[0][1]["messages"][1]["content"][1]["image_url"]["url"], "https://example.test/white.png")
self.assertEqual(calls[0][3], 120)
def test_provider_requires_model(self):
provider = OpenAIVisionProvider(VisionProviderConfig(provider_id="missing", base_url="https://example.test"))
with self.assertRaises(VisionProviderError) as raised:
provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png"))
self.assertEqual(raised.exception.error_type, "configuration_error")
def test_provider_relabels_transport_errors_with_configured_provider(self):
def transport(url, payload, headers, timeout):
raise VisionProviderError(
"remote unauthorized",
provider_id="unknown",
status_code=401,
error_type="http_error",
details={"body_preview": "invalid credentials"},
)
provider = OpenAIVisionProvider(
VisionProviderConfig(
provider_id="site_vlm",
base_url="https://example.test",
api_key="secret",
vision_model="vision-model",
),
transport=transport,
)
with self.assertRaises(VisionProviderError) as raised:
provider.recognize_material(VisionRecognitionRequest(display_name="Item", image_uri="storage://x.png"))
self.assertEqual(raised.exception.provider_id, "site_vlm")
self.assertEqual(raised.exception.status_code, 401)
self.assertEqual(raised.exception.error_type, "http_error")
if __name__ == "__main__":
unittest.main()