diff --git a/backend/services/generation_service.py b/backend/services/generation_service.py index 3ba00f0..22ea648 100644 --- a/backend/services/generation_service.py +++ b/backend/services/generation_service.py @@ -118,13 +118,19 @@ class GenerationService: ) for index, asset in enumerate(assets) ] + provider_failure = _provider_failure_from_assets([asset.metadata for asset in assets]) + result: dict[str, Any] = {"image_ids": [image.image_id for image in images]} + update_payload: dict[str, Any] = { + "status": "succeeded", + "progress": 100, + "result": result, + } + if provider_failure: + result["provider_failure"] = provider_failure + update_payload["metadata"] = {**task.metadata, "provider_failure": provider_failure} completed_task = self.queue_service.update_task( task.task_id, - { - "status": "succeeded", - "progress": 100, - "result": {"image_ids": [image.image_id for image in images]}, - }, + update_payload, ctx, ) completed_node = self.workflow_service.update_node(node_id, {"status": "completed"}, ctx) @@ -189,3 +195,11 @@ def _optional_text(value: Any) -> str | None: return None text = str(value).strip() return text or None + + +def _provider_failure_from_assets(metadata_items: list[dict[str, Any]]) -> dict[str, Any] | None: + for metadata in metadata_items: + failure = metadata.get("provider_error") + if isinstance(failure, dict): + return failure + return None diff --git a/core/generation_errors.py b/core/generation_errors.py new file mode 100644 index 0000000..9d29844 --- /dev/null +++ b/core/generation_errors.py @@ -0,0 +1,99 @@ +"""Structured generation failure attribution helpers.""" + +from __future__ import annotations + +from typing import Any + +from engines.image_provider import ImageProviderError + + +SECRET_KEYWORDS = ("api_key", "apikey", "authorization", "bearer", "password", "secret", "token") + + +def provider_failure_attribution( + exc: ImageProviderError, + *, + route_context: dict[str, Any] | None = None, + fallback_used: bool = False, +) -> dict[str, Any]: + route = route_context or {} + status_code = exc.status_code + category = _category(exc.error_type, status_code) + attribution = { + "source": "image_provider", + "category": category, + "error_type": exc.error_type, + "message": _safe_message(str(exc)), + "provider_id": _first_text(exc.provider_id, route.get("provider_id"), "unknown"), + "status_code": status_code, + "retryable": _is_retryable(category, status_code), + "fallback_used": bool(fallback_used), + "user_action_required": category in {"configuration_error", "auth_error", "request_error"}, + } + for key in ("route_name", "route_id", "model_id", "modality"): + value = route.get(key) + if value is not None: + attribution[key] = value + details = _safe_details(exc.details) + if details: + attribution["details"] = details + return attribution + + +def _category(error_type: str, status_code: int | None) -> str: + if error_type == "configuration_error": + return "configuration_error" + if error_type == "network_error": + return "network_error" + if error_type == "invalid_response": + return "invalid_response" + if status_code in {401, 403}: + return "auth_error" + if status_code == 429: + return "rate_limited" + if status_code in {408, 504}: + return "timeout" + if status_code is not None and 500 <= status_code <= 599: + return "provider_unavailable" + if status_code is not None and 400 <= status_code <= 499: + return "request_error" + return error_type or "provider_error" + + +def _is_retryable(category: str, status_code: int | None) -> bool: + if category in {"network_error", "timeout", "rate_limited", "provider_unavailable"}: + return True + return bool(status_code is not None and 500 <= status_code <= 599) + + +def _safe_details(details: dict[str, Any]) -> dict[str, Any]: + cleaned: dict[str, Any] = {} + for key, value in details.items(): + key_text = str(key) + if _is_secret_key(key_text): + cleaned[key_text] = "[REDACTED]" + elif isinstance(value, str): + cleaned[key_text] = _safe_message(value, limit=500) + elif isinstance(value, (int, float, bool)) or value is None: + cleaned[key_text] = value + else: + cleaned[key_text] = _safe_message(str(value), limit=500) + return cleaned + + +def _safe_message(value: str, *, limit: int = 240) -> str: + text = " ".join(str(value or "").split()) + return text[:limit] if text else "image provider failed" + + +def _is_secret_key(key: str) -> bool: + lowered = key.lower() + return any(keyword in lowered for keyword in SECRET_KEYWORDS) + + +def _first_text(*values: Any) -> str: + for value in values: + text = str(value or "").strip() + if text: + return text + return "unknown" diff --git a/engines/image_provider.py b/engines/image_provider.py index e8c98ac..9d2c47d 100644 --- a/engines/image_provider.py +++ b/engines/image_provider.py @@ -121,7 +121,18 @@ class OpenAIImageProvider: "Content-Type": "application/json", **self.config.extra_headers, } - response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds) + try: + response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds) + except ImageProviderError as exc: + if exc.provider_id != "unknown": + raise + raise ImageProviderError( + str(exc), + provider_id=self.provider_id, + status_code=exc.status_code, + error_type=exc.error_type, + details=exc.details, + ) from exc return _parse_image_response(self.provider_id, model, response) diff --git a/engines/single_flow.py b/engines/single_flow.py index 5b7e82b..7239e99 100644 --- a/engines/single_flow.py +++ b/engines/single_flow.py @@ -11,6 +11,7 @@ from typing import Any, Callable from backend.dto.job_dto import GenerationTask from backend.dto.workflow_dto import WorkflowNode +from core.generation_errors import provider_failure_attribution from core.storage import VfsStorage from engines.image_provider import ImageGenerationRequest, ImageProvider, ImageProviderError @@ -51,6 +52,7 @@ class SingleFlowEngine: try: return self._generate_with_provider(provider, task, request, prompt, model_route, width, height, count) except ImageProviderError as exc: + route = request.get("resolved_model_route") if isinstance(request.get("resolved_model_route"), dict) else {} return self._generate_mock( task, request, @@ -59,11 +61,7 @@ class SingleFlowEngine: width, height, count, - provider_error={ - "provider_id": exc.provider_id, - "error_type": exc.error_type, - "status_code": exc.status_code, - }, + provider_error=provider_failure_attribution(exc, route_context=route, fallback_used=True), ) return self._generate_mock(task, request, prompt, model_route, width, height, count) @@ -133,7 +131,7 @@ class SingleFlowEngine: for index, image in enumerate(result.images): if not image.has_image_data: continue - image_bytes = _image_bytes(image.b64_json, image.url) + image_bytes = _image_bytes(image.b64_json, image.url, provider_id=result.provider_id) image_uri = f"storage://images/single/{task_stem}_{index + 1}.png" thumbnail_uri = f"storage://thumbnails/single/{task_stem}_{index + 1}.png" self._write_png(image_uri, image_bytes) @@ -207,10 +205,18 @@ def _safe_file_stem(value: str) -> str: return stem or "task" -def _image_bytes(b64_json: str | None, url: str | None) -> bytes: +def _image_bytes(b64_json: str | None, url: str | None, *, provider_id: str = "unknown") -> bytes: if b64_json: return base64.b64decode(re.sub(r"\s+", "", b64_json)) if url: - with urllib.request.urlopen(url, timeout=120) as response: - return response.read() - raise ImageProviderError("image payload is empty", provider_id="unknown", error_type="invalid_response") + try: + with urllib.request.urlopen(url, timeout=120) as response: + return response.read() + except Exception as exc: + raise ImageProviderError( + "image provider image URL could not be downloaded", + provider_id=provider_id, + error_type="network_error", + details={"reason": str(exc)}, + ) from exc + raise ImageProviderError("image payload is empty", provider_id=provider_id, error_type="invalid_response") diff --git a/tests/test_image_provider.py b/tests/test_image_provider.py index 65520fe..25617aa 100644 --- a/tests/test_image_provider.py +++ b/tests/test_image_provider.py @@ -66,6 +66,33 @@ class ImageProviderTests(unittest.TestCase): self.assertEqual(raised.exception.error_type, "configuration_error") + def test_provider_relabels_transport_errors_with_configured_provider(self): + def transport(url, payload, headers, timeout): + raise ImageProviderError( + "remote unauthorized", + provider_id="unknown", + status_code=401, + error_type="http_error", + details={"body_preview": "invalid credentials"}, + ) + + provider = OpenAIImageProvider( + ImageProviderConfig( + provider_id="site_b", + base_url="https://botcf.com", + api_key="secret", + image_model="image2", + ), + transport=transport, + ) + + with self.assertRaises(ImageProviderError) as raised: + provider.generate_image(ImageGenerationRequest(prompt="red dot")) + + self.assertEqual(raised.exception.provider_id, "site_b") + self.assertEqual(raised.exception.status_code, 401) + self.assertEqual(raised.exception.error_type, "http_error") + def test_mock_provider_returns_image_payload(self): result = MockImageProvider().generate_image(ImageGenerationRequest(prompt="red dot", model="mock")) diff --git a/tests/test_single_flow_generation.py b/tests/test_single_flow_generation.py index b145c7f..616b4cd 100644 --- a/tests/test_single_flow_generation.py +++ b/tests/test_single_flow_generation.py @@ -26,7 +26,7 @@ from database.repositories.model_provider_repository import ModelProviderReposit from database.repositories.profile_repository import ProfileRepository from database.repositories.workflow_repository import WorkflowRepository from engines.single_flow import SingleFlowEngine -from engines.image_provider import GeneratedImagePayload, ImageGenerationResult +from engines.image_provider import GeneratedImagePayload, ImageGenerationResult, ImageProviderError class SingleFlowGenerationTests(unittest.TestCase): @@ -325,6 +325,73 @@ storage: self.assertEqual(assets[0].metadata["usage"]["total_tokens"], 12) self.assertTrue(VfsStorage(self.config_service).ensure_file(assets[0].image_uri).exists()) + def test_generate_node_records_provider_failure_attribution_on_fallback(self): + class FailingProvider: + provider_id = "site_b" + + def generate_image(self, request): + raise ImageProviderError( + "remote unauthorized", + provider_id=self.provider_id, + status_code=401, + error_type="http_error", + details={"body_preview": "invalid credentials", "api_key": "secret-value"}, + ) + + self.model_provider_service.upsert_provider( + { + "provider_id": "site_b", + "base_url": "https://botcf.com", + "image_endpoint": "/v1/images/generations", + } + ) + self.model_provider_service.upsert_route( + { + "route_name": "image_high", + "provider_id": "site_b", + "model_id": "image2", + "priority": 5, + } + ) + generation_service = GenerationService( + workflow_service=self.workflow_service, + queue_service=self.queue_service, + image_service=self.image_service, + single_flow_engine=SingleFlowEngine( + VfsStorage(self.config_service), + image_provider_factory=lambda route: FailingProvider(), + ), + model_provider_service=self.model_provider_service, + event_bus=self.event_bus, + ) + profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx) + workflow = self.workflow_service.create_workflow( + profile.profile_id, + {"workflow_type": "single_image", "display_name": "Single"}, + self.ctx, + ) + + response = generate_node( + self.ctx, + workflow.nodes[0].node_id, + {"prompt_patch": "white shirt", "model_route": "image_high"}, + generation_service=generation_service, + ) + + self.assertTrue(response["ok"]) + task = self.queue_service.get_job(response["data"]["task_id"]) + failure = task.result["provider_failure"] + self.assertEqual(failure["category"], "auth_error") + self.assertEqual(failure["provider_id"], "site_b") + self.assertEqual(failure["route_name"], "image_high") + self.assertFalse(failure["retryable"]) + self.assertTrue(failure["fallback_used"]) + self.assertEqual(failure["details"]["api_key"], "[REDACTED]") + self.assertNotIn("secret-value", str(task.to_dict())) + image = self.image_service.get_image(response["data"]["images"][0]["image_id"]) + self.assertTrue(image.metadata["mock"]) + self.assertEqual(image.metadata["provider_error"]["category"], "auth_error") + def test_queue_service_rejects_invalid_task_transition(self): task = self.queue_service.create_task({"status": "succeeded"}, self.ctx)