S82 生成失败错误归因
这个提交存在于:
父节点
25a061c3a5
当前提交
1aeb290737
共有 6 个文件被更改,包括 241 次插入 和 17 次删除
|
|
@ -118,13 +118,19 @@ class GenerationService:
|
|||
)
|
||||
for index, asset in enumerate(assets)
|
||||
]
|
||||
provider_failure = _provider_failure_from_assets([asset.metadata for asset in assets])
|
||||
result: dict[str, Any] = {"image_ids": [image.image_id for image in images]}
|
||||
update_payload: dict[str, Any] = {
|
||||
"status": "succeeded",
|
||||
"progress": 100,
|
||||
"result": result,
|
||||
}
|
||||
if provider_failure:
|
||||
result["provider_failure"] = provider_failure
|
||||
update_payload["metadata"] = {**task.metadata, "provider_failure": provider_failure}
|
||||
completed_task = self.queue_service.update_task(
|
||||
task.task_id,
|
||||
{
|
||||
"status": "succeeded",
|
||||
"progress": 100,
|
||||
"result": {"image_ids": [image.image_id for image in images]},
|
||||
},
|
||||
update_payload,
|
||||
ctx,
|
||||
)
|
||||
completed_node = self.workflow_service.update_node(node_id, {"status": "completed"}, ctx)
|
||||
|
|
@ -189,3 +195,11 @@ def _optional_text(value: Any) -> str | None:
|
|||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _provider_failure_from_assets(metadata_items: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
for metadata in metadata_items:
|
||||
failure = metadata.get("provider_error")
|
||||
if isinstance(failure, dict):
|
||||
return failure
|
||||
return None
|
||||
|
|
|
|||
99
core/generation_errors.py
普通文件
99
core/generation_errors.py
普通文件
|
|
@ -0,0 +1,99 @@
|
|||
"""Structured generation failure attribution helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from engines.image_provider import ImageProviderError
|
||||
|
||||
|
||||
SECRET_KEYWORDS = ("api_key", "apikey", "authorization", "bearer", "password", "secret", "token")
|
||||
|
||||
|
||||
def provider_failure_attribution(
|
||||
exc: ImageProviderError,
|
||||
*,
|
||||
route_context: dict[str, Any] | None = None,
|
||||
fallback_used: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
route = route_context or {}
|
||||
status_code = exc.status_code
|
||||
category = _category(exc.error_type, status_code)
|
||||
attribution = {
|
||||
"source": "image_provider",
|
||||
"category": category,
|
||||
"error_type": exc.error_type,
|
||||
"message": _safe_message(str(exc)),
|
||||
"provider_id": _first_text(exc.provider_id, route.get("provider_id"), "unknown"),
|
||||
"status_code": status_code,
|
||||
"retryable": _is_retryable(category, status_code),
|
||||
"fallback_used": bool(fallback_used),
|
||||
"user_action_required": category in {"configuration_error", "auth_error", "request_error"},
|
||||
}
|
||||
for key in ("route_name", "route_id", "model_id", "modality"):
|
||||
value = route.get(key)
|
||||
if value is not None:
|
||||
attribution[key] = value
|
||||
details = _safe_details(exc.details)
|
||||
if details:
|
||||
attribution["details"] = details
|
||||
return attribution
|
||||
|
||||
|
||||
def _category(error_type: str, status_code: int | None) -> str:
|
||||
if error_type == "configuration_error":
|
||||
return "configuration_error"
|
||||
if error_type == "network_error":
|
||||
return "network_error"
|
||||
if error_type == "invalid_response":
|
||||
return "invalid_response"
|
||||
if status_code in {401, 403}:
|
||||
return "auth_error"
|
||||
if status_code == 429:
|
||||
return "rate_limited"
|
||||
if status_code in {408, 504}:
|
||||
return "timeout"
|
||||
if status_code is not None and 500 <= status_code <= 599:
|
||||
return "provider_unavailable"
|
||||
if status_code is not None and 400 <= status_code <= 499:
|
||||
return "request_error"
|
||||
return error_type or "provider_error"
|
||||
|
||||
|
||||
def _is_retryable(category: str, status_code: int | None) -> bool:
|
||||
if category in {"network_error", "timeout", "rate_limited", "provider_unavailable"}:
|
||||
return True
|
||||
return bool(status_code is not None and 500 <= status_code <= 599)
|
||||
|
||||
|
||||
def _safe_details(details: dict[str, Any]) -> dict[str, Any]:
|
||||
cleaned: dict[str, Any] = {}
|
||||
for key, value in details.items():
|
||||
key_text = str(key)
|
||||
if _is_secret_key(key_text):
|
||||
cleaned[key_text] = "[REDACTED]"
|
||||
elif isinstance(value, str):
|
||||
cleaned[key_text] = _safe_message(value, limit=500)
|
||||
elif isinstance(value, (int, float, bool)) or value is None:
|
||||
cleaned[key_text] = value
|
||||
else:
|
||||
cleaned[key_text] = _safe_message(str(value), limit=500)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _safe_message(value: str, *, limit: int = 240) -> str:
|
||||
text = " ".join(str(value or "").split())
|
||||
return text[:limit] if text else "image provider failed"
|
||||
|
||||
|
||||
def _is_secret_key(key: str) -> bool:
|
||||
lowered = key.lower()
|
||||
return any(keyword in lowered for keyword in SECRET_KEYWORDS)
|
||||
|
||||
|
||||
def _first_text(*values: Any) -> str:
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return "unknown"
|
||||
|
|
@ -121,7 +121,18 @@ class OpenAIImageProvider:
|
|||
"Content-Type": "application/json",
|
||||
**self.config.extra_headers,
|
||||
}
|
||||
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
|
||||
try:
|
||||
response = self._transport(self.config.endpoint_url(), payload, headers, self.config.timeout_seconds)
|
||||
except ImageProviderError as exc:
|
||||
if exc.provider_id != "unknown":
|
||||
raise
|
||||
raise ImageProviderError(
|
||||
str(exc),
|
||||
provider_id=self.provider_id,
|
||||
status_code=exc.status_code,
|
||||
error_type=exc.error_type,
|
||||
details=exc.details,
|
||||
) from exc
|
||||
return _parse_image_response(self.provider_id, model, response)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from typing import Any, Callable
|
|||
|
||||
from backend.dto.job_dto import GenerationTask
|
||||
from backend.dto.workflow_dto import WorkflowNode
|
||||
from core.generation_errors import provider_failure_attribution
|
||||
from core.storage import VfsStorage
|
||||
from engines.image_provider import ImageGenerationRequest, ImageProvider, ImageProviderError
|
||||
|
||||
|
|
@ -51,6 +52,7 @@ class SingleFlowEngine:
|
|||
try:
|
||||
return self._generate_with_provider(provider, task, request, prompt, model_route, width, height, count)
|
||||
except ImageProviderError as exc:
|
||||
route = request.get("resolved_model_route") if isinstance(request.get("resolved_model_route"), dict) else {}
|
||||
return self._generate_mock(
|
||||
task,
|
||||
request,
|
||||
|
|
@ -59,11 +61,7 @@ class SingleFlowEngine:
|
|||
width,
|
||||
height,
|
||||
count,
|
||||
provider_error={
|
||||
"provider_id": exc.provider_id,
|
||||
"error_type": exc.error_type,
|
||||
"status_code": exc.status_code,
|
||||
},
|
||||
provider_error=provider_failure_attribution(exc, route_context=route, fallback_used=True),
|
||||
)
|
||||
return self._generate_mock(task, request, prompt, model_route, width, height, count)
|
||||
|
||||
|
|
@ -133,7 +131,7 @@ class SingleFlowEngine:
|
|||
for index, image in enumerate(result.images):
|
||||
if not image.has_image_data:
|
||||
continue
|
||||
image_bytes = _image_bytes(image.b64_json, image.url)
|
||||
image_bytes = _image_bytes(image.b64_json, image.url, provider_id=result.provider_id)
|
||||
image_uri = f"storage://images/single/{task_stem}_{index + 1}.png"
|
||||
thumbnail_uri = f"storage://thumbnails/single/{task_stem}_{index + 1}.png"
|
||||
self._write_png(image_uri, image_bytes)
|
||||
|
|
@ -207,10 +205,18 @@ def _safe_file_stem(value: str) -> str:
|
|||
return stem or "task"
|
||||
|
||||
|
||||
def _image_bytes(b64_json: str | None, url: str | None) -> bytes:
|
||||
def _image_bytes(b64_json: str | None, url: str | None, *, provider_id: str = "unknown") -> bytes:
|
||||
if b64_json:
|
||||
return base64.b64decode(re.sub(r"\s+", "", b64_json))
|
||||
if url:
|
||||
with urllib.request.urlopen(url, timeout=120) as response:
|
||||
return response.read()
|
||||
raise ImageProviderError("image payload is empty", provider_id="unknown", error_type="invalid_response")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=120) as response:
|
||||
return response.read()
|
||||
except Exception as exc:
|
||||
raise ImageProviderError(
|
||||
"image provider image URL could not be downloaded",
|
||||
provider_id=provider_id,
|
||||
error_type="network_error",
|
||||
details={"reason": str(exc)},
|
||||
) from exc
|
||||
raise ImageProviderError("image payload is empty", provider_id=provider_id, error_type="invalid_response")
|
||||
|
|
|
|||
|
|
@ -66,6 +66,33 @@ class ImageProviderTests(unittest.TestCase):
|
|||
|
||||
self.assertEqual(raised.exception.error_type, "configuration_error")
|
||||
|
||||
def test_provider_relabels_transport_errors_with_configured_provider(self):
|
||||
def transport(url, payload, headers, timeout):
|
||||
raise ImageProviderError(
|
||||
"remote unauthorized",
|
||||
provider_id="unknown",
|
||||
status_code=401,
|
||||
error_type="http_error",
|
||||
details={"body_preview": "invalid credentials"},
|
||||
)
|
||||
|
||||
provider = OpenAIImageProvider(
|
||||
ImageProviderConfig(
|
||||
provider_id="site_b",
|
||||
base_url="https://botcf.com",
|
||||
api_key="secret",
|
||||
image_model="image2",
|
||||
),
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
with self.assertRaises(ImageProviderError) as raised:
|
||||
provider.generate_image(ImageGenerationRequest(prompt="red dot"))
|
||||
|
||||
self.assertEqual(raised.exception.provider_id, "site_b")
|
||||
self.assertEqual(raised.exception.status_code, 401)
|
||||
self.assertEqual(raised.exception.error_type, "http_error")
|
||||
|
||||
def test_mock_provider_returns_image_payload(self):
|
||||
result = MockImageProvider().generate_image(ImageGenerationRequest(prompt="red dot", model="mock"))
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from database.repositories.model_provider_repository import ModelProviderReposit
|
|||
from database.repositories.profile_repository import ProfileRepository
|
||||
from database.repositories.workflow_repository import WorkflowRepository
|
||||
from engines.single_flow import SingleFlowEngine
|
||||
from engines.image_provider import GeneratedImagePayload, ImageGenerationResult
|
||||
from engines.image_provider import GeneratedImagePayload, ImageGenerationResult, ImageProviderError
|
||||
|
||||
|
||||
class SingleFlowGenerationTests(unittest.TestCase):
|
||||
|
|
@ -325,6 +325,73 @@ storage:
|
|||
self.assertEqual(assets[0].metadata["usage"]["total_tokens"], 12)
|
||||
self.assertTrue(VfsStorage(self.config_service).ensure_file(assets[0].image_uri).exists())
|
||||
|
||||
def test_generate_node_records_provider_failure_attribution_on_fallback(self):
|
||||
class FailingProvider:
|
||||
provider_id = "site_b"
|
||||
|
||||
def generate_image(self, request):
|
||||
raise ImageProviderError(
|
||||
"remote unauthorized",
|
||||
provider_id=self.provider_id,
|
||||
status_code=401,
|
||||
error_type="http_error",
|
||||
details={"body_preview": "invalid credentials", "api_key": "secret-value"},
|
||||
)
|
||||
|
||||
self.model_provider_service.upsert_provider(
|
||||
{
|
||||
"provider_id": "site_b",
|
||||
"base_url": "https://botcf.com",
|
||||
"image_endpoint": "/v1/images/generations",
|
||||
}
|
||||
)
|
||||
self.model_provider_service.upsert_route(
|
||||
{
|
||||
"route_name": "image_high",
|
||||
"provider_id": "site_b",
|
||||
"model_id": "image2",
|
||||
"priority": 5,
|
||||
}
|
||||
)
|
||||
generation_service = GenerationService(
|
||||
workflow_service=self.workflow_service,
|
||||
queue_service=self.queue_service,
|
||||
image_service=self.image_service,
|
||||
single_flow_engine=SingleFlowEngine(
|
||||
VfsStorage(self.config_service),
|
||||
image_provider_factory=lambda route: FailingProvider(),
|
||||
),
|
||||
model_provider_service=self.model_provider_service,
|
||||
event_bus=self.event_bus,
|
||||
)
|
||||
profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx)
|
||||
workflow = self.workflow_service.create_workflow(
|
||||
profile.profile_id,
|
||||
{"workflow_type": "single_image", "display_name": "Single"},
|
||||
self.ctx,
|
||||
)
|
||||
|
||||
response = generate_node(
|
||||
self.ctx,
|
||||
workflow.nodes[0].node_id,
|
||||
{"prompt_patch": "white shirt", "model_route": "image_high"},
|
||||
generation_service=generation_service,
|
||||
)
|
||||
|
||||
self.assertTrue(response["ok"])
|
||||
task = self.queue_service.get_job(response["data"]["task_id"])
|
||||
failure = task.result["provider_failure"]
|
||||
self.assertEqual(failure["category"], "auth_error")
|
||||
self.assertEqual(failure["provider_id"], "site_b")
|
||||
self.assertEqual(failure["route_name"], "image_high")
|
||||
self.assertFalse(failure["retryable"])
|
||||
self.assertTrue(failure["fallback_used"])
|
||||
self.assertEqual(failure["details"]["api_key"], "[REDACTED]")
|
||||
self.assertNotIn("secret-value", str(task.to_dict()))
|
||||
image = self.image_service.get_image(response["data"]["images"][0]["image_id"])
|
||||
self.assertTrue(image.metadata["mock"])
|
||||
self.assertEqual(image.metadata["provider_error"]["category"], "auth_error")
|
||||
|
||||
def test_queue_service_rejects_invalid_task_transition(self):
|
||||
task = self.queue_service.create_task({"status": "succeeded"}, self.ctx)
|
||||
|
||||
|
|
|
|||
正在加载…
添加表格
在新工单中引用