2026-07-06 02:14:13 +08:00
|
|
|
|
"""Structured generation failure attribution helpers."""
|
|
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
|
import re
|
2026-07-06 02:14:13 +08:00
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
|
|
from engines.image_provider import ImageProviderError
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
SECRET_KEYWORDS = ("api_key", "apikey", "authorization", "bearer", "password", "secret", "token")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def provider_failure_attribution(
|
|
|
|
|
|
exc: ImageProviderError,
|
|
|
|
|
|
*,
|
|
|
|
|
|
route_context: dict[str, Any] | None = None,
|
|
|
|
|
|
fallback_used: bool = False,
|
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
|
route = route_context or {}
|
|
|
|
|
|
status_code = exc.status_code
|
|
|
|
|
|
category = _category(exc.error_type, status_code)
|
|
|
|
|
|
attribution = {
|
|
|
|
|
|
"source": "image_provider",
|
|
|
|
|
|
"category": category,
|
|
|
|
|
|
"error_type": exc.error_type,
|
|
|
|
|
|
"message": _safe_message(str(exc)),
|
|
|
|
|
|
"provider_id": _first_text(exc.provider_id, route.get("provider_id"), "unknown"),
|
|
|
|
|
|
"status_code": status_code,
|
|
|
|
|
|
"retryable": _is_retryable(category, status_code),
|
|
|
|
|
|
"fallback_used": bool(fallback_used),
|
|
|
|
|
|
"user_action_required": category in {"configuration_error", "auth_error", "request_error"},
|
|
|
|
|
|
}
|
|
|
|
|
|
for key in ("route_name", "route_id", "model_id", "modality"):
|
|
|
|
|
|
value = route.get(key)
|
|
|
|
|
|
if value is not None:
|
|
|
|
|
|
attribution[key] = value
|
|
|
|
|
|
details = _safe_details(exc.details)
|
|
|
|
|
|
if details:
|
|
|
|
|
|
attribution["details"] = details
|
|
|
|
|
|
return attribution
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
|
def provider_failure_user_message(failure: dict[str, Any]) -> str:
|
|
|
|
|
|
"""Build a concise, sanitized error that is useful in the UI and logs."""
|
|
|
|
|
|
|
|
|
|
|
|
category = str(failure.get("category") or "provider_error")
|
|
|
|
|
|
labels = {
|
|
|
|
|
|
"auth_error": "生成服务鉴权失败",
|
|
|
|
|
|
"configuration_error": "生成服务配置错误",
|
|
|
|
|
|
"network_error": "无法连接生成服务",
|
|
|
|
|
|
"timeout": "生成服务请求超时",
|
|
|
|
|
|
"rate_limited": "生成服务请求过于频繁",
|
|
|
|
|
|
"provider_unavailable": "生成服务暂时不可用",
|
|
|
|
|
|
"request_error": "生成服务拒绝了请求",
|
|
|
|
|
|
"invalid_response": "生成服务返回了无法识别的数据",
|
|
|
|
|
|
"provider_pool_exhausted": "所有候选生成服务均调用失败",
|
|
|
|
|
|
}
|
|
|
|
|
|
label = labels.get(category, "图像生成服务调用失败")
|
|
|
|
|
|
status_code = failure.get("status_code")
|
|
|
|
|
|
if status_code is not None:
|
|
|
|
|
|
label = f"{label}(HTTP {status_code})"
|
|
|
|
|
|
detail = _provider_response_message(failure.get("details"))
|
|
|
|
|
|
route_name = _first_text(failure.get("route_name"), failure.get("model_id"), "")
|
|
|
|
|
|
suffix = f";路由:{route_name}" if route_name != "unknown" else ""
|
|
|
|
|
|
return f"{label}{f':{detail}' if detail else ''}{suffix}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _provider_response_message(details: Any) -> str:
|
|
|
|
|
|
if not isinstance(details, dict):
|
|
|
|
|
|
return ""
|
|
|
|
|
|
body = details.get("body_preview")
|
|
|
|
|
|
if isinstance(body, str) and body.strip():
|
|
|
|
|
|
try:
|
|
|
|
|
|
import json
|
|
|
|
|
|
|
|
|
|
|
|
parsed = json.loads(body)
|
|
|
|
|
|
except (ValueError, TypeError):
|
|
|
|
|
|
return _safe_message(body, limit=180)
|
|
|
|
|
|
if isinstance(parsed, dict):
|
|
|
|
|
|
error = parsed.get("error")
|
|
|
|
|
|
if isinstance(error, dict):
|
|
|
|
|
|
message = error.get("message") or error.get("detail") or error.get("type")
|
|
|
|
|
|
if message:
|
|
|
|
|
|
return _safe_message(str(message), limit=180)
|
|
|
|
|
|
for key in ("message", "detail", "error"):
|
|
|
|
|
|
value = parsed.get(key)
|
|
|
|
|
|
if isinstance(value, str) and value.strip():
|
|
|
|
|
|
return _safe_message(value, limit=180)
|
|
|
|
|
|
reason = details.get("reason")
|
|
|
|
|
|
if not reason:
|
|
|
|
|
|
return ""
|
|
|
|
|
|
safe_reason = _safe_message(str(reason), limit=180)
|
|
|
|
|
|
lowered = safe_reason.lower()
|
|
|
|
|
|
if "unexpected_eof_while_reading" in lowered or "eof occurred in violation of protocol" in lowered:
|
|
|
|
|
|
return "上游生成服务在返回结果前提前关闭了 TLS 连接,通常是线路波动或网关超时,请稍后在队列中重试"
|
|
|
|
|
|
if "read operation timed out" in lowered or "readtimeout" in lowered:
|
|
|
|
|
|
return "上游生成服务长时间未返回结果,请稍后在队列中重试或提高适配器超时时间"
|
|
|
|
|
|
return safe_reason
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:14:13 +08:00
|
|
|
|
def _category(error_type: str, status_code: int | None) -> str:
|
|
|
|
|
|
if error_type == "configuration_error":
|
|
|
|
|
|
return "configuration_error"
|
|
|
|
|
|
if error_type == "network_error":
|
|
|
|
|
|
return "network_error"
|
|
|
|
|
|
if error_type == "invalid_response":
|
|
|
|
|
|
return "invalid_response"
|
|
|
|
|
|
if status_code in {401, 403}:
|
|
|
|
|
|
return "auth_error"
|
|
|
|
|
|
if status_code == 429:
|
|
|
|
|
|
return "rate_limited"
|
|
|
|
|
|
if status_code in {408, 504}:
|
|
|
|
|
|
return "timeout"
|
|
|
|
|
|
if status_code is not None and 500 <= status_code <= 599:
|
|
|
|
|
|
return "provider_unavailable"
|
|
|
|
|
|
if status_code is not None and 400 <= status_code <= 499:
|
|
|
|
|
|
return "request_error"
|
|
|
|
|
|
return error_type or "provider_error"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_retryable(category: str, status_code: int | None) -> bool:
|
|
|
|
|
|
if category in {"network_error", "timeout", "rate_limited", "provider_unavailable"}:
|
|
|
|
|
|
return True
|
|
|
|
|
|
return bool(status_code is not None and 500 <= status_code <= 599)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _safe_details(details: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
|
cleaned: dict[str, Any] = {}
|
|
|
|
|
|
for key, value in details.items():
|
|
|
|
|
|
key_text = str(key)
|
|
|
|
|
|
if _is_secret_key(key_text):
|
|
|
|
|
|
cleaned[key_text] = "[REDACTED]"
|
|
|
|
|
|
elif isinstance(value, str):
|
|
|
|
|
|
cleaned[key_text] = _safe_message(value, limit=500)
|
|
|
|
|
|
elif isinstance(value, (int, float, bool)) or value is None:
|
|
|
|
|
|
cleaned[key_text] = value
|
|
|
|
|
|
else:
|
|
|
|
|
|
cleaned[key_text] = _safe_message(str(value), limit=500)
|
|
|
|
|
|
return cleaned
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _safe_message(value: str, *, limit: int = 240) -> str:
|
|
|
|
|
|
text = " ".join(str(value or "").split())
|
2026-07-11 21:50:34 +08:00
|
|
|
|
text = re.sub(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+", "Bearer [REDACTED]", text)
|
|
|
|
|
|
text = re.sub(
|
|
|
|
|
|
r'''(?ix)(["']?(?:api[_-]?key|apikey|authorization|token|password|secret)["']?\s*[:=]\s*["']?)(?:Bearer\s+)?[^"'\s,;}]+''',
|
|
|
|
|
|
r"\1[REDACTED]",
|
|
|
|
|
|
text,
|
|
|
|
|
|
)
|
|
|
|
|
|
text = re.sub(r"\bsk-[A-Za-z0-9_-]{8,}\b", "[REDACTED]", text)
|
2026-07-06 02:14:13 +08:00
|
|
|
|
return text[:limit] if text else "image provider failed"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_secret_key(key: str) -> bool:
|
|
|
|
|
|
lowered = key.lower()
|
|
|
|
|
|
return any(keyword in lowered for keyword in SECRET_KEYWORDS)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _first_text(*values: Any) -> str:
|
|
|
|
|
|
for value in values:
|
|
|
|
|
|
text = str(value or "").strip()
|
|
|
|
|
|
if text:
|
|
|
|
|
|
return text
|
|
|
|
|
|
return "unknown"
|