433 行
16 KiB
Python
433 行
16 KiB
Python
"""Synchronous, lineage-preserving retries for persisted generation jobs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
from backend.dto.job_dto import GenerationTask
|
|
from backend.services.generation_service import GenerationService
|
|
from backend.services.queue_service import QueueService
|
|
from core.identity import RequestContext
|
|
|
|
|
|
MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES = 5
|
|
|
|
|
|
class GenerationJobRetryService:
|
|
"""Create and execute a replacement task from a persisted task request.
|
|
|
|
ImageStudio intentionally does not start QueueWorker in the AstrBot plugin
|
|
process: no production worker handler or lifecycle owner exists yet. A web
|
|
retry therefore has to execute through GenerationService instead of merely
|
|
changing the old row back to ``queued``.
|
|
"""
|
|
|
|
def __init__(self, queue_service: QueueService, generation_service: GenerationService):
|
|
self.queue_service = queue_service
|
|
self.generation_service = generation_service
|
|
|
|
def retry_job(
|
|
self,
|
|
task_id: str,
|
|
ctx: RequestContext,
|
|
*,
|
|
overrides: dict[str, Any] | None = None,
|
|
) -> GenerationTask:
|
|
original = self.queue_service.get_job(task_id)
|
|
self._validate_executable(original)
|
|
request_overrides = overrides or {}
|
|
self._validate_ephemeral_references(original, request_overrides)
|
|
operation_id = f"retry_{uuid4().hex}"
|
|
retry_key = f"{operation_id}:{original.task_id}"
|
|
claimed = self.queue_service.begin_retry(task_id, ctx, operation_id=operation_id)
|
|
previous_status = str(claimed.metadata["retry_execution"]["previous_status"])
|
|
request = self._replacement_request(
|
|
claimed,
|
|
operation_id=operation_id,
|
|
retry_key=retry_key,
|
|
overrides=request_overrides,
|
|
)
|
|
prepared_node = None
|
|
|
|
try:
|
|
prepared_node = self._prepare_node(claimed, ctx)
|
|
response = self.generation_service.generate_node(str(claimed.node_id), request, ctx)
|
|
replacement = self.queue_service.get_job(str(response["task_id"]))
|
|
except Exception as exc:
|
|
replacement = self.queue_service.get_job_by_idempotency_key(retry_key)
|
|
node_status_restored = replacement is not None
|
|
if replacement is None:
|
|
try:
|
|
node_status_restored = self._restore_prepared_node(prepared_node, claimed.task_id, ctx)
|
|
except Exception:
|
|
node_status_restored = False
|
|
self._restore_original(claimed, previous_status=previous_status, exc=exc, ctx=ctx)
|
|
else:
|
|
replacement = self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx)
|
|
raise self._retry_error(
|
|
exc,
|
|
original=claimed,
|
|
replacement=replacement,
|
|
operation_id=operation_id,
|
|
original_status_restored=replacement is None,
|
|
node_status_restored=node_status_restored,
|
|
) from exc
|
|
|
|
return self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx)
|
|
|
|
def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 5) -> dict[str, Any]:
|
|
applied_limit = _synchronous_batch_limit(limit)
|
|
jobs = self.queue_service.list_jobs(
|
|
status="dead_letter",
|
|
page=1,
|
|
page_size=applied_limit,
|
|
).items
|
|
retried: list[dict[str, Any]] = []
|
|
errors: list[dict[str, Any]] = []
|
|
for job in jobs:
|
|
try:
|
|
retried.append(self.retry_job(job.task_id, ctx).to_dict())
|
|
except ApiError as exc:
|
|
details = dict(exc.details or {})
|
|
errors.append(
|
|
{
|
|
"task_id": job.task_id,
|
|
"error_code": exc.error_code.value,
|
|
"message": exc.message,
|
|
"replacement_task_id": details.get("replacement_task_id"),
|
|
"retry_lineage": details.get("retry_lineage"),
|
|
}
|
|
)
|
|
return {
|
|
"execution_mode": "synchronous",
|
|
"requested_limit": _int_or_default(limit, 5),
|
|
"applied_limit": applied_limit,
|
|
"max_limit": MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES,
|
|
"retried": retried,
|
|
"errors": errors,
|
|
"total": len(retried),
|
|
"processed": len(retried) + len(errors),
|
|
"remaining": self.queue_service.queue_summary()["dead_letter"],
|
|
}
|
|
|
|
def _validate_executable(self, task: GenerationTask) -> None:
|
|
if task.task_type != "generation" or not task.node_id or not isinstance(task.request, dict):
|
|
raise ApiError(
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
"task has no executable generation request",
|
|
{
|
|
"task_id": task.task_id,
|
|
"task_type": task.task_type,
|
|
"node_id": task.node_id,
|
|
"request_type": type(task.request).__name__,
|
|
},
|
|
)
|
|
|
|
def _validate_ephemeral_references(self, task: GenerationTask, overrides: dict[str, Any]) -> None:
|
|
omitted = _ephemeral_reference_count(task.request)
|
|
if omitted and not _has_reference_override(overrides):
|
|
raise ApiError(
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
"task used inline reference images that were not persisted; provide replacement references",
|
|
{
|
|
"task_id": task.task_id,
|
|
"ephemeral_data_urls_omitted": omitted,
|
|
"required_fields": ["input_images", "image", "selected_draft_image_ids", "stage_inputs"],
|
|
},
|
|
)
|
|
|
|
def _prepare_node(self, task: GenerationTask, ctx: RequestContext) -> Any | None:
|
|
node = self.generation_service.workflow_service.get_node(str(task.node_id))
|
|
if node.status not in {"queued", "running"}:
|
|
return None
|
|
last_task_id = _optional_text(node.metadata.get("last_task_id"))
|
|
if last_task_id != task.task_id:
|
|
raise ApiError(
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
"node is already owned by another active generation task",
|
|
{
|
|
"task_id": task.task_id,
|
|
"node_id": node.node_id,
|
|
"active_task_id": last_task_id,
|
|
"node_status": node.status,
|
|
},
|
|
)
|
|
self.generation_service.workflow_service.update_node(
|
|
node.node_id,
|
|
{
|
|
"status": "failed",
|
|
"metadata": {
|
|
**node.metadata,
|
|
"retry_preparation": {
|
|
"task_id": task.task_id,
|
|
"prepared_at": _now(),
|
|
"previous_status": node.status,
|
|
},
|
|
},
|
|
},
|
|
ctx,
|
|
)
|
|
return node
|
|
|
|
def _restore_prepared_node(self, original_node: Any | None, task_id: str, ctx: RequestContext) -> bool:
|
|
if original_node is None:
|
|
return True
|
|
workflow_service = self.generation_service.workflow_service
|
|
current = workflow_service.get_node(original_node.node_id)
|
|
preparation = current.metadata.get("retry_preparation") if isinstance(current.metadata, dict) else None
|
|
if (
|
|
current.status != "failed"
|
|
or not isinstance(preparation, dict)
|
|
or _optional_text(preparation.get("task_id")) != task_id
|
|
):
|
|
return False
|
|
after = workflow_service.workflow_repository.update_node(
|
|
original_node.node_id,
|
|
{"status": original_node.status, "metadata": original_node.metadata},
|
|
)
|
|
if after is None:
|
|
return False
|
|
workflow_service.audit_service.record(
|
|
ctx=ctx,
|
|
action="rollback_retry_preparation",
|
|
target_type="workflow_node",
|
|
target_id=original_node.node_id,
|
|
before_snapshot=current.to_dict(),
|
|
after_snapshot=after.to_dict(),
|
|
metadata={"task_id": task_id},
|
|
)
|
|
workflow_service._publish(
|
|
"workflow.node_status_changed",
|
|
{
|
|
"workflow_id": after.workflow_id,
|
|
"node_id": after.node_id,
|
|
"before_status": current.status,
|
|
"after_status": after.status,
|
|
},
|
|
)
|
|
workflow_service._publish(
|
|
"workflow.node_updated",
|
|
{"workflow_id": after.workflow_id, "node_id": after.node_id},
|
|
)
|
|
return True
|
|
|
|
def _replacement_request(
|
|
self,
|
|
task: GenerationTask,
|
|
*,
|
|
operation_id: str,
|
|
retry_key: str,
|
|
overrides: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
request = {
|
|
key: value
|
|
for key, value in dict(task.request or {}).items()
|
|
if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key"
|
|
}
|
|
for key, value in overrides.items():
|
|
if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key":
|
|
request[key] = value
|
|
request.update(
|
|
{
|
|
"idempotency_key": retry_key,
|
|
"retry_of_task_id": task.task_id,
|
|
"retry_operation_id": operation_id,
|
|
}
|
|
)
|
|
if task.model_route and not _optional_text(request.get("model_route")):
|
|
request["model_route"] = task.model_route
|
|
return request
|
|
|
|
def _finalize_lineage(
|
|
self,
|
|
original: GenerationTask,
|
|
replacement: GenerationTask,
|
|
*,
|
|
operation_id: str,
|
|
ctx: RequestContext,
|
|
) -> GenerationTask:
|
|
previous_lineage = original.metadata.get("retry_lineage")
|
|
root_task_id = (
|
|
_optional_text(previous_lineage.get("root_task_id"))
|
|
if isinstance(previous_lineage, dict)
|
|
else None
|
|
) or original.task_id
|
|
lineage = {
|
|
"operation_id": operation_id,
|
|
"retry_of_task_id": original.task_id,
|
|
"root_task_id": root_task_id,
|
|
"original_status": original.metadata["retry_execution"]["previous_status"],
|
|
"replacement_task_id": replacement.task_id,
|
|
"replacement_status": replacement.status,
|
|
"completed_at": _now(),
|
|
}
|
|
updated_replacement = self.queue_service.update_task(
|
|
replacement.task_id,
|
|
{"metadata": {**replacement.metadata, "retry_lineage": lineage}},
|
|
ctx,
|
|
)
|
|
latest_original = self.queue_service.get_job(original.task_id)
|
|
retry_execution = dict(latest_original.metadata.get("retry_execution") or {})
|
|
self.queue_service.update_task(
|
|
original.task_id,
|
|
{
|
|
"status": "superseded",
|
|
"progress": 100,
|
|
"result": {
|
|
**latest_original.result,
|
|
"replacement_task_id": updated_replacement.task_id,
|
|
},
|
|
"metadata": {
|
|
**latest_original.metadata,
|
|
"retry_execution": {
|
|
**retry_execution,
|
|
"status": "completed",
|
|
"completed_at": lineage["completed_at"],
|
|
"replacement_task_id": updated_replacement.task_id,
|
|
"replacement_status": updated_replacement.status,
|
|
},
|
|
},
|
|
},
|
|
ctx,
|
|
)
|
|
return updated_replacement
|
|
|
|
def _restore_original(
|
|
self,
|
|
original: GenerationTask,
|
|
*,
|
|
previous_status: str,
|
|
exc: Exception,
|
|
ctx: RequestContext,
|
|
) -> None:
|
|
latest = self.queue_service.get_job(original.task_id)
|
|
if latest.status != "retrying":
|
|
return
|
|
retry_execution = dict(latest.metadata.get("retry_execution") or {})
|
|
self.queue_service.update_task(
|
|
original.task_id,
|
|
{
|
|
"status": previous_status,
|
|
"metadata": {
|
|
**latest.metadata,
|
|
"retry_execution": {
|
|
**retry_execution,
|
|
"status": "rejected",
|
|
"rejected_at": _now(),
|
|
"error_type": type(exc).__name__,
|
|
"error_message": _safe_error_message(exc),
|
|
},
|
|
},
|
|
},
|
|
ctx,
|
|
)
|
|
|
|
def _retry_error(
|
|
self,
|
|
exc: Exception,
|
|
*,
|
|
original: GenerationTask,
|
|
replacement: GenerationTask | None,
|
|
operation_id: str,
|
|
original_status_restored: bool,
|
|
node_status_restored: bool,
|
|
) -> ApiError:
|
|
replacement_task_id = replacement.task_id if replacement else None
|
|
lineage = {
|
|
"operation_id": operation_id,
|
|
"retry_of_task_id": original.task_id,
|
|
"replacement_task_id": replacement_task_id,
|
|
"replacement_status": replacement.status if replacement else None,
|
|
}
|
|
details = {
|
|
**(dict(exc.details or {}) if isinstance(exc, ApiError) else {}),
|
|
"retry_of_task_id": original.task_id,
|
|
"replacement_task_id": replacement_task_id,
|
|
"original_status_restored": original_status_restored,
|
|
"node_status_restored": node_status_restored,
|
|
"retry_lineage": lineage,
|
|
}
|
|
if isinstance(exc, ApiError):
|
|
return ApiError(exc.error_code, exc.message, details)
|
|
return ApiError(ErrorCode.GENERATION_FAILED, "task retry execution failed", details)
|
|
|
|
|
|
def _synchronous_batch_limit(value: Any) -> int:
|
|
return min(max(_int_or_default(value, 5), 1), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
|
|
|
|
def _int_or_default(value: Any, default: int) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
def _safe_error_message(exc: Exception) -> str:
|
|
text = " ".join(str(exc or type(exc).__name__).split())
|
|
return text[:500] if text else type(exc).__name__
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
_DERIVED_REQUEST_FIELDS = frozenset(
|
|
{
|
|
"cost_guard",
|
|
"expanded_negative_prompt",
|
|
"expanded_prompt",
|
|
"prompt_expansion",
|
|
"reference_persistence",
|
|
"resolved_model_route",
|
|
}
|
|
)
|
|
|
|
|
|
def _ephemeral_reference_count(request: dict[str, Any]) -> int:
|
|
persistence = request.get("reference_persistence")
|
|
if not isinstance(persistence, dict):
|
|
return 0
|
|
try:
|
|
return max(int(persistence.get("ephemeral_data_urls_omitted") or 0), 0)
|
|
except (TypeError, ValueError):
|
|
return 0
|
|
|
|
|
|
def _has_reference_override(payload: dict[str, Any]) -> bool:
|
|
for key in ("input_images", "image", "selected_draft_image_ids", "stage_inputs"):
|
|
if _has_nonempty_value(payload.get(key)):
|
|
return True
|
|
metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}
|
|
dependency = payload.get("dependency_context") if isinstance(payload.get("dependency_context"), dict) else {}
|
|
return any(
|
|
_has_nonempty_value(value)
|
|
for value in (
|
|
metadata.get("selected_draft_image_ids"),
|
|
metadata.get("stage_inputs"),
|
|
dependency.get("selected_image_ids"),
|
|
dependency.get("selected_draft_image_ids"),
|
|
dependency.get("stage_inputs"),
|
|
dependency.get("material_bindings"),
|
|
)
|
|
)
|
|
|
|
|
|
def _has_nonempty_value(value: Any) -> bool:
|
|
if value is None:
|
|
return False
|
|
if isinstance(value, str):
|
|
return bool(value.strip())
|
|
if isinstance(value, (list, tuple, dict, set)):
|
|
return bool(value)
|
|
return True
|