astrbot_plugin_imagestudio/backend/services/generation_job_retry_service.py

545 行
21 KiB
Python

"""Synchronous, lineage-preserving retries for persisted generation jobs."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import uuid4
from backend.dto.common import ApiError, ErrorCode
from backend.dto.job_dto import GenerationTask
from backend.services.generation_service import GenerationService
from backend.services.queue_service import QueueService
from core.identity import RequestContext
MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES = 5
class GenerationJobRetryService:
"""Create and execute a replacement task from a persisted task request.
ImageStudio intentionally does not start QueueWorker in the AstrBot plugin
process: no production worker handler or lifecycle owner exists yet. A web
retry therefore has to execute through GenerationService instead of merely
changing the old row back to ``queued``.
"""
def __init__(self, queue_service: QueueService, generation_service: GenerationService):
self.queue_service = queue_service
self.generation_service = generation_service
def retry_job(
self,
task_id: str,
ctx: RequestContext,
*,
overrides: dict[str, Any] | None = None,
) -> GenerationTask:
original = self.queue_service.get_job(task_id)
self._validate_executable(original)
request_overrides = overrides or {}
self._validate_ephemeral_references(original, request_overrides)
operation_id = f"retry_{uuid4().hex}"
retry_key = f"{operation_id}:{original.task_id}"
claimed = self.queue_service.begin_retry(task_id, ctx, operation_id=operation_id)
previous_status = str(claimed.metadata["retry_execution"]["previous_status"])
request = self._replacement_request(
claimed,
operation_id=operation_id,
retry_key=retry_key,
overrides=request_overrides,
)
prepared_node = None
prepared_workflow = None
try:
prepared_workflow = self._prepare_workflow(claimed, ctx)
prepared_node = self._prepare_node(claimed, ctx)
response = self.generation_service.generate_node(str(claimed.node_id), request, ctx)
replacement = self.queue_service.get_job(str(response["task_id"]))
except Exception as exc:
replacement = self.queue_service.get_job_by_idempotency_key(retry_key)
node_status_restored = replacement is not None
if replacement is None:
try:
node_status_restored = self._restore_prepared_node(prepared_node, claimed.task_id, ctx)
except Exception:
node_status_restored = False
try:
self._restore_prepared_workflow(prepared_workflow, claimed.task_id, ctx)
except Exception:
pass
self._restore_original(claimed, previous_status=previous_status, exc=exc, ctx=ctx)
else:
replacement = self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx)
raise self._retry_error(
exc,
original=claimed,
replacement=replacement,
operation_id=operation_id,
original_status_restored=replacement is None,
node_status_restored=node_status_restored,
) from exc
return self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx)
def _prepare_workflow(self, task: GenerationTask, ctx: RequestContext) -> Any | None:
if not task.workflow_id:
return None
workflow_service = self.generation_service.workflow_service
workflow = workflow_service.get_workflow(str(task.workflow_id)).workflow
if workflow.workflow_type not in {"character_fast", "character_gradual"}:
return None
if workflow.status in {"active", "running"}:
return None
if workflow.status not in {"failed", "cancelled"}:
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"character workflow cannot be reopened for this retry",
{
"task_id": task.task_id,
"workflow_id": workflow.workflow_id,
"workflow_status": workflow.status,
},
)
prepared_metadata = dict(workflow.metadata or {})
prepared_metadata.pop("quick_character_failure", None)
prepared_metadata["retry_preparation"] = {
"task_id": task.task_id,
"prepared_at": _now(),
"previous_status": workflow.status,
}
workflow_service.update_workflow(
workflow.workflow_id,
{
"status": "running",
"metadata": prepared_metadata,
},
ctx,
)
return workflow
def _restore_prepared_workflow(self, original_workflow: Any | None, task_id: str, ctx: RequestContext) -> bool:
if original_workflow is None:
return True
workflow_service = self.generation_service.workflow_service
current = workflow_service.get_workflow(original_workflow.workflow_id).workflow
preparation = current.metadata.get("retry_preparation") if isinstance(current.metadata, dict) else None
if (
current.status != "running"
or not isinstance(preparation, dict)
or _optional_text(preparation.get("task_id")) != task_id
):
return False
workflow_service.update_workflow(
original_workflow.workflow_id,
{
"status": original_workflow.status,
"metadata": _metadata_without_owned_retry_preparation(current.metadata, task_id),
},
ctx,
)
return True
def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 5) -> dict[str, Any]:
applied_limit = _synchronous_batch_limit(limit)
jobs = self.queue_service.list_jobs(
status="dead_letter",
page=1,
page_size=applied_limit,
).items
retried: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
for job in jobs:
try:
retried.append(self.retry_job(job.task_id, ctx).to_dict())
except ApiError as exc:
details = dict(exc.details or {})
errors.append(
{
"task_id": job.task_id,
"error_code": exc.error_code.value,
"message": exc.message,
"replacement_task_id": details.get("replacement_task_id"),
"retry_lineage": details.get("retry_lineage"),
}
)
return {
"execution_mode": "synchronous",
"requested_limit": _int_or_default(limit, 5),
"applied_limit": applied_limit,
"max_limit": MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES,
"retried": retried,
"errors": errors,
"total": len(retried),
"processed": len(retried) + len(errors),
"remaining": self.queue_service.queue_summary()["dead_letter"],
}
def _validate_executable(self, task: GenerationTask) -> None:
if task.task_type != "generation" or not task.node_id or not isinstance(task.request, dict):
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task has no executable generation request",
{
"task_id": task.task_id,
"task_type": task.task_type,
"node_id": task.node_id,
"request_type": type(task.request).__name__,
},
)
def _validate_ephemeral_references(self, task: GenerationTask, overrides: dict[str, Any]) -> None:
omitted = _ephemeral_reference_count(task.request)
if omitted and not _has_reference_override(overrides):
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task used inline reference images that were not persisted; provide replacement references",
{
"task_id": task.task_id,
"ephemeral_data_urls_omitted": omitted,
"required_fields": ["input_images", "image", "selected_draft_image_ids", "stage_inputs"],
},
)
def _prepare_node(self, task: GenerationTask, ctx: RequestContext) -> Any | None:
node = self.generation_service.workflow_service.get_node(str(task.node_id))
if node.status not in {"queued", "running"}:
return None
last_task_id = _optional_text(node.metadata.get("last_task_id"))
if last_task_id != task.task_id:
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"node is already owned by another active generation task",
{
"task_id": task.task_id,
"node_id": node.node_id,
"active_task_id": last_task_id,
"node_status": node.status,
},
)
self.generation_service.workflow_service.update_node(
node.node_id,
{
"status": "failed",
"metadata": {
**node.metadata,
"retry_preparation": {
"task_id": task.task_id,
"prepared_at": _now(),
"previous_status": node.status,
},
},
},
ctx,
)
return node
def _restore_prepared_node(self, original_node: Any | None, task_id: str, ctx: RequestContext) -> bool:
if original_node is None:
return True
workflow_service = self.generation_service.workflow_service
current = workflow_service.get_node(original_node.node_id)
preparation = current.metadata.get("retry_preparation") if isinstance(current.metadata, dict) else None
if (
current.status != "failed"
or not isinstance(preparation, dict)
or _optional_text(preparation.get("task_id")) != task_id
):
return False
after = workflow_service.workflow_repository.update_node(
original_node.node_id,
{
"status": original_node.status,
"metadata": _metadata_without_owned_retry_preparation(current.metadata, task_id),
},
)
if after is None:
return False
workflow_service.audit_service.record(
ctx=ctx,
action="rollback_retry_preparation",
target_type="workflow_node",
target_id=original_node.node_id,
before_snapshot=current.to_dict(),
after_snapshot=after.to_dict(),
metadata={"task_id": task_id},
)
workflow_service._publish(
"workflow.node_status_changed",
{
"workflow_id": after.workflow_id,
"node_id": after.node_id,
"before_status": current.status,
"after_status": after.status,
},
)
workflow_service._publish(
"workflow.node_updated",
{"workflow_id": after.workflow_id, "node_id": after.node_id},
)
return True
def _replacement_request(
self,
task: GenerationTask,
*,
operation_id: str,
retry_key: str,
overrides: dict[str, Any],
) -> dict[str, Any]:
request = {
key: value
for key, value in dict(task.request or {}).items()
if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key"
}
for key, value in overrides.items():
if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key":
request[key] = value
request.update(
{
"idempotency_key": retry_key,
"retry_of_task_id": task.task_id,
"retry_operation_id": operation_id,
}
)
if task.model_route and not _optional_text(request.get("model_route")):
request["model_route"] = task.model_route
return request
def _finalize_lineage(
self,
original: GenerationTask,
replacement: GenerationTask,
*,
operation_id: str,
ctx: RequestContext,
) -> GenerationTask:
previous_lineage = original.metadata.get("retry_lineage")
root_task_id = (
_optional_text(previous_lineage.get("root_task_id"))
if isinstance(previous_lineage, dict)
else None
) or original.task_id
lineage = {
"operation_id": operation_id,
"retry_of_task_id": original.task_id,
"root_task_id": root_task_id,
"original_status": original.metadata["retry_execution"]["previous_status"],
"replacement_task_id": replacement.task_id,
"replacement_status": replacement.status,
"completed_at": _now(),
}
updated_replacement = self.queue_service.update_task(
replacement.task_id,
{"metadata": {**replacement.metadata, "retry_lineage": lineage}},
ctx,
)
latest_original = self.queue_service.get_job(original.task_id)
retry_execution = dict(latest_original.metadata.get("retry_execution") or {})
superseded = self.queue_service.update_task_if_status(
original.task_id,
{"retrying"},
{
"status": "superseded",
"progress": 100,
"result": {
**latest_original.result,
"replacement_task_id": updated_replacement.task_id,
},
"metadata": {
**latest_original.metadata,
"retry_execution": {
**retry_execution,
"status": "completed",
"completed_at": lineage["completed_at"],
"replacement_task_id": updated_replacement.task_id,
"replacement_status": updated_replacement.status,
},
},
},
ctx,
)
if superseded is None:
current = self.queue_service.get_job(original.task_id)
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"original task changed before retry lineage was finalized",
{
"task_id": original.task_id,
"status": current.status,
"replacement_task_id": updated_replacement.task_id,
},
)
self._clear_retry_preparation(original, ctx)
return updated_replacement
def _clear_retry_preparation(self, original: GenerationTask, ctx: RequestContext) -> None:
workflow_service = self.generation_service.workflow_service
if original.node_id:
try:
node = workflow_service.get_node(str(original.node_id))
metadata = dict(node.metadata or {})
preparation = metadata.get("retry_preparation")
if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == original.task_id:
metadata.pop("retry_preparation", None)
workflow_service.update_node(node.node_id, {"metadata": metadata}, ctx)
except Exception:
pass
if original.workflow_id:
try:
workflow = workflow_service.get_workflow(str(original.workflow_id)).workflow
metadata = dict(workflow.metadata or {})
preparation = metadata.get("retry_preparation")
if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == original.task_id:
metadata.pop("retry_preparation", None)
workflow_service.update_workflow(workflow.workflow_id, {"metadata": metadata}, ctx)
except Exception:
pass
def _restore_original(
self,
original: GenerationTask,
*,
previous_status: str,
exc: Exception,
ctx: RequestContext,
) -> None:
latest = self.queue_service.get_job(original.task_id)
if latest.status != "retrying":
return
retry_execution = dict(latest.metadata.get("retry_execution") or {})
self.queue_service.update_task_if_status(
original.task_id,
{"retrying"},
{
"status": previous_status,
"metadata": {
**latest.metadata,
"retry_execution": {
**retry_execution,
"status": "rejected",
"rejected_at": _now(),
"error_type": type(exc).__name__,
"error_message": _safe_error_message(exc),
},
},
},
ctx,
)
def _retry_error(
self,
exc: Exception,
*,
original: GenerationTask,
replacement: GenerationTask | None,
operation_id: str,
original_status_restored: bool,
node_status_restored: bool,
) -> ApiError:
replacement_task_id = replacement.task_id if replacement else None
lineage = {
"operation_id": operation_id,
"retry_of_task_id": original.task_id,
"replacement_task_id": replacement_task_id,
"replacement_status": replacement.status if replacement else None,
}
details = {
**(dict(exc.details or {}) if isinstance(exc, ApiError) else {}),
"retry_of_task_id": original.task_id,
"replacement_task_id": replacement_task_id,
"original_status_restored": original_status_restored,
"node_status_restored": node_status_restored,
"retry_lineage": lineage,
}
if isinstance(exc, ApiError):
return ApiError(exc.error_code, exc.message, details)
return ApiError(ErrorCode.GENERATION_FAILED, "task retry execution failed", details)
def _synchronous_batch_limit(value: Any) -> int:
return min(max(_int_or_default(value, 5), 1), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
def _int_or_default(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _optional_text(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _safe_error_message(exc: Exception) -> str:
text = " ".join(str(exc or type(exc).__name__).split())
return text[:500] if text else type(exc).__name__
def _metadata_without_owned_retry_preparation(metadata: Any, task_id: str) -> dict[str, Any]:
current = dict(metadata or {}) if isinstance(metadata, dict) else {}
preparation = current.get("retry_preparation")
if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == task_id:
current.pop("retry_preparation", None)
return current
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
_DERIVED_REQUEST_FIELDS = frozenset(
{
"cost_guard",
"expanded_negative_prompt",
"expanded_prompt",
"prompt_expansion",
"reference_persistence",
"resolved_model_route",
}
)
def _ephemeral_reference_count(request: dict[str, Any]) -> int:
persistence = request.get("reference_persistence")
if not isinstance(persistence, dict):
return 0
try:
return max(int(persistence.get("ephemeral_data_urls_omitted") or 0), 0)
except (TypeError, ValueError):
return 0
def _has_reference_override(payload: dict[str, Any]) -> bool:
for key in ("input_images", "image", "selected_draft_image_ids", "stage_inputs"):
if _has_nonempty_value(payload.get(key)):
return True
metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}
dependency = payload.get("dependency_context") if isinstance(payload.get("dependency_context"), dict) else {}
return any(
_has_nonempty_value(value)
for value in (
metadata.get("selected_draft_image_ids"),
metadata.get("stage_inputs"),
dependency.get("selected_image_ids"),
dependency.get("selected_draft_image_ids"),
dependency.get("stage_inputs"),
dependency.get("material_bindings"),
)
)
def _has_nonempty_value(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple, dict, set)):
return bool(value)
return True