"""Synchronous, lineage-preserving retries for persisted generation jobs.""" from __future__ import annotations from datetime import datetime, timezone from typing import Any from uuid import uuid4 from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask from backend.services.generation_service import GenerationService from backend.services.queue_service import QueueService from core.identity import RequestContext MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES = 5 class GenerationJobRetryService: """Create and execute a replacement task from a persisted task request. ImageStudio intentionally does not start QueueWorker in the AstrBot plugin process: no production worker handler or lifecycle owner exists yet. A web retry therefore has to execute through GenerationService instead of merely changing the old row back to ``queued``. """ def __init__(self, queue_service: QueueService, generation_service: GenerationService): self.queue_service = queue_service self.generation_service = generation_service def retry_job( self, task_id: str, ctx: RequestContext, *, overrides: dict[str, Any] | None = None, ) -> GenerationTask: original = self.queue_service.get_job(task_id) self._validate_executable(original) request_overrides = overrides or {} self._validate_ephemeral_references(original, request_overrides) operation_id = f"retry_{uuid4().hex}" retry_key = f"{operation_id}:{original.task_id}" claimed = self.queue_service.begin_retry(task_id, ctx, operation_id=operation_id) previous_status = str(claimed.metadata["retry_execution"]["previous_status"]) request = self._replacement_request( claimed, operation_id=operation_id, retry_key=retry_key, overrides=request_overrides, ) prepared_node = None prepared_workflow = None try: prepared_workflow = self._prepare_workflow(claimed, ctx) prepared_node = self._prepare_node(claimed, ctx) response = self.generation_service.generate_node(str(claimed.node_id), request, ctx) replacement = self.queue_service.get_job(str(response["task_id"])) except Exception as exc: replacement = self.queue_service.get_job_by_idempotency_key(retry_key) node_status_restored = replacement is not None if replacement is None: try: node_status_restored = self._restore_prepared_node(prepared_node, claimed.task_id, ctx) except Exception: node_status_restored = False try: self._restore_prepared_workflow(prepared_workflow, claimed.task_id, ctx) except Exception: pass self._restore_original(claimed, previous_status=previous_status, exc=exc, ctx=ctx) else: replacement = self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx) raise self._retry_error( exc, original=claimed, replacement=replacement, operation_id=operation_id, original_status_restored=replacement is None, node_status_restored=node_status_restored, ) from exc return self._finalize_lineage(claimed, replacement, operation_id=operation_id, ctx=ctx) def _prepare_workflow(self, task: GenerationTask, ctx: RequestContext) -> Any | None: if not task.workflow_id: return None workflow_service = self.generation_service.workflow_service workflow = workflow_service.get_workflow(str(task.workflow_id)).workflow if workflow.workflow_type not in {"character_fast", "character_gradual"}: return None if workflow.status in {"active", "running"}: return None if workflow.status not in {"failed", "cancelled"}: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "character workflow cannot be reopened for this retry", { "task_id": task.task_id, "workflow_id": workflow.workflow_id, "workflow_status": workflow.status, }, ) prepared_metadata = dict(workflow.metadata or {}) prepared_metadata.pop("quick_character_failure", None) prepared_metadata["retry_preparation"] = { "task_id": task.task_id, "prepared_at": _now(), "previous_status": workflow.status, } workflow_service.update_workflow( workflow.workflow_id, { "status": "running", "metadata": prepared_metadata, }, ctx, ) return workflow def _restore_prepared_workflow(self, original_workflow: Any | None, task_id: str, ctx: RequestContext) -> bool: if original_workflow is None: return True workflow_service = self.generation_service.workflow_service current = workflow_service.get_workflow(original_workflow.workflow_id).workflow preparation = current.metadata.get("retry_preparation") if isinstance(current.metadata, dict) else None if ( current.status != "running" or not isinstance(preparation, dict) or _optional_text(preparation.get("task_id")) != task_id ): return False workflow_service.update_workflow( original_workflow.workflow_id, { "status": original_workflow.status, "metadata": _metadata_without_owned_retry_preparation(current.metadata, task_id), }, ctx, ) return True def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 5) -> dict[str, Any]: applied_limit = _synchronous_batch_limit(limit) jobs = self.queue_service.list_jobs( status="dead_letter", page=1, page_size=applied_limit, ).items retried: list[dict[str, Any]] = [] errors: list[dict[str, Any]] = [] for job in jobs: try: retried.append(self.retry_job(job.task_id, ctx).to_dict()) except ApiError as exc: details = dict(exc.details or {}) errors.append( { "task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message, "replacement_task_id": details.get("replacement_task_id"), "retry_lineage": details.get("retry_lineage"), } ) return { "execution_mode": "synchronous", "requested_limit": _int_or_default(limit, 5), "applied_limit": applied_limit, "max_limit": MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES, "retried": retried, "errors": errors, "total": len(retried), "processed": len(retried) + len(errors), "remaining": self.queue_service.queue_summary()["dead_letter"], } def _validate_executable(self, task: GenerationTask) -> None: if task.task_type != "generation" or not task.node_id or not isinstance(task.request, dict): raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task has no executable generation request", { "task_id": task.task_id, "task_type": task.task_type, "node_id": task.node_id, "request_type": type(task.request).__name__, }, ) def _validate_ephemeral_references(self, task: GenerationTask, overrides: dict[str, Any]) -> None: omitted = _ephemeral_reference_count(task.request) if omitted and not _has_reference_override(overrides): raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task used inline reference images that were not persisted; provide replacement references", { "task_id": task.task_id, "ephemeral_data_urls_omitted": omitted, "required_fields": ["input_images", "image", "selected_draft_image_ids", "stage_inputs"], }, ) def _prepare_node(self, task: GenerationTask, ctx: RequestContext) -> Any | None: node = self.generation_service.workflow_service.get_node(str(task.node_id)) if node.status not in {"queued", "running"}: return None last_task_id = _optional_text(node.metadata.get("last_task_id")) if last_task_id != task.task_id: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "node is already owned by another active generation task", { "task_id": task.task_id, "node_id": node.node_id, "active_task_id": last_task_id, "node_status": node.status, }, ) self.generation_service.workflow_service.update_node( node.node_id, { "status": "failed", "metadata": { **node.metadata, "retry_preparation": { "task_id": task.task_id, "prepared_at": _now(), "previous_status": node.status, }, }, }, ctx, ) return node def _restore_prepared_node(self, original_node: Any | None, task_id: str, ctx: RequestContext) -> bool: if original_node is None: return True workflow_service = self.generation_service.workflow_service current = workflow_service.get_node(original_node.node_id) preparation = current.metadata.get("retry_preparation") if isinstance(current.metadata, dict) else None if ( current.status != "failed" or not isinstance(preparation, dict) or _optional_text(preparation.get("task_id")) != task_id ): return False after = workflow_service.workflow_repository.update_node( original_node.node_id, { "status": original_node.status, "metadata": _metadata_without_owned_retry_preparation(current.metadata, task_id), }, ) if after is None: return False workflow_service.audit_service.record( ctx=ctx, action="rollback_retry_preparation", target_type="workflow_node", target_id=original_node.node_id, before_snapshot=current.to_dict(), after_snapshot=after.to_dict(), metadata={"task_id": task_id}, ) workflow_service._publish( "workflow.node_status_changed", { "workflow_id": after.workflow_id, "node_id": after.node_id, "before_status": current.status, "after_status": after.status, }, ) workflow_service._publish( "workflow.node_updated", {"workflow_id": after.workflow_id, "node_id": after.node_id}, ) return True def _replacement_request( self, task: GenerationTask, *, operation_id: str, retry_key: str, overrides: dict[str, Any], ) -> dict[str, Any]: request = { key: value for key, value in dict(task.request or {}).items() if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key" } for key, value in overrides.items(): if key not in _DERIVED_REQUEST_FIELDS and key != "idempotency_key": request[key] = value request.update( { "idempotency_key": retry_key, "retry_of_task_id": task.task_id, "retry_operation_id": operation_id, } ) if task.model_route and not _optional_text(request.get("model_route")): request["model_route"] = task.model_route return request def _finalize_lineage( self, original: GenerationTask, replacement: GenerationTask, *, operation_id: str, ctx: RequestContext, ) -> GenerationTask: previous_lineage = original.metadata.get("retry_lineage") root_task_id = ( _optional_text(previous_lineage.get("root_task_id")) if isinstance(previous_lineage, dict) else None ) or original.task_id lineage = { "operation_id": operation_id, "retry_of_task_id": original.task_id, "root_task_id": root_task_id, "original_status": original.metadata["retry_execution"]["previous_status"], "replacement_task_id": replacement.task_id, "replacement_status": replacement.status, "completed_at": _now(), } updated_replacement = self.queue_service.update_task( replacement.task_id, {"metadata": {**replacement.metadata, "retry_lineage": lineage}}, ctx, ) latest_original = self.queue_service.get_job(original.task_id) retry_execution = dict(latest_original.metadata.get("retry_execution") or {}) superseded = self.queue_service.update_task_if_status( original.task_id, {"retrying"}, { "status": "superseded", "progress": 100, "result": { **latest_original.result, "replacement_task_id": updated_replacement.task_id, }, "metadata": { **latest_original.metadata, "retry_execution": { **retry_execution, "status": "completed", "completed_at": lineage["completed_at"], "replacement_task_id": updated_replacement.task_id, "replacement_status": updated_replacement.status, }, }, }, ctx, ) if superseded is None: current = self.queue_service.get_job(original.task_id) raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "original task changed before retry lineage was finalized", { "task_id": original.task_id, "status": current.status, "replacement_task_id": updated_replacement.task_id, }, ) self._clear_retry_preparation(original, ctx) return updated_replacement def _clear_retry_preparation(self, original: GenerationTask, ctx: RequestContext) -> None: workflow_service = self.generation_service.workflow_service if original.node_id: try: node = workflow_service.get_node(str(original.node_id)) metadata = dict(node.metadata or {}) preparation = metadata.get("retry_preparation") if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == original.task_id: metadata.pop("retry_preparation", None) workflow_service.update_node(node.node_id, {"metadata": metadata}, ctx) except Exception: pass if original.workflow_id: try: workflow = workflow_service.get_workflow(str(original.workflow_id)).workflow metadata = dict(workflow.metadata or {}) preparation = metadata.get("retry_preparation") if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == original.task_id: metadata.pop("retry_preparation", None) workflow_service.update_workflow(workflow.workflow_id, {"metadata": metadata}, ctx) except Exception: pass def _restore_original( self, original: GenerationTask, *, previous_status: str, exc: Exception, ctx: RequestContext, ) -> None: latest = self.queue_service.get_job(original.task_id) if latest.status != "retrying": return retry_execution = dict(latest.metadata.get("retry_execution") or {}) self.queue_service.update_task_if_status( original.task_id, {"retrying"}, { "status": previous_status, "metadata": { **latest.metadata, "retry_execution": { **retry_execution, "status": "rejected", "rejected_at": _now(), "error_type": type(exc).__name__, "error_message": _safe_error_message(exc), }, }, }, ctx, ) def _retry_error( self, exc: Exception, *, original: GenerationTask, replacement: GenerationTask | None, operation_id: str, original_status_restored: bool, node_status_restored: bool, ) -> ApiError: replacement_task_id = replacement.task_id if replacement else None lineage = { "operation_id": operation_id, "retry_of_task_id": original.task_id, "replacement_task_id": replacement_task_id, "replacement_status": replacement.status if replacement else None, } details = { **(dict(exc.details or {}) if isinstance(exc, ApiError) else {}), "retry_of_task_id": original.task_id, "replacement_task_id": replacement_task_id, "original_status_restored": original_status_restored, "node_status_restored": node_status_restored, "retry_lineage": lineage, } if isinstance(exc, ApiError): return ApiError(exc.error_code, exc.message, details) return ApiError(ErrorCode.GENERATION_FAILED, "task retry execution failed", details) def _synchronous_batch_limit(value: Any) -> int: return min(max(_int_or_default(value, 5), 1), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) def _int_or_default(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _safe_error_message(exc: Exception) -> str: text = " ".join(str(exc or type(exc).__name__).split()) return text[:500] if text else type(exc).__name__ def _metadata_without_owned_retry_preparation(metadata: Any, task_id: str) -> dict[str, Any]: current = dict(metadata or {}) if isinstance(metadata, dict) else {} preparation = current.get("retry_preparation") if isinstance(preparation, dict) and _optional_text(preparation.get("task_id")) == task_id: current.pop("retry_preparation", None) return current def _now() -> str: return datetime.now(timezone.utc).isoformat() _DERIVED_REQUEST_FIELDS = frozenset( { "cost_guard", "expanded_negative_prompt", "expanded_prompt", "prompt_expansion", "reference_persistence", "resolved_model_route", } ) def _ephemeral_reference_count(request: dict[str, Any]) -> int: persistence = request.get("reference_persistence") if not isinstance(persistence, dict): return 0 try: return max(int(persistence.get("ephemeral_data_urls_omitted") or 0), 0) except (TypeError, ValueError): return 0 def _has_reference_override(payload: dict[str, Any]) -> bool: for key in ("input_images", "image", "selected_draft_image_ids", "stage_inputs"): if _has_nonempty_value(payload.get(key)): return True metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} dependency = payload.get("dependency_context") if isinstance(payload.get("dependency_context"), dict) else {} return any( _has_nonempty_value(value) for value in ( metadata.get("selected_draft_image_ids"), metadata.get("stage_inputs"), dependency.get("selected_image_ids"), dependency.get("selected_draft_image_ids"), dependency.get("stage_inputs"), dependency.get("material_bindings"), ) ) def _has_nonempty_value(value: Any) -> bool: if value is None: return False if isinstance(value, str): return bool(value.strip()) if isinstance(value, (list, tuple, dict, set)): return bool(value) return True