"""Queue and job service.""" from __future__ import annotations from datetime import datetime, timezone from typing import Any from uuid import uuid4 from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask, JobListResult from core.audit_service import AuditService from core.event_bus import EventBus from core.fsm import task_transition from core.identity import RequestContext from database.repositories.job_repository import JobRepository TASK_STATUSES = { "created", "queued", "running", "completed", "succeeded", "failed", "cancelled", "dead_letter", "retrying", "superseded", } RETRYABLE_TASK_STATUSES = {"failed", "dead_letter", "cancelled"} class QueueService: def __init__(self, repository: JobRepository, audit_service: AuditService, event_bus: EventBus | None = None): self.repository = repository self.audit_service = audit_service self.event_bus = event_bus def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask: status = str(payload.get("status", "created")) if status not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) idempotency_key = _optional_text(payload.get("idempotency_key")) if idempotency_key: existing = self.repository.get_by_idempotency_key(idempotency_key) if existing is not None: return existing task = self.repository.create( { "task_id": payload.get("task_id") or new_task_id(), "profile_id": payload.get("profile_id"), "workflow_id": payload.get("workflow_id"), "node_id": payload.get("node_id"), "task_type": payload.get("task_type", "generation"), "status": status, "model_route": payload.get("model_route"), "progress": payload.get("progress", 0), "idempotency_key": idempotency_key, "request": payload.get("request") or {}, "result": payload.get("result") or {}, "error_message": payload.get("error_message"), "metadata": payload.get("metadata") or {}, } ) self.audit_service.record( ctx=ctx, action="create_task", target_type="generation_task", target_id=task.task_id, after_snapshot=task.to_dict(), ) self._publish( "generation.task_created", { "task_id": task.task_id, "status": task.status, "workflow_id": task.workflow_id, "node_id": task.node_id, }, ) return task def list_jobs( self, *, status: str | None = None, workflow_id: str | None = None, profile_id: str | None = None, page: int = 1, page_size: int = 30, ) -> JobListResult: if status and status not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) normalized_page = max(page, 1) normalized_page_size = min(max(page_size, 1), 100) items, total = self.repository.list( status=status, workflow_id=workflow_id, profile_id=profile_id, page=normalized_page, page_size=normalized_page_size, ) return JobListResult( items=items, total=total, page=normalized_page, page_size=normalized_page_size, ) def get_job(self, task_id: str) -> GenerationTask: task = self.repository.get(task_id) if task is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) return task def get_job_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None: normalized_key = _optional_text(idempotency_key) if normalized_key is None: return None return self.repository.get_by_idempotency_key(normalized_key) def cancel_job(self, task_id: str, ctx: RequestContext, *, reason: str | None = None) -> GenerationTask: while True: before = self.get_job(task_id) if before.status == "cancelled": return before if before.status == "retrying": raise ApiError( ErrorCode.TASK_NOT_CANCELABLE, "task retry execution is already in progress", {"task_id": task_id, "status": before.status}, ) if before.status in {"completed", "succeeded", "superseded"}: raise ApiError( ErrorCode.TASK_NOT_CANCELABLE, "task cannot be cancelled from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "cancelled") after = self.repository.update_if_status( task_id, {before.status}, { "status": "cancelled", "error_message": _optional_text(reason) or before.error_message, "metadata": { **before.metadata, "cancellation": { "requested_by": ctx.actor_id or "system", "requested_at": _now(), "reason": _optional_text(reason) or "cancelled", "previous_status": before.status, }, }, }, ) if after is not None: break # The task changed after it was read. Re-evaluate the current # state instead of applying cancellation from a stale snapshot. self.audit_service.record( ctx=ctx, action="cancel_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def update_task_if_status( self, task_id: str, expected_statuses: set[str] | tuple[str, ...] | list[str], payload: dict[str, Any], ctx: RequestContext, *, require_workflow_not_cancelled: bool = False, ) -> GenerationTask | None: """Update a task only if the status observed by this caller still owns it.""" normalized_statuses = { str(status).strip() for status in expected_statuses if str(status).strip() } before = self.get_job(task_id) if before.status not in normalized_statuses: return None status = payload.get("status") if status is not None: self._validate_task_status_transition(before.status, str(status)) if require_workflow_not_cancelled: after = self.repository.update_if_status_unless_workflow_cancelled( task_id, {before.status}, payload, ) else: after = self.repository.update_if_status(task_id, {before.status}, payload) if after is None: return None self.audit_service.record( ctx=ctx, action="update_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), metadata={"compare_and_set": True}, ) if before.status != after.status: self._publish_status_change(before, after) self._publish("generation.task_updated", {"task_id": task_id, "status": after.status}) return after def cancel_workflow_jobs( self, workflow_id: str, ctx: RequestContext, *, reason: str = "workflow cancelled", ) -> dict[str, Any]: """CAS-cancel every queued/running task currently owned by a workflow.""" cancelled_ids: list[str] = [] skipped: list[dict[str, str]] = [] for status in ("queued", "running"): while True: candidates, _ = self.repository.list( status=status, workflow_id=workflow_id, page=1, page_size=100, ) if not candidates: break changed = 0 for candidate in candidates: try: cancelled = self.cancel_job(candidate.task_id, ctx, reason=reason) if cancelled.status == "cancelled": cancelled_ids.append(cancelled.task_id) except ApiError as exc: current = self.get_job(candidate.task_id) skipped.append( { "task_id": candidate.task_id, "status": current.status, "error_code": exc.error_code.value, } ) current = self.get_job(candidate.task_id) if current.status != status: changed += 1 if changed == 0: break return { "workflow_id": workflow_id, "cancelled_task_ids": cancelled_ids, "skipped": skipped, } def is_cancelled(self, task_id: str) -> bool: return self.get_job(task_id).status == "cancelled" def claim_queued_task( self, task_id: str, ctx: RequestContext, *, worker_id: str, ) -> GenerationTask | None: """Atomically claim one queued task for a worker.""" before = self.get_job(task_id) if before.status != "queued": return None self._validate_task_status_transition(before.status, "running") attempt = _int_or_default(before.metadata.get("worker_attempt"), 0) + 1 after = self.repository.update_if_status( task_id, {"queued"}, { "status": "running", "progress": max(before.progress, 1), "metadata": { **before.metadata, "worker_attempt": attempt, "worker": { "worker_id": worker_id, "status": "running", "claimed_at": _now(), }, }, }, ) if after is None: return None self.audit_service.record( ctx=ctx, action="claim_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def requeue_job(self, task_id: str, ctx: RequestContext) -> GenerationTask: """Requeue a task for an explicitly managed QueueWorker. Web retry routes use GenerationJobRetryService instead because the AstrBot plugin does not start a persistent QueueWorker by default. """ before = self.get_job(task_id) if before.status not in RETRYABLE_TASK_STATUSES: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task cannot be retried from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "queued") after = self.repository.update( task_id, { "status": "queued", "progress": 0, "error_message": None, "metadata": {**before.metadata, "requeue_requested": True}, }, ) if after is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) self.audit_service.record( ctx=ctx, action="requeue_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def begin_retry(self, task_id: str, ctx: RequestContext, *, operation_id: str) -> GenerationTask: """Atomically reserve a failed task for synchronous retry execution.""" before = self.get_job(task_id) if before.status not in RETRYABLE_TASK_STATUSES: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task cannot be retried from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "retrying") retry_metadata = { "operation_id": operation_id, "status": "running", "requested_by": ctx.actor_id or "system", "requested_at": _now(), "previous_status": before.status, } after = self.repository.update_if_status( task_id, {before.status}, { "status": "retrying", "progress": 0, "metadata": {**before.metadata, "retry_execution": retry_metadata}, }, ) if after is None: current = self.get_job(task_id) raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task retry was already claimed or its status changed", {"task_id": task_id, "status": current.status}, ) self.audit_service.record( ctx=ctx, action="begin_retry_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def update_task(self, task_id: str, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask: before = self.get_job(task_id) status = payload.get("status") if status is not None: self._validate_task_status_transition(before.status, str(status)) after = self.repository.update(task_id, payload) if after is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) self.audit_service.record( ctx=ctx, action="update_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) if before.status != after.status: self._publish_status_change(before, after) self._publish("generation.task_updated", {"task_id": task_id, "status": after.status}) return after def queue_summary(self) -> dict[str, int]: counts = self.repository.count_by_status() return { "pending": counts.get("created", 0) + counts.get("queued", 0), "running": counts.get("running", 0) + counts.get("retrying", 0), "failed": counts.get("failed", 0), "dead_letter": counts.get("dead_letter", 0), } def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]: return [task.to_dict() for task in self.repository.recent(limit=limit)] def requeue_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: jobs = self.list_jobs(status="dead_letter", page=1, page_size=_batch_limit(limit)).items requeued = [] errors = [] for job in jobs: try: requeued.append(self.requeue_job(job.task_id, ctx).to_dict()) except ApiError as exc: errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message}) return {"requeued": requeued, "errors": errors, "total": len(requeued)} def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: jobs = self.list_jobs(status="failed", page=1, page_size=_batch_limit(limit)).items moved = [] for job in jobs: moved.append( self.update_task( job.task_id, { "status": "dead_letter", "metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"}, }, ctx, ).to_dict() ) return {"moved": moved, "total": len(moved)} def _validate_task_status_transition(self, current: str, target: str) -> None: if target not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) transition = task_transition(current, target) if not transition.allowed: raise ApiError( ErrorCode.VALIDATION_FAILED, "task status transition is not allowed", {"current": current, "target": target, "reason": transition.reason}, ) def _publish_status_change(self, before: GenerationTask, after: GenerationTask) -> None: self._publish( "generation.task_status_changed", { "task_id": after.task_id, "workflow_id": after.workflow_id, "node_id": after.node_id, "before_status": before.status, "after_status": after.status, }, ) def _publish(self, event_type: str, payload: dict[str, Any]) -> None: if self.event_bus: self.event_bus.publish(event_type, payload) def new_task_id() -> str: return f"task_{uuid4().hex}" def _now() -> str: return datetime.now(timezone.utc).isoformat() def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _batch_limit(value: int) -> int: try: limit = int(value) except (TypeError, ValueError): limit = 50 return min(max(limit, 1), 100) def _int_or_default(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default