"""Queue and job service.""" from __future__ import annotations from datetime import datetime, timedelta, timezone from typing import Any from uuid import uuid4 from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask, JobListResult from core.audit_service import AuditService from core.event_bus import EventBus from core.fsm import task_transition from core.identity import RequestContext from database.repositories.job_repository import JobRepository TASK_STATUSES = { "created", "queued", "running", "completed", "succeeded", "failed", "cancelled", "dead_letter", "retrying", "superseded", } RETRYABLE_TASK_STATUSES = {"failed", "dead_letter", "cancelled"} FAILED_TASK_STATUSES = ("failed", "dead_letter", "cancelled") COMPLETED_TASK_STATUSES = ("completed", "succeeded", "superseded") DELETABLE_TASK_STATUSES = frozenset((*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES)) class QueueService: def __init__(self, repository: JobRepository, audit_service: AuditService, event_bus: EventBus | None = None): self.repository = repository self.audit_service = audit_service self.event_bus = event_bus def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask: status = str(payload.get("status", "created")) if status not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) idempotency_key = _optional_text(payload.get("idempotency_key")) if idempotency_key: existing = self.repository.get_by_idempotency_key(idempotency_key) if existing is not None: return existing task = self.repository.create( { "task_id": payload.get("task_id") or new_task_id(), "profile_id": payload.get("profile_id"), "workflow_id": payload.get("workflow_id"), "node_id": payload.get("node_id"), "task_type": payload.get("task_type", "generation"), "status": status, "model_route": payload.get("model_route"), "progress": payload.get("progress", 0), "idempotency_key": idempotency_key, "request": payload.get("request") or {}, "result": payload.get("result") or {}, "error_message": payload.get("error_message"), "metadata": payload.get("metadata") or {}, } ) self.audit_service.record( ctx=ctx, action="create_task", target_type="generation_task", target_id=task.task_id, after_snapshot=task.to_dict(), ) self._publish( "generation.task_created", { "task_id": task.task_id, "status": task.status, "workflow_id": task.workflow_id, "node_id": task.node_id, }, ) return task def list_jobs( self, *, status: str | None = None, workflow_id: str | None = None, profile_id: str | None = None, page: int = 1, page_size: int = 30, ) -> JobListResult: if status and status not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) normalized_page = max(page, 1) normalized_page_size = min(max(page_size, 1), 100) items, total = self.repository.list( status=status, workflow_id=workflow_id, profile_id=profile_id, page=normalized_page, page_size=normalized_page_size, ) return JobListResult( items=items, total=total, page=normalized_page, page_size=normalized_page_size, ) def get_job(self, task_id: str) -> GenerationTask: task = self.repository.get(task_id) if task is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) return task def get_job_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None: normalized_key = _optional_text(idempotency_key) if normalized_key is None: return None return self.repository.get_by_idempotency_key(normalized_key) def cancel_job(self, task_id: str, ctx: RequestContext, *, reason: str | None = None) -> GenerationTask: while True: before = self.get_job(task_id) if before.status == "cancelled": return before if before.status == "retrying": raise ApiError( ErrorCode.TASK_NOT_CANCELABLE, "task retry execution is already in progress", {"task_id": task_id, "status": before.status}, ) if before.status in {"completed", "succeeded", "superseded"}: raise ApiError( ErrorCode.TASK_NOT_CANCELABLE, "task cannot be cancelled from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "cancelled") after = self.repository.update_if_status( task_id, {before.status}, { "status": "cancelled", "error_message": _optional_text(reason) or before.error_message, "metadata": { **before.metadata, "cancellation": { "requested_by": ctx.actor_id or "system", "requested_at": _now(), "reason": _optional_text(reason) or "cancelled", "previous_status": before.status, }, }, }, ) if after is not None: break # The task changed after it was read. Re-evaluate the current # state instead of applying cancellation from a stale snapshot. self.audit_service.record( ctx=ctx, action="cancel_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def update_task_if_status( self, task_id: str, expected_statuses: set[str] | tuple[str, ...] | list[str], payload: dict[str, Any], ctx: RequestContext, *, require_workflow_not_cancelled: bool = False, ) -> GenerationTask | None: """Update a task only if the status observed by this caller still owns it.""" normalized_statuses = { str(status).strip() for status in expected_statuses if str(status).strip() } before = self.get_job(task_id) if before.status not in normalized_statuses: return None status = payload.get("status") if status is not None: self._validate_task_status_transition(before.status, str(status)) if require_workflow_not_cancelled: after = self.repository.update_if_status_unless_workflow_cancelled( task_id, {before.status}, payload, ) else: after = self.repository.update_if_status(task_id, {before.status}, payload) if after is None: return None self.audit_service.record( ctx=ctx, action="update_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), metadata={"compare_and_set": True}, ) if before.status != after.status: self._publish_status_change(before, after) self._publish("generation.task_updated", {"task_id": task_id, "status": after.status}) return after def cancel_workflow_jobs( self, workflow_id: str, ctx: RequestContext, *, reason: str = "workflow cancelled", ) -> dict[str, Any]: """CAS-cancel every queued/running task currently owned by a workflow.""" cancelled_ids: list[str] = [] skipped: list[dict[str, str]] = [] for status in ("queued", "running"): while True: candidates, _ = self.repository.list( status=status, workflow_id=workflow_id, page=1, page_size=100, ) if not candidates: break changed = 0 for candidate in candidates: try: cancelled = self.cancel_job(candidate.task_id, ctx, reason=reason) if cancelled.status == "cancelled": cancelled_ids.append(cancelled.task_id) except ApiError as exc: current = self.get_job(candidate.task_id) skipped.append( { "task_id": candidate.task_id, "status": current.status, "error_code": exc.error_code.value, } ) current = self.get_job(candidate.task_id) if current.status != status: changed += 1 if changed == 0: break return { "workflow_id": workflow_id, "cancelled_task_ids": cancelled_ids, "skipped": skipped, } def is_cancelled(self, task_id: str) -> bool: return self.get_job(task_id).status == "cancelled" def claim_queued_task( self, task_id: str, ctx: RequestContext, *, worker_id: str, ) -> GenerationTask | None: """Atomically claim one queued task for a worker.""" before = self.get_job(task_id) if before.status != "queued": return None self._validate_task_status_transition(before.status, "running") attempt = _int_or_default(before.metadata.get("worker_attempt"), 0) + 1 after = self.repository.update_if_status( task_id, {"queued"}, { "status": "running", "progress": max(before.progress, 1), "metadata": { **before.metadata, "worker_attempt": attempt, "worker": { "worker_id": worker_id, "status": "running", "claimed_at": _now(), }, }, }, ) if after is None: return None self.audit_service.record( ctx=ctx, action="claim_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def requeue_job(self, task_id: str, ctx: RequestContext) -> GenerationTask: """Requeue a task for an explicitly managed QueueWorker. Web retry routes use GenerationJobRetryService instead because the AstrBot plugin does not start a persistent QueueWorker by default. """ before = self.get_job(task_id) if before.status not in RETRYABLE_TASK_STATUSES: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task cannot be retried from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "queued") after = self.repository.update( task_id, { "status": "queued", "progress": 0, "error_message": None, "metadata": {**before.metadata, "requeue_requested": True}, }, ) if after is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) self.audit_service.record( ctx=ctx, action="requeue_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def begin_retry(self, task_id: str, ctx: RequestContext, *, operation_id: str) -> GenerationTask: """Atomically reserve a failed task for synchronous retry execution.""" before = self.get_job(task_id) if before.status not in RETRYABLE_TASK_STATUSES: raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task cannot be retried from current status", {"task_id": task_id, "status": before.status}, ) self._validate_task_status_transition(before.status, "retrying") retry_metadata = { "operation_id": operation_id, "status": "running", "requested_by": ctx.actor_id or "system", "requested_at": _now(), "previous_status": before.status, } after = self.repository.update_if_status( task_id, {before.status}, { "status": "retrying", "progress": 0, "metadata": {**before.metadata, "retry_execution": retry_metadata}, }, ) if after is None: current = self.get_job(task_id) raise ApiError( ErrorCode.TASK_NOT_RETRYABLE, "task retry was already claimed or its status changed", {"task_id": task_id, "status": current.status}, ) self.audit_service.record( ctx=ctx, action="begin_retry_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) self._publish_status_change(before, after) return after def update_task(self, task_id: str, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask: before = self.get_job(task_id) status = payload.get("status") if status is not None: self._validate_task_status_transition(before.status, str(status)) after = self.repository.update(task_id, payload) if after is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) self.audit_service.record( ctx=ctx, action="update_task", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), after_snapshot=after.to_dict(), ) if before.status != after.status: self._publish_status_change(before, after) self._publish("generation.task_updated", {"task_id": task_id, "status": after.status}) return after def queue_summary(self) -> dict[str, int]: counts = self.repository.count_by_status() return { "pending": counts.get("created", 0) + counts.get("queued", 0), "running": counts.get("running", 0) + counts.get("retrying", 0), "failed": counts.get("failed", 0), "dead_letter": counts.get("dead_letter", 0), } def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]: return [task.to_dict() for task in self.repository.recent(limit=limit)] def recover_stale_tasks( self, ctx: RequestContext, *, stale_after_seconds: int = 300, limit: int = 100, source: str | None = "workflow_round", now: datetime | None = None, ) -> dict[str, Any]: """Recover tasks abandoned by a previous plugin process. The task FSM intentionally does not allow ``running -> queued``. Recovery therefore records a failed terminal observation first and only then requeues it. The two compare-and-set operations also avoid stealing work from a live worker that updated the row concurrently. """ observed_at = now or datetime.now(timezone.utc) if observed_at.tzinfo is None: observed_at = observed_at.replace(tzinfo=timezone.utc) cutoff = observed_at.astimezone(timezone.utc) - timedelta( seconds=max(int(stale_after_seconds), 1) ) cutoff_value = cutoff.strftime("%Y-%m-%d %H:%M:%S") candidates = self.repository.list_stale( statuses=("running", "retrying"), updated_before=cutoff_value, limit=min(max(int(limit), 1), 500), ) recovered: list[str] = [] skipped: list[dict[str, str]] = [] for candidate in candidates: task_source = str(candidate.metadata.get("source") or "").strip() if source is not None and task_source != source: skipped.append( { "task_id": candidate.task_id, "status": candidate.status, "reason": "source_not_managed", } ) continue recovery = { "source": "plugin_startup_stale_recovery", "observed_at": observed_at.astimezone(timezone.utc).isoformat(), "updated_before": cutoff_value, "previous_status": candidate.status, } failed = self.update_task_if_status( candidate.task_id, {candidate.status}, { "status": "failed", "progress": candidate.progress, "error_message": "task was interrupted by a previous plugin process", "metadata": {**candidate.metadata, "recovery": recovery}, }, ctx, ) if failed is None: skipped.append( { "task_id": candidate.task_id, "status": self.get_job(candidate.task_id).status, "reason": "status_changed", } ) continue queued = self.update_task_if_status( failed.task_id, {"failed"}, { "status": "queued", "progress": 0, "error_message": None, "metadata": { **failed.metadata, "recovery": {**recovery, "status": "requeued"}, }, }, ctx, require_workflow_not_cancelled=True, ) if queued is None: skipped.append( { "task_id": candidate.task_id, "status": self.get_job(candidate.task_id).status, "reason": "workflow_cancelled_or_status_changed", } ) continue recovered.append(queued.task_id) return { "recovered_task_ids": recovered, "recovered": len(recovered), "skipped": skipped, "cutoff": cutoff_value, } def requeue_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: jobs = self.list_jobs(status="dead_letter", page=1, page_size=_batch_limit(limit)).items requeued = [] errors = [] for job in jobs: try: requeued.append(self.requeue_job(job.task_id, ctx).to_dict()) except ApiError as exc: errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message}) return {"requeued": requeued, "errors": errors, "total": len(requeued)} def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: jobs = self.list_jobs(status="failed", page=1, page_size=_batch_limit(limit)).items moved = [] for job in jobs: moved.append( self.update_task( job.task_id, { "status": "dead_letter", "metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"}, }, ctx, ).to_dict() ) return {"moved": moved, "total": len(moved)} def delete_job(self, task_id: str, ctx: RequestContext) -> dict[str, Any]: before = self.get_job(task_id) if before.status not in DELETABLE_TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "only terminal task records can be deleted", { "task_id": task_id, "status": before.status, "allowed_statuses": sorted(DELETABLE_TASK_STATUSES), }, ) # Compare against the status that was authorized above, rather than # any deletable status. This keeps the audit snapshot truthful and # prevents a concurrent state change from being silently deleted. deleted = self.repository.delete_if_status(task_id, {before.status}) if deleted is None: current = self.repository.get(task_id) if current is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id}) raise ApiError( ErrorCode.VALIDATION_FAILED, "task status changed before deletion", {"task_id": task_id, "status": current.status}, ) node_cleanup = dict(deleted.get("node_cleanup") or {}) workflow_cleanup = dict(deleted.get("workflow_cleanup") or {}) self.audit_service.record( ctx=ctx, action="delete_task_record", target_type="generation_task", target_id=task_id, before_snapshot=before.to_dict(), metadata={"node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup}, ) self._publish( "generation.task_deleted", { "task_id": task_id, "status": before.status, "node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup, }, ) return { "task_id": task_id, "deleted": True, "previous_status": before.status, "node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup, } def clear_failed_jobs(self, ctx: RequestContext, *, limit: int = 100) -> dict[str, Any]: return self._clear_job_records(ctx, statuses=FAILED_TASK_STATUSES, limit=limit, scope="failed") def clear_terminal_jobs( self, ctx: RequestContext, *, statuses: list[str] | tuple[str, ...] | set[str] | None = None, limit: int = 100, ) -> dict[str, Any]: return self._clear_job_records( ctx, statuses=_terminal_cleanup_statuses(statuses), limit=limit, scope="terminal", ) def _clear_job_records( self, ctx: RequestContext, *, statuses: tuple[str, ...], limit: int, scope: str, ) -> dict[str, Any]: applied_limit = _batch_limit(limit) deleted: list[dict[str, Any]] = [] errors: list[dict[str, str]] = [] for status in statuses: # Failed compare-and-delete attempts are still processed records. # Counting only successful deletions could exceed the caller's # batch limit when statuses change concurrently. remaining = applied_limit - len(deleted) - len(errors) if remaining <= 0: break jobs = self.list_jobs(status=status, page=1, page_size=remaining).items for job in jobs: try: deleted.append(self.delete_job(job.task_id, ctx)) except ApiError as exc: errors.append( {"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message} ) remaining_counts = self.repository.count_by_status() return { "scope": scope, "requested_statuses": list(statuses), "deleted": deleted, "errors": errors, "total": len(deleted), "processed": len(deleted) + len(errors), "limit": applied_limit, "remaining_by_status": {status: remaining_counts.get(status, 0) for status in statuses}, } def _validate_task_status_transition(self, current: str, target: str) -> None: if target not in TASK_STATUSES: raise ApiError( ErrorCode.VALIDATION_FAILED, "unsupported task status", {"field": "status", "allowed": sorted(TASK_STATUSES)}, ) transition = task_transition(current, target) if not transition.allowed: raise ApiError( ErrorCode.VALIDATION_FAILED, "task status transition is not allowed", {"current": current, "target": target, "reason": transition.reason}, ) def _publish_status_change(self, before: GenerationTask, after: GenerationTask) -> None: self._publish( "generation.task_status_changed", { "task_id": after.task_id, "workflow_id": after.workflow_id, "node_id": after.node_id, "before_status": before.status, "after_status": after.status, }, ) def _publish(self, event_type: str, payload: dict[str, Any]) -> None: if self.event_bus: self.event_bus.publish(event_type, payload) def new_task_id() -> str: return f"task_{uuid4().hex}" def _now() -> str: return datetime.now(timezone.utc).isoformat() def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _batch_limit(value: int) -> int: try: limit = int(value) except (TypeError, ValueError): limit = 50 return min(max(limit, 1), 100) def _terminal_cleanup_statuses( statuses: list[str] | tuple[str, ...] | set[str] | None, ) -> tuple[str, ...]: if statuses is None: return (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES) if not isinstance(statuses, (list, tuple, set)): raise ApiError( ErrorCode.VALIDATION_FAILED, "terminal cleanup statuses must be an array", {"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)}, ) requested = {str(status or "").strip().lower() for status in statuses if str(status or "").strip()} if not requested: raise ApiError( ErrorCode.VALIDATION_FAILED, "terminal cleanup requires at least one status", {"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)}, ) unsupported = sorted(requested - DELETABLE_TASK_STATUSES) if unsupported: raise ApiError( ErrorCode.VALIDATION_FAILED, "terminal cleanup cannot include active task statuses", { "field": "statuses", "unsupported_statuses": unsupported, "allowed_statuses": sorted(DELETABLE_TASK_STATUSES), }, ) canonical_order = (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES) return tuple(status for status in canonical_order if status in requested) def _int_or_default(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default