2026-07-05 09:07:34 +08:00
|
|
|
"""Queue and job service."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
from datetime import datetime, timedelta, timezone
|
2026-07-05 09:07:34 +08:00
|
|
|
from typing import Any
|
|
|
|
|
from uuid import uuid4
|
|
|
|
|
|
|
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
|
|
|
from backend.dto.job_dto import GenerationTask, JobListResult
|
|
|
|
|
from core.audit_service import AuditService
|
2026-07-05 15:54:48 +08:00
|
|
|
from core.event_bus import EventBus
|
|
|
|
|
from core.fsm import task_transition
|
2026-07-05 09:07:34 +08:00
|
|
|
from core.identity import RequestContext
|
|
|
|
|
from database.repositories.job_repository import JobRepository
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TASK_STATUSES = {
|
|
|
|
|
"created",
|
|
|
|
|
"queued",
|
|
|
|
|
"running",
|
|
|
|
|
"completed",
|
|
|
|
|
"succeeded",
|
|
|
|
|
"failed",
|
|
|
|
|
"cancelled",
|
|
|
|
|
"dead_letter",
|
2026-07-11 21:50:34 +08:00
|
|
|
"retrying",
|
|
|
|
|
"superseded",
|
2026-07-05 09:07:34 +08:00
|
|
|
}
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
RETRYABLE_TASK_STATUSES = {"failed", "dead_letter", "cancelled"}
|
2026-07-15 23:01:07 +08:00
|
|
|
FAILED_TASK_STATUSES = ("failed", "dead_letter", "cancelled")
|
|
|
|
|
COMPLETED_TASK_STATUSES = ("completed", "succeeded", "superseded")
|
|
|
|
|
DELETABLE_TASK_STATUSES = frozenset((*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES))
|
2026-07-11 21:50:34 +08:00
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
|
|
|
|
|
class QueueService:
|
2026-07-05 15:54:48 +08:00
|
|
|
def __init__(self, repository: JobRepository, audit_service: AuditService, event_bus: EventBus | None = None):
|
2026-07-05 09:07:34 +08:00
|
|
|
self.repository = repository
|
|
|
|
|
self.audit_service = audit_service
|
2026-07-05 15:54:48 +08:00
|
|
|
self.event_bus = event_bus
|
2026-07-05 09:07:34 +08:00
|
|
|
|
|
|
|
|
def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
status = str(payload.get("status", "created"))
|
|
|
|
|
if status not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
2026-07-05 18:20:14 +08:00
|
|
|
idempotency_key = _optional_text(payload.get("idempotency_key"))
|
|
|
|
|
if idempotency_key:
|
|
|
|
|
existing = self.repository.get_by_idempotency_key(idempotency_key)
|
|
|
|
|
if existing is not None:
|
|
|
|
|
return existing
|
2026-07-05 09:07:34 +08:00
|
|
|
task = self.repository.create(
|
|
|
|
|
{
|
|
|
|
|
"task_id": payload.get("task_id") or new_task_id(),
|
|
|
|
|
"profile_id": payload.get("profile_id"),
|
|
|
|
|
"workflow_id": payload.get("workflow_id"),
|
|
|
|
|
"node_id": payload.get("node_id"),
|
|
|
|
|
"task_type": payload.get("task_type", "generation"),
|
|
|
|
|
"status": status,
|
|
|
|
|
"model_route": payload.get("model_route"),
|
|
|
|
|
"progress": payload.get("progress", 0),
|
2026-07-05 18:20:14 +08:00
|
|
|
"idempotency_key": idempotency_key,
|
2026-07-05 09:07:34 +08:00
|
|
|
"request": payload.get("request") or {},
|
|
|
|
|
"result": payload.get("result") or {},
|
|
|
|
|
"error_message": payload.get("error_message"),
|
|
|
|
|
"metadata": payload.get("metadata") or {},
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="create_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task.task_id,
|
|
|
|
|
after_snapshot=task.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish(
|
|
|
|
|
"generation.task_created",
|
|
|
|
|
{
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
"status": task.status,
|
|
|
|
|
"workflow_id": task.workflow_id,
|
|
|
|
|
"node_id": task.node_id,
|
|
|
|
|
},
|
|
|
|
|
)
|
2026-07-05 09:07:34 +08:00
|
|
|
return task
|
|
|
|
|
|
|
|
|
|
def list_jobs(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
status: str | None = None,
|
|
|
|
|
workflow_id: str | None = None,
|
|
|
|
|
profile_id: str | None = None,
|
|
|
|
|
page: int = 1,
|
|
|
|
|
page_size: int = 30,
|
|
|
|
|
) -> JobListResult:
|
|
|
|
|
if status and status not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
normalized_page = max(page, 1)
|
|
|
|
|
normalized_page_size = min(max(page_size, 1), 100)
|
|
|
|
|
items, total = self.repository.list(
|
|
|
|
|
status=status,
|
|
|
|
|
workflow_id=workflow_id,
|
|
|
|
|
profile_id=profile_id,
|
|
|
|
|
page=normalized_page,
|
|
|
|
|
page_size=normalized_page_size,
|
|
|
|
|
)
|
|
|
|
|
return JobListResult(
|
|
|
|
|
items=items,
|
|
|
|
|
total=total,
|
|
|
|
|
page=normalized_page,
|
|
|
|
|
page_size=normalized_page_size,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def get_job(self, task_id: str) -> GenerationTask:
|
|
|
|
|
task = self.repository.get(task_id)
|
|
|
|
|
if task is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
return task
|
|
|
|
|
|
2026-07-05 18:23:35 +08:00
|
|
|
def get_job_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None:
|
|
|
|
|
normalized_key = _optional_text(idempotency_key)
|
|
|
|
|
if normalized_key is None:
|
|
|
|
|
return None
|
|
|
|
|
return self.repository.get_by_idempotency_key(normalized_key)
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def cancel_job(self, task_id: str, ctx: RequestContext, *, reason: str | None = None) -> GenerationTask:
|
2026-07-11 21:50:34 +08:00
|
|
|
while True:
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
if before.status == "cancelled":
|
|
|
|
|
return before
|
|
|
|
|
if before.status == "retrying":
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_CANCELABLE,
|
|
|
|
|
"task retry execution is already in progress",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
|
|
|
|
if before.status in {"completed", "succeeded", "superseded"}:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_CANCELABLE,
|
|
|
|
|
"task cannot be cancelled from current status",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
|
|
|
|
self._validate_task_status_transition(before.status, "cancelled")
|
|
|
|
|
after = self.repository.update_if_status(
|
|
|
|
|
task_id,
|
|
|
|
|
{before.status},
|
|
|
|
|
{
|
|
|
|
|
"status": "cancelled",
|
|
|
|
|
"error_message": _optional_text(reason) or before.error_message,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**before.metadata,
|
|
|
|
|
"cancellation": {
|
|
|
|
|
"requested_by": ctx.actor_id or "system",
|
|
|
|
|
"requested_at": _now(),
|
|
|
|
|
"reason": _optional_text(reason) or "cancelled",
|
|
|
|
|
"previous_status": before.status,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if after is not None:
|
|
|
|
|
break
|
|
|
|
|
# The task changed after it was read. Re-evaluate the current
|
|
|
|
|
# state instead of applying cancellation from a stale snapshot.
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="cancel_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
|
|
|
|
self._publish_status_change(before, after)
|
|
|
|
|
return after
|
|
|
|
|
|
|
|
|
|
def update_task_if_status(
|
|
|
|
|
self,
|
|
|
|
|
task_id: str,
|
|
|
|
|
expected_statuses: set[str] | tuple[str, ...] | list[str],
|
|
|
|
|
payload: dict[str, Any],
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
require_workflow_not_cancelled: bool = False,
|
|
|
|
|
) -> GenerationTask | None:
|
|
|
|
|
"""Update a task only if the status observed by this caller still owns it."""
|
|
|
|
|
|
|
|
|
|
normalized_statuses = {
|
|
|
|
|
str(status).strip()
|
|
|
|
|
for status in expected_statuses
|
|
|
|
|
if str(status).strip()
|
|
|
|
|
}
|
2026-07-05 09:07:34 +08:00
|
|
|
before = self.get_job(task_id)
|
2026-07-11 21:50:34 +08:00
|
|
|
if before.status not in normalized_statuses:
|
|
|
|
|
return None
|
|
|
|
|
status = payload.get("status")
|
|
|
|
|
if status is not None:
|
|
|
|
|
self._validate_task_status_transition(before.status, str(status))
|
|
|
|
|
if require_workflow_not_cancelled:
|
|
|
|
|
after = self.repository.update_if_status_unless_workflow_cancelled(
|
|
|
|
|
task_id,
|
|
|
|
|
{before.status},
|
|
|
|
|
payload,
|
2026-07-05 09:07:34 +08:00
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
else:
|
|
|
|
|
after = self.repository.update_if_status(task_id, {before.status}, payload)
|
|
|
|
|
if after is None:
|
|
|
|
|
return None
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="update_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
metadata={"compare_and_set": True},
|
|
|
|
|
)
|
|
|
|
|
if before.status != after.status:
|
|
|
|
|
self._publish_status_change(before, after)
|
|
|
|
|
self._publish("generation.task_updated", {"task_id": task_id, "status": after.status})
|
|
|
|
|
return after
|
|
|
|
|
|
|
|
|
|
def cancel_workflow_jobs(
|
|
|
|
|
self,
|
|
|
|
|
workflow_id: str,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
reason: str = "workflow cancelled",
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
"""CAS-cancel every queued/running task currently owned by a workflow."""
|
|
|
|
|
|
|
|
|
|
cancelled_ids: list[str] = []
|
|
|
|
|
skipped: list[dict[str, str]] = []
|
|
|
|
|
for status in ("queued", "running"):
|
|
|
|
|
while True:
|
|
|
|
|
candidates, _ = self.repository.list(
|
|
|
|
|
status=status,
|
|
|
|
|
workflow_id=workflow_id,
|
|
|
|
|
page=1,
|
|
|
|
|
page_size=100,
|
|
|
|
|
)
|
|
|
|
|
if not candidates:
|
|
|
|
|
break
|
|
|
|
|
changed = 0
|
|
|
|
|
for candidate in candidates:
|
|
|
|
|
try:
|
|
|
|
|
cancelled = self.cancel_job(candidate.task_id, ctx, reason=reason)
|
|
|
|
|
if cancelled.status == "cancelled":
|
|
|
|
|
cancelled_ids.append(cancelled.task_id)
|
|
|
|
|
except ApiError as exc:
|
|
|
|
|
current = self.get_job(candidate.task_id)
|
|
|
|
|
skipped.append(
|
|
|
|
|
{
|
|
|
|
|
"task_id": candidate.task_id,
|
|
|
|
|
"status": current.status,
|
|
|
|
|
"error_code": exc.error_code.value,
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
current = self.get_job(candidate.task_id)
|
|
|
|
|
if current.status != status:
|
|
|
|
|
changed += 1
|
|
|
|
|
if changed == 0:
|
|
|
|
|
break
|
|
|
|
|
return {
|
|
|
|
|
"workflow_id": workflow_id,
|
|
|
|
|
"cancelled_task_ids": cancelled_ids,
|
|
|
|
|
"skipped": skipped,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def is_cancelled(self, task_id: str) -> bool:
|
|
|
|
|
return self.get_job(task_id).status == "cancelled"
|
|
|
|
|
|
|
|
|
|
def claim_queued_task(
|
|
|
|
|
self,
|
|
|
|
|
task_id: str,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
worker_id: str,
|
|
|
|
|
) -> GenerationTask | None:
|
|
|
|
|
"""Atomically claim one queued task for a worker."""
|
|
|
|
|
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
if before.status != "queued":
|
|
|
|
|
return None
|
|
|
|
|
self._validate_task_status_transition(before.status, "running")
|
|
|
|
|
attempt = _int_or_default(before.metadata.get("worker_attempt"), 0) + 1
|
|
|
|
|
after = self.repository.update_if_status(
|
2026-07-06 02:23:05 +08:00
|
|
|
task_id,
|
2026-07-11 21:50:34 +08:00
|
|
|
{"queued"},
|
2026-07-06 02:23:05 +08:00
|
|
|
{
|
2026-07-11 21:50:34 +08:00
|
|
|
"status": "running",
|
|
|
|
|
"progress": max(before.progress, 1),
|
2026-07-06 02:23:05 +08:00
|
|
|
"metadata": {
|
|
|
|
|
**before.metadata,
|
2026-07-11 21:50:34 +08:00
|
|
|
"worker_attempt": attempt,
|
|
|
|
|
"worker": {
|
|
|
|
|
"worker_id": worker_id,
|
|
|
|
|
"status": "running",
|
|
|
|
|
"claimed_at": _now(),
|
2026-07-06 02:23:05 +08:00
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
)
|
2026-07-05 09:07:34 +08:00
|
|
|
if after is None:
|
2026-07-11 21:50:34 +08:00
|
|
|
return None
|
2026-07-05 09:07:34 +08:00
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
2026-07-11 21:50:34 +08:00
|
|
|
action="claim_task",
|
2026-07-05 09:07:34 +08:00
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish_status_change(before, after)
|
2026-07-05 09:07:34 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def requeue_job(self, task_id: str, ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
"""Requeue a task for an explicitly managed QueueWorker.
|
|
|
|
|
|
|
|
|
|
Web retry routes use GenerationJobRetryService instead because the
|
|
|
|
|
AstrBot plugin does not start a persistent QueueWorker by default.
|
|
|
|
|
"""
|
2026-07-06 02:23:05 +08:00
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
before = self.get_job(task_id)
|
2026-07-11 21:50:34 +08:00
|
|
|
if before.status not in RETRYABLE_TASK_STATUSES:
|
2026-07-05 09:07:34 +08:00
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
|
|
|
"task cannot be retried from current status",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._validate_task_status_transition(before.status, "queued")
|
2026-07-05 09:07:34 +08:00
|
|
|
after = self.repository.update(
|
|
|
|
|
task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"error_message": None,
|
2026-07-11 21:50:34 +08:00
|
|
|
"metadata": {**before.metadata, "requeue_requested": True},
|
2026-07-05 09:07:34 +08:00
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
2026-07-11 21:50:34 +08:00
|
|
|
action="requeue_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
|
|
|
|
self._publish_status_change(before, after)
|
|
|
|
|
return after
|
|
|
|
|
|
|
|
|
|
def begin_retry(self, task_id: str, ctx: RequestContext, *, operation_id: str) -> GenerationTask:
|
|
|
|
|
"""Atomically reserve a failed task for synchronous retry execution."""
|
|
|
|
|
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
if before.status not in RETRYABLE_TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
|
|
|
"task cannot be retried from current status",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
|
|
|
|
self._validate_task_status_transition(before.status, "retrying")
|
|
|
|
|
retry_metadata = {
|
|
|
|
|
"operation_id": operation_id,
|
|
|
|
|
"status": "running",
|
|
|
|
|
"requested_by": ctx.actor_id or "system",
|
|
|
|
|
"requested_at": _now(),
|
|
|
|
|
"previous_status": before.status,
|
|
|
|
|
}
|
|
|
|
|
after = self.repository.update_if_status(
|
|
|
|
|
task_id,
|
|
|
|
|
{before.status},
|
|
|
|
|
{
|
|
|
|
|
"status": "retrying",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"metadata": {**before.metadata, "retry_execution": retry_metadata},
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if after is None:
|
|
|
|
|
current = self.get_job(task_id)
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
|
|
|
"task retry was already claimed or its status changed",
|
|
|
|
|
{"task_id": task_id, "status": current.status},
|
|
|
|
|
)
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="begin_retry_task",
|
2026-07-05 09:07:34 +08:00
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish_status_change(before, after)
|
2026-07-05 09:07:34 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-05 09:28:39 +08:00
|
|
|
def update_task(self, task_id: str, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
status = payload.get("status")
|
2026-07-05 15:54:48 +08:00
|
|
|
if status is not None:
|
|
|
|
|
self._validate_task_status_transition(before.status, str(status))
|
2026-07-05 09:28:39 +08:00
|
|
|
after = self.repository.update(task_id, payload)
|
|
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="update_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
if before.status != after.status:
|
|
|
|
|
self._publish_status_change(before, after)
|
|
|
|
|
self._publish("generation.task_updated", {"task_id": task_id, "status": after.status})
|
2026-07-05 09:28:39 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
def queue_summary(self) -> dict[str, int]:
|
|
|
|
|
counts = self.repository.count_by_status()
|
|
|
|
|
return {
|
|
|
|
|
"pending": counts.get("created", 0) + counts.get("queued", 0),
|
2026-07-11 21:50:34 +08:00
|
|
|
"running": counts.get("running", 0) + counts.get("retrying", 0),
|
2026-07-05 09:07:34 +08:00
|
|
|
"failed": counts.get("failed", 0),
|
|
|
|
|
"dead_letter": counts.get("dead_letter", 0),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
|
|
|
|
|
return [task.to_dict() for task in self.repository.recent(limit=limit)]
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
def recover_stale_tasks(
|
|
|
|
|
self,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
stale_after_seconds: int = 300,
|
|
|
|
|
limit: int = 100,
|
|
|
|
|
source: str | None = "workflow_round",
|
|
|
|
|
now: datetime | None = None,
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
"""Recover tasks abandoned by a previous plugin process.
|
|
|
|
|
|
|
|
|
|
The task FSM intentionally does not allow ``running -> queued``.
|
|
|
|
|
Recovery therefore records a failed terminal observation first and
|
|
|
|
|
only then requeues it. The two compare-and-set operations also avoid
|
|
|
|
|
stealing work from a live worker that updated the row concurrently.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
observed_at = now or datetime.now(timezone.utc)
|
|
|
|
|
if observed_at.tzinfo is None:
|
|
|
|
|
observed_at = observed_at.replace(tzinfo=timezone.utc)
|
|
|
|
|
cutoff = observed_at.astimezone(timezone.utc) - timedelta(
|
|
|
|
|
seconds=max(int(stale_after_seconds), 1)
|
|
|
|
|
)
|
|
|
|
|
cutoff_value = cutoff.strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
candidates = self.repository.list_stale(
|
|
|
|
|
statuses=("running", "retrying"),
|
|
|
|
|
updated_before=cutoff_value,
|
|
|
|
|
limit=min(max(int(limit), 1), 500),
|
|
|
|
|
)
|
|
|
|
|
recovered: list[str] = []
|
|
|
|
|
skipped: list[dict[str, str]] = []
|
|
|
|
|
for candidate in candidates:
|
|
|
|
|
task_source = str(candidate.metadata.get("source") or "").strip()
|
|
|
|
|
if source is not None and task_source != source:
|
|
|
|
|
skipped.append(
|
|
|
|
|
{
|
|
|
|
|
"task_id": candidate.task_id,
|
|
|
|
|
"status": candidate.status,
|
|
|
|
|
"reason": "source_not_managed",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
continue
|
|
|
|
|
recovery = {
|
|
|
|
|
"source": "plugin_startup_stale_recovery",
|
|
|
|
|
"observed_at": observed_at.astimezone(timezone.utc).isoformat(),
|
|
|
|
|
"updated_before": cutoff_value,
|
|
|
|
|
"previous_status": candidate.status,
|
|
|
|
|
}
|
|
|
|
|
failed = self.update_task_if_status(
|
|
|
|
|
candidate.task_id,
|
|
|
|
|
{candidate.status},
|
|
|
|
|
{
|
|
|
|
|
"status": "failed",
|
|
|
|
|
"progress": candidate.progress,
|
|
|
|
|
"error_message": "task was interrupted by a previous plugin process",
|
|
|
|
|
"metadata": {**candidate.metadata, "recovery": recovery},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
if failed is None:
|
|
|
|
|
skipped.append(
|
|
|
|
|
{
|
|
|
|
|
"task_id": candidate.task_id,
|
|
|
|
|
"status": self.get_job(candidate.task_id).status,
|
|
|
|
|
"reason": "status_changed",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
continue
|
|
|
|
|
queued = self.update_task_if_status(
|
|
|
|
|
failed.task_id,
|
|
|
|
|
{"failed"},
|
|
|
|
|
{
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"error_message": None,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**failed.metadata,
|
|
|
|
|
"recovery": {**recovery, "status": "requeued"},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
require_workflow_not_cancelled=True,
|
|
|
|
|
)
|
|
|
|
|
if queued is None:
|
|
|
|
|
skipped.append(
|
|
|
|
|
{
|
|
|
|
|
"task_id": candidate.task_id,
|
|
|
|
|
"status": self.get_job(candidate.task_id).status,
|
|
|
|
|
"reason": "workflow_cancelled_or_status_changed",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
continue
|
|
|
|
|
recovered.append(queued.task_id)
|
|
|
|
|
return {
|
|
|
|
|
"recovered_task_ids": recovered,
|
|
|
|
|
"recovered": len(recovered),
|
|
|
|
|
"skipped": skipped,
|
|
|
|
|
"cutoff": cutoff_value,
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def requeue_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
2026-07-05 19:01:09 +08:00
|
|
|
jobs = self.list_jobs(status="dead_letter", page=1, page_size=_batch_limit(limit)).items
|
2026-07-11 21:50:34 +08:00
|
|
|
requeued = []
|
2026-07-05 10:56:13 +08:00
|
|
|
errors = []
|
|
|
|
|
for job in jobs:
|
|
|
|
|
try:
|
2026-07-11 21:50:34 +08:00
|
|
|
requeued.append(self.requeue_job(job.task_id, ctx).to_dict())
|
2026-07-05 10:56:13 +08:00
|
|
|
except ApiError as exc:
|
|
|
|
|
errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message})
|
2026-07-11 21:50:34 +08:00
|
|
|
return {"requeued": requeued, "errors": errors, "total": len(requeued)}
|
2026-07-05 10:56:13 +08:00
|
|
|
|
|
|
|
|
def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
2026-07-05 19:01:09 +08:00
|
|
|
jobs = self.list_jobs(status="failed", page=1, page_size=_batch_limit(limit)).items
|
2026-07-05 10:56:13 +08:00
|
|
|
moved = []
|
|
|
|
|
for job in jobs:
|
|
|
|
|
moved.append(
|
|
|
|
|
self.update_task(
|
|
|
|
|
job.task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "dead_letter",
|
|
|
|
|
"metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
).to_dict()
|
|
|
|
|
)
|
|
|
|
|
return {"moved": moved, "total": len(moved)}
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
def delete_job(self, task_id: str, ctx: RequestContext) -> dict[str, Any]:
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
if before.status not in DELETABLE_TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"only terminal task records can be deleted",
|
|
|
|
|
{
|
|
|
|
|
"task_id": task_id,
|
|
|
|
|
"status": before.status,
|
|
|
|
|
"allowed_statuses": sorted(DELETABLE_TASK_STATUSES),
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
# Compare against the status that was authorized above, rather than
|
|
|
|
|
# any deletable status. This keeps the audit snapshot truthful and
|
|
|
|
|
# prevents a concurrent state change from being silently deleted.
|
|
|
|
|
deleted = self.repository.delete_if_status(task_id, {before.status})
|
|
|
|
|
if deleted is None:
|
|
|
|
|
current = self.repository.get(task_id)
|
|
|
|
|
if current is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"task status changed before deletion",
|
|
|
|
|
{"task_id": task_id, "status": current.status},
|
|
|
|
|
)
|
|
|
|
|
node_cleanup = dict(deleted.get("node_cleanup") or {})
|
|
|
|
|
workflow_cleanup = dict(deleted.get("workflow_cleanup") or {})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="delete_task_record",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
metadata={"node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup},
|
|
|
|
|
)
|
|
|
|
|
self._publish(
|
|
|
|
|
"generation.task_deleted",
|
|
|
|
|
{
|
|
|
|
|
"task_id": task_id,
|
|
|
|
|
"status": before.status,
|
|
|
|
|
"node_cleanup": node_cleanup,
|
|
|
|
|
"workflow_cleanup": workflow_cleanup,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return {
|
|
|
|
|
"task_id": task_id,
|
|
|
|
|
"deleted": True,
|
|
|
|
|
"previous_status": before.status,
|
|
|
|
|
"node_cleanup": node_cleanup,
|
|
|
|
|
"workflow_cleanup": workflow_cleanup,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def clear_failed_jobs(self, ctx: RequestContext, *, limit: int = 100) -> dict[str, Any]:
|
|
|
|
|
return self._clear_job_records(ctx, statuses=FAILED_TASK_STATUSES, limit=limit, scope="failed")
|
|
|
|
|
|
|
|
|
|
def clear_terminal_jobs(
|
|
|
|
|
self,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
statuses: list[str] | tuple[str, ...] | set[str] | None = None,
|
|
|
|
|
limit: int = 100,
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
return self._clear_job_records(
|
|
|
|
|
ctx,
|
|
|
|
|
statuses=_terminal_cleanup_statuses(statuses),
|
|
|
|
|
limit=limit,
|
|
|
|
|
scope="terminal",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _clear_job_records(
|
|
|
|
|
self,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
*,
|
|
|
|
|
statuses: tuple[str, ...],
|
|
|
|
|
limit: int,
|
|
|
|
|
scope: str,
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
applied_limit = _batch_limit(limit)
|
|
|
|
|
deleted: list[dict[str, Any]] = []
|
|
|
|
|
errors: list[dict[str, str]] = []
|
|
|
|
|
for status in statuses:
|
|
|
|
|
# Failed compare-and-delete attempts are still processed records.
|
|
|
|
|
# Counting only successful deletions could exceed the caller's
|
|
|
|
|
# batch limit when statuses change concurrently.
|
|
|
|
|
remaining = applied_limit - len(deleted) - len(errors)
|
|
|
|
|
if remaining <= 0:
|
|
|
|
|
break
|
|
|
|
|
jobs = self.list_jobs(status=status, page=1, page_size=remaining).items
|
|
|
|
|
for job in jobs:
|
|
|
|
|
try:
|
|
|
|
|
deleted.append(self.delete_job(job.task_id, ctx))
|
|
|
|
|
except ApiError as exc:
|
|
|
|
|
errors.append(
|
|
|
|
|
{"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message}
|
|
|
|
|
)
|
|
|
|
|
remaining_counts = self.repository.count_by_status()
|
|
|
|
|
return {
|
|
|
|
|
"scope": scope,
|
|
|
|
|
"requested_statuses": list(statuses),
|
|
|
|
|
"deleted": deleted,
|
|
|
|
|
"errors": errors,
|
|
|
|
|
"total": len(deleted),
|
|
|
|
|
"processed": len(deleted) + len(errors),
|
|
|
|
|
"limit": applied_limit,
|
|
|
|
|
"remaining_by_status": {status: remaining_counts.get(status, 0) for status in statuses},
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-05 15:54:48 +08:00
|
|
|
def _validate_task_status_transition(self, current: str, target: str) -> None:
|
|
|
|
|
if target not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
transition = task_transition(current, target)
|
|
|
|
|
if not transition.allowed:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"task status transition is not allowed",
|
|
|
|
|
{"current": current, "target": target, "reason": transition.reason},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _publish_status_change(self, before: GenerationTask, after: GenerationTask) -> None:
|
|
|
|
|
self._publish(
|
|
|
|
|
"generation.task_status_changed",
|
|
|
|
|
{
|
|
|
|
|
"task_id": after.task_id,
|
|
|
|
|
"workflow_id": after.workflow_id,
|
|
|
|
|
"node_id": after.node_id,
|
|
|
|
|
"before_status": before.status,
|
|
|
|
|
"after_status": after.status,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _publish(self, event_type: str, payload: dict[str, Any]) -> None:
|
|
|
|
|
if self.event_bus:
|
|
|
|
|
self.event_bus.publish(event_type, payload)
|
|
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
|
|
|
|
|
def new_task_id() -> str:
|
|
|
|
|
return f"task_{uuid4().hex}"
|
2026-07-05 18:20:14 +08:00
|
|
|
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def _now() -> str:
|
|
|
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
|
|
|
|
|
2026-07-05 18:20:14 +08:00
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
if value is None:
|
|
|
|
|
return None
|
|
|
|
|
text = str(value).strip()
|
|
|
|
|
return text or None
|
2026-07-05 19:01:09 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _batch_limit(value: int) -> int:
|
|
|
|
|
try:
|
|
|
|
|
limit = int(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
limit = 50
|
|
|
|
|
return min(max(limit, 1), 100)
|
2026-07-11 21:50:34 +08:00
|
|
|
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
def _terminal_cleanup_statuses(
|
|
|
|
|
statuses: list[str] | tuple[str, ...] | set[str] | None,
|
|
|
|
|
) -> tuple[str, ...]:
|
|
|
|
|
if statuses is None:
|
|
|
|
|
return (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES)
|
|
|
|
|
if not isinstance(statuses, (list, tuple, set)):
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"terminal cleanup statuses must be an array",
|
|
|
|
|
{"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
requested = {str(status or "").strip().lower() for status in statuses if str(status or "").strip()}
|
|
|
|
|
if not requested:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"terminal cleanup requires at least one status",
|
|
|
|
|
{"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
unsupported = sorted(requested - DELETABLE_TASK_STATUSES)
|
|
|
|
|
if unsupported:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"terminal cleanup cannot include active task statuses",
|
|
|
|
|
{
|
|
|
|
|
"field": "statuses",
|
|
|
|
|
"unsupported_statuses": unsupported,
|
|
|
|
|
"allowed_statuses": sorted(DELETABLE_TASK_STATUSES),
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
canonical_order = (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES)
|
|
|
|
|
return tuple(status for status in canonical_order if status in requested)
|
|
|
|
|
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def _int_or_default(value: Any, default: int) -> int:
|
|
|
|
|
try:
|
|
|
|
|
return int(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return default
|