astrbot_plugin_imagestudio/backend/services/queue_service.py

772 行
29 KiB
Python

2026-07-05 09:07:34 +08:00
"""Queue and job service."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
2026-07-05 09:07:34 +08:00
from typing import Any
from uuid import uuid4
from backend.dto.common import ApiError, ErrorCode
from backend.dto.job_dto import GenerationTask, JobListResult
from core.audit_service import AuditService
from core.event_bus import EventBus
from core.fsm import task_transition
2026-07-05 09:07:34 +08:00
from core.identity import RequestContext
from database.repositories.job_repository import JobRepository
from core.text_utils import optional_text as _optional_text
2026-07-05 09:07:34 +08:00
TASK_STATUSES = {
"created",
"queued",
"running",
"completed",
"succeeded",
"failed",
"cancelled",
"dead_letter",
"retrying",
"superseded",
2026-07-05 09:07:34 +08:00
}
RETRYABLE_TASK_STATUSES = {"failed", "dead_letter", "cancelled"}
FAILED_TASK_STATUSES = ("failed", "dead_letter", "cancelled")
COMPLETED_TASK_STATUSES = ("completed", "succeeded", "superseded")
DELETABLE_TASK_STATUSES = frozenset((*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES))
2026-07-05 09:07:34 +08:00
class QueueService:
def __init__(self, repository: JobRepository, audit_service: AuditService, event_bus: EventBus | None = None):
2026-07-05 09:07:34 +08:00
self.repository = repository
self.audit_service = audit_service
self.event_bus = event_bus
2026-07-05 09:07:34 +08:00
def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
status = str(payload.get("status", "created"))
if status not in TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"unsupported task status",
{"field": "status", "allowed": sorted(TASK_STATUSES)},
)
2026-07-05 18:20:14 +08:00
idempotency_key = _optional_text(payload.get("idempotency_key"))
if idempotency_key:
existing = self.repository.get_by_idempotency_key(idempotency_key)
if existing is not None:
return existing
2026-07-05 09:07:34 +08:00
task = self.repository.create(
{
"task_id": payload.get("task_id") or new_task_id(),
"profile_id": payload.get("profile_id"),
"workflow_id": payload.get("workflow_id"),
"node_id": payload.get("node_id"),
"task_type": payload.get("task_type", "generation"),
"status": status,
"model_route": payload.get("model_route"),
"progress": payload.get("progress", 0),
2026-07-05 18:20:14 +08:00
"idempotency_key": idempotency_key,
2026-07-05 09:07:34 +08:00
"request": payload.get("request") or {},
"result": payload.get("result") or {},
"error_message": payload.get("error_message"),
"metadata": payload.get("metadata") or {},
}
)
self.audit_service.record(
ctx=ctx,
action="create_task",
target_type="generation_task",
target_id=task.task_id,
after_snapshot=task.to_dict(),
)
self._publish(
"generation.task_created",
{
"task_id": task.task_id,
"status": task.status,
"workflow_id": task.workflow_id,
"node_id": task.node_id,
},
)
2026-07-05 09:07:34 +08:00
return task
def list_claimable_jobs(self, *, status: str = "queued", limit: int = 25) -> list[GenerationTask]:
"""Oldest-first queued tasks for worker claim (FIFO).
Separate from ``list_jobs``, which is the newest-first *display*
listing; consuming the queue in display order made it a LIFO and
starved the backlog.
"""
return self.repository.list_claimable(status=status, limit=limit)
2026-07-05 09:07:34 +08:00
def list_jobs(
self,
*,
status: str | None = None,
workflow_id: str | None = None,
profile_id: str | None = None,
page: int = 1,
page_size: int = 30,
) -> JobListResult:
if status and status not in TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"unsupported task status",
{"field": "status", "allowed": sorted(TASK_STATUSES)},
)
normalized_page = max(page, 1)
normalized_page_size = min(max(page_size, 1), 100)
items, total = self.repository.list(
status=status,
workflow_id=workflow_id,
profile_id=profile_id,
page=normalized_page,
page_size=normalized_page_size,
)
return JobListResult(
items=items,
total=total,
page=normalized_page,
page_size=normalized_page_size,
)
def get_job(self, task_id: str) -> GenerationTask:
task = self.repository.get(task_id)
if task is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
return task
def get_job_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None:
normalized_key = _optional_text(idempotency_key)
if normalized_key is None:
return None
return self.repository.get_by_idempotency_key(normalized_key)
2026-07-06 02:23:05 +08:00
def cancel_job(self, task_id: str, ctx: RequestContext, *, reason: str | None = None) -> GenerationTask:
while True:
before = self.get_job(task_id)
if before.status == "cancelled":
return before
if before.status == "retrying":
raise ApiError(
ErrorCode.TASK_NOT_CANCELABLE,
"task retry execution is already in progress",
{"task_id": task_id, "status": before.status},
)
if before.status in {"completed", "succeeded", "superseded"}:
raise ApiError(
ErrorCode.TASK_NOT_CANCELABLE,
"task cannot be cancelled from current status",
{"task_id": task_id, "status": before.status},
)
self._validate_task_status_transition(before.status, "cancelled")
after = self.repository.update_if_status(
task_id,
{before.status},
{
"status": "cancelled",
"error_message": _optional_text(reason) or before.error_message,
"metadata": {
**before.metadata,
"cancellation": {
"requested_by": ctx.actor_id or "system",
"requested_at": _now(),
"reason": _optional_text(reason) or "cancelled",
"previous_status": before.status,
},
},
},
)
if after is not None:
break
# The task changed after it was read. Re-evaluate the current
# state instead of applying cancellation from a stale snapshot.
self.audit_service.record(
ctx=ctx,
action="cancel_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
self._publish_status_change(before, after)
return after
def update_task_if_status(
self,
task_id: str,
expected_statuses: set[str] | tuple[str, ...] | list[str],
payload: dict[str, Any],
ctx: RequestContext,
*,
require_workflow_not_cancelled: bool = False,
) -> GenerationTask | None:
"""Update a task only if the status observed by this caller still owns it."""
normalized_statuses = {
str(status).strip()
for status in expected_statuses
if str(status).strip()
}
2026-07-05 09:07:34 +08:00
before = self.get_job(task_id)
if before.status not in normalized_statuses:
return None
status = payload.get("status")
if status is not None:
self._validate_task_status_transition(before.status, str(status))
if require_workflow_not_cancelled:
after = self.repository.update_if_status_unless_workflow_cancelled(
task_id,
{before.status},
payload,
2026-07-05 09:07:34 +08:00
)
else:
after = self.repository.update_if_status(task_id, {before.status}, payload)
if after is None:
return None
self.audit_service.record(
ctx=ctx,
action="update_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
metadata={"compare_and_set": True},
)
if before.status != after.status:
self._publish_status_change(before, after)
self._publish("generation.task_updated", {"task_id": task_id, "status": after.status})
return after
def cancel_workflow_jobs(
self,
workflow_id: str,
ctx: RequestContext,
*,
reason: str = "workflow cancelled",
) -> dict[str, Any]:
"""CAS-cancel every queued/running task currently owned by a workflow."""
cancelled_ids: list[str] = []
skipped: list[dict[str, str]] = []
for status in ("queued", "running"):
while True:
candidates, _ = self.repository.list(
status=status,
workflow_id=workflow_id,
page=1,
page_size=100,
)
if not candidates:
break
changed = 0
for candidate in candidates:
try:
cancelled = self.cancel_job(candidate.task_id, ctx, reason=reason)
if cancelled.status == "cancelled":
cancelled_ids.append(cancelled.task_id)
except ApiError as exc:
current = self.get_job(candidate.task_id)
skipped.append(
{
"task_id": candidate.task_id,
"status": current.status,
"error_code": exc.error_code.value,
}
)
current = self.get_job(candidate.task_id)
if current.status != status:
changed += 1
if changed == 0:
break
return {
"workflow_id": workflow_id,
"cancelled_task_ids": cancelled_ids,
"skipped": skipped,
}
def is_cancelled(self, task_id: str) -> bool:
return self.get_job(task_id).status == "cancelled"
def claim_queued_task(
self,
task_id: str,
ctx: RequestContext,
*,
worker_id: str,
) -> GenerationTask | None:
"""Atomically claim one queued task for a worker."""
before = self.get_job(task_id)
if before.status != "queued":
return None
self._validate_task_status_transition(before.status, "running")
attempt = _int_or_default(before.metadata.get("worker_attempt"), 0) + 1
after = self.repository.update_if_status(
2026-07-06 02:23:05 +08:00
task_id,
{"queued"},
2026-07-06 02:23:05 +08:00
{
"status": "running",
"progress": max(before.progress, 1),
2026-07-06 02:23:05 +08:00
"metadata": {
**before.metadata,
"worker_attempt": attempt,
"worker": {
"worker_id": worker_id,
"status": "running",
"claimed_at": _now(),
2026-07-06 02:23:05 +08:00
},
},
},
)
2026-07-05 09:07:34 +08:00
if after is None:
return None
2026-07-05 09:07:34 +08:00
self.audit_service.record(
ctx=ctx,
action="claim_task",
2026-07-05 09:07:34 +08:00
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
self._publish_status_change(before, after)
2026-07-05 09:07:34 +08:00
return after
def requeue_job(self, task_id: str, ctx: RequestContext) -> GenerationTask:
"""Requeue a task for an explicitly managed QueueWorker.
Web retry routes use GenerationJobRetryService instead because the
AstrBot plugin does not start a persistent QueueWorker by default.
"""
2026-07-06 02:23:05 +08:00
2026-07-05 09:07:34 +08:00
before = self.get_job(task_id)
if before.status not in RETRYABLE_TASK_STATUSES:
2026-07-05 09:07:34 +08:00
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task cannot be retried from current status",
{"task_id": task_id, "status": before.status},
)
self._validate_task_status_transition(before.status, "queued")
2026-07-05 09:07:34 +08:00
after = self.repository.update(
task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {**before.metadata, "requeue_requested": True},
2026-07-05 09:07:34 +08:00
},
)
if after is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
self.audit_service.record(
ctx=ctx,
action="requeue_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
self._publish_status_change(before, after)
return after
def begin_retry(self, task_id: str, ctx: RequestContext, *, operation_id: str) -> GenerationTask:
"""Atomically reserve a failed task for synchronous retry execution."""
before = self.get_job(task_id)
if before.status not in RETRYABLE_TASK_STATUSES:
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task cannot be retried from current status",
{"task_id": task_id, "status": before.status},
)
self._validate_task_status_transition(before.status, "retrying")
retry_metadata = {
"operation_id": operation_id,
"status": "running",
"requested_by": ctx.actor_id or "system",
"requested_at": _now(),
"previous_status": before.status,
}
after = self.repository.update_if_status(
task_id,
{before.status},
{
"status": "retrying",
"progress": 0,
"metadata": {**before.metadata, "retry_execution": retry_metadata},
},
)
if after is None:
current = self.get_job(task_id)
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task retry was already claimed or its status changed",
{"task_id": task_id, "status": current.status},
)
self.audit_service.record(
ctx=ctx,
action="begin_retry_task",
2026-07-05 09:07:34 +08:00
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
self._publish_status_change(before, after)
2026-07-05 09:07:34 +08:00
return after
def update_task(self, task_id: str, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
before = self.get_job(task_id)
status = payload.get("status")
if status is not None:
self._validate_task_status_transition(before.status, str(status))
after = self.repository.update(task_id, payload)
if after is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
self.audit_service.record(
ctx=ctx,
action="update_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
if before.status != after.status:
self._publish_status_change(before, after)
self._publish("generation.task_updated", {"task_id": task_id, "status": after.status})
return after
2026-07-05 09:07:34 +08:00
def queue_summary(self) -> dict[str, int]:
counts = self.repository.count_by_status()
return {
"pending": counts.get("created", 0) + counts.get("queued", 0),
"running": counts.get("running", 0) + counts.get("retrying", 0),
2026-07-05 09:07:34 +08:00
"failed": counts.get("failed", 0),
"dead_letter": counts.get("dead_letter", 0),
}
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
return [task.to_dict() for task in self.repository.recent(limit=limit)]
def recover_stale_tasks(
self,
ctx: RequestContext,
*,
stale_after_seconds: int = 300,
limit: int = 100,
source: str | None = "workflow_round",
now: datetime | None = None,
) -> dict[str, Any]:
"""Recover tasks abandoned by a previous plugin process.
The task FSM intentionally does not allow ``running -> queued``.
Recovery therefore records a failed terminal observation first and
only then requeues it. The two compare-and-set operations also avoid
stealing work from a live worker that updated the row concurrently.
"""
observed_at = now or datetime.now(timezone.utc)
if observed_at.tzinfo is None:
observed_at = observed_at.replace(tzinfo=timezone.utc)
cutoff = observed_at.astimezone(timezone.utc) - timedelta(
seconds=max(int(stale_after_seconds), 1)
)
cutoff_value = cutoff.strftime("%Y-%m-%d %H:%M:%S")
candidates = self.repository.list_stale(
statuses=("running", "retrying"),
updated_before=cutoff_value,
limit=min(max(int(limit), 1), 500),
)
recovered: list[str] = []
skipped: list[dict[str, str]] = []
for candidate in candidates:
task_source = str(candidate.metadata.get("source") or "").strip()
if source is not None and task_source != source:
skipped.append(
{
"task_id": candidate.task_id,
"status": candidate.status,
"reason": "source_not_managed",
}
)
continue
recovery = {
"source": "plugin_startup_stale_recovery",
"observed_at": observed_at.astimezone(timezone.utc).isoformat(),
"updated_before": cutoff_value,
"previous_status": candidate.status,
}
failed = self.update_task_if_status(
candidate.task_id,
{candidate.status},
{
"status": "failed",
"progress": candidate.progress,
"error_message": "task was interrupted by a previous plugin process",
"metadata": {**candidate.metadata, "recovery": recovery},
},
ctx,
)
if failed is None:
skipped.append(
{
"task_id": candidate.task_id,
"status": self.get_job(candidate.task_id).status,
"reason": "status_changed",
}
)
continue
queued = self.update_task_if_status(
failed.task_id,
{"failed"},
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**failed.metadata,
"recovery": {**recovery, "status": "requeued"},
},
},
ctx,
require_workflow_not_cancelled=True,
)
if queued is None:
skipped.append(
{
"task_id": candidate.task_id,
"status": self.get_job(candidate.task_id).status,
"reason": "workflow_cancelled_or_status_changed",
}
)
continue
recovered.append(queued.task_id)
return {
"recovered_task_ids": recovered,
"recovered": len(recovered),
"skipped": skipped,
"cutoff": cutoff_value,
}
def requeue_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
2026-07-05 19:01:09 +08:00
jobs = self.list_jobs(status="dead_letter", page=1, page_size=_batch_limit(limit)).items
requeued = []
errors = []
for job in jobs:
try:
requeued.append(self.requeue_job(job.task_id, ctx).to_dict())
except ApiError as exc:
errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message})
return {"requeued": requeued, "errors": errors, "total": len(requeued)}
def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
2026-07-05 19:01:09 +08:00
jobs = self.list_jobs(status="failed", page=1, page_size=_batch_limit(limit)).items
moved = []
for job in jobs:
moved.append(
self.update_task(
job.task_id,
{
"status": "dead_letter",
"metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"},
},
ctx,
).to_dict()
)
return {"moved": moved, "total": len(moved)}
def delete_job(self, task_id: str, ctx: RequestContext) -> dict[str, Any]:
before = self.get_job(task_id)
if before.status not in DELETABLE_TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"only terminal task records can be deleted",
{
"task_id": task_id,
"status": before.status,
"allowed_statuses": sorted(DELETABLE_TASK_STATUSES),
},
)
# Compare against the status that was authorized above, rather than
# any deletable status. This keeps the audit snapshot truthful and
# prevents a concurrent state change from being silently deleted.
deleted = self.repository.delete_if_status(task_id, {before.status})
if deleted is None:
current = self.repository.get(task_id)
if current is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"task status changed before deletion",
{"task_id": task_id, "status": current.status},
)
node_cleanup = dict(deleted.get("node_cleanup") or {})
workflow_cleanup = dict(deleted.get("workflow_cleanup") or {})
self.audit_service.record(
ctx=ctx,
action="delete_task_record",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
metadata={"node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup},
)
self._publish(
"generation.task_deleted",
{
"task_id": task_id,
"status": before.status,
"node_cleanup": node_cleanup,
"workflow_cleanup": workflow_cleanup,
},
)
return {
"task_id": task_id,
"deleted": True,
"previous_status": before.status,
"node_cleanup": node_cleanup,
"workflow_cleanup": workflow_cleanup,
}
def clear_failed_jobs(self, ctx: RequestContext, *, limit: int = 100) -> dict[str, Any]:
return self._clear_job_records(ctx, statuses=FAILED_TASK_STATUSES, limit=limit, scope="failed")
def clear_terminal_jobs(
self,
ctx: RequestContext,
*,
statuses: list[str] | tuple[str, ...] | set[str] | None = None,
limit: int = 100,
) -> dict[str, Any]:
return self._clear_job_records(
ctx,
statuses=_terminal_cleanup_statuses(statuses),
limit=limit,
scope="terminal",
)
def _clear_job_records(
self,
ctx: RequestContext,
*,
statuses: tuple[str, ...],
limit: int,
scope: str,
) -> dict[str, Any]:
applied_limit = _batch_limit(limit)
deleted: list[dict[str, Any]] = []
errors: list[dict[str, str]] = []
for status in statuses:
# Failed compare-and-delete attempts are still processed records.
# Counting only successful deletions could exceed the caller's
# batch limit when statuses change concurrently.
remaining = applied_limit - len(deleted) - len(errors)
if remaining <= 0:
break
jobs = self.list_jobs(status=status, page=1, page_size=remaining).items
for job in jobs:
try:
deleted.append(self.delete_job(job.task_id, ctx))
except ApiError as exc:
errors.append(
{"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message}
)
remaining_counts = self.repository.count_by_status()
return {
"scope": scope,
"requested_statuses": list(statuses),
"deleted": deleted,
"errors": errors,
"total": len(deleted),
"processed": len(deleted) + len(errors),
"limit": applied_limit,
"remaining_by_status": {status: remaining_counts.get(status, 0) for status in statuses},
}
def _validate_task_status_transition(self, current: str, target: str) -> None:
if target not in TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"unsupported task status",
{"field": "status", "allowed": sorted(TASK_STATUSES)},
)
transition = task_transition(current, target)
if not transition.allowed:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"task status transition is not allowed",
{"current": current, "target": target, "reason": transition.reason},
)
def _publish_status_change(self, before: GenerationTask, after: GenerationTask) -> None:
self._publish(
"generation.task_status_changed",
{
"task_id": after.task_id,
"workflow_id": after.workflow_id,
"node_id": after.node_id,
"before_status": before.status,
"after_status": after.status,
},
)
def _publish(self, event_type: str, payload: dict[str, Any]) -> None:
if self.event_bus:
self.event_bus.publish(event_type, payload)
2026-07-05 09:07:34 +08:00
def new_task_id() -> str:
return f"task_{uuid4().hex}"
2026-07-05 18:20:14 +08:00
2026-07-06 02:23:05 +08:00
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
2026-07-05 19:01:09 +08:00
def _batch_limit(value: int) -> int:
try:
limit = int(value)
except (TypeError, ValueError):
limit = 50
return min(max(limit, 1), 100)
def _terminal_cleanup_statuses(
statuses: list[str] | tuple[str, ...] | set[str] | None,
) -> tuple[str, ...]:
if statuses is None:
return (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES)
if not isinstance(statuses, (list, tuple, set)):
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"terminal cleanup statuses must be an array",
{"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)},
)
requested = {str(status or "").strip().lower() for status in statuses if str(status or "").strip()}
if not requested:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"terminal cleanup requires at least one status",
{"field": "statuses", "allowed_statuses": sorted(DELETABLE_TASK_STATUSES)},
)
unsupported = sorted(requested - DELETABLE_TASK_STATUSES)
if unsupported:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"terminal cleanup cannot include active task statuses",
{
"field": "statuses",
"unsupported_statuses": unsupported,
"allowed_statuses": sorted(DELETABLE_TASK_STATUSES),
},
)
canonical_order = (*FAILED_TASK_STATUSES, *COMPLETED_TASK_STATUSES)
return tuple(status for status in canonical_order if status in requested)
def _int_or_default(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default