2026-07-05 09:07:34 +08:00
|
|
|
"""Queue and job service."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
from datetime import datetime, timezone
|
2026-07-05 09:07:34 +08:00
|
|
|
from typing import Any
|
|
|
|
|
from uuid import uuid4
|
|
|
|
|
|
|
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
|
|
|
from backend.dto.job_dto import GenerationTask, JobListResult
|
|
|
|
|
from core.audit_service import AuditService
|
2026-07-05 15:54:48 +08:00
|
|
|
from core.event_bus import EventBus
|
|
|
|
|
from core.fsm import task_transition
|
2026-07-05 09:07:34 +08:00
|
|
|
from core.identity import RequestContext
|
|
|
|
|
from database.repositories.job_repository import JobRepository
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
TASK_STATUSES = {
|
|
|
|
|
"created",
|
|
|
|
|
"queued",
|
|
|
|
|
"running",
|
|
|
|
|
"completed",
|
|
|
|
|
"succeeded",
|
|
|
|
|
"failed",
|
|
|
|
|
"cancelled",
|
|
|
|
|
"dead_letter",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class QueueService:
|
2026-07-05 15:54:48 +08:00
|
|
|
def __init__(self, repository: JobRepository, audit_service: AuditService, event_bus: EventBus | None = None):
|
2026-07-05 09:07:34 +08:00
|
|
|
self.repository = repository
|
|
|
|
|
self.audit_service = audit_service
|
2026-07-05 15:54:48 +08:00
|
|
|
self.event_bus = event_bus
|
2026-07-05 09:07:34 +08:00
|
|
|
|
|
|
|
|
def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
status = str(payload.get("status", "created"))
|
|
|
|
|
if status not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
2026-07-05 18:20:14 +08:00
|
|
|
idempotency_key = _optional_text(payload.get("idempotency_key"))
|
|
|
|
|
if idempotency_key:
|
|
|
|
|
existing = self.repository.get_by_idempotency_key(idempotency_key)
|
|
|
|
|
if existing is not None:
|
|
|
|
|
return existing
|
2026-07-05 09:07:34 +08:00
|
|
|
task = self.repository.create(
|
|
|
|
|
{
|
|
|
|
|
"task_id": payload.get("task_id") or new_task_id(),
|
|
|
|
|
"profile_id": payload.get("profile_id"),
|
|
|
|
|
"workflow_id": payload.get("workflow_id"),
|
|
|
|
|
"node_id": payload.get("node_id"),
|
|
|
|
|
"task_type": payload.get("task_type", "generation"),
|
|
|
|
|
"status": status,
|
|
|
|
|
"model_route": payload.get("model_route"),
|
|
|
|
|
"progress": payload.get("progress", 0),
|
2026-07-05 18:20:14 +08:00
|
|
|
"idempotency_key": idempotency_key,
|
2026-07-05 09:07:34 +08:00
|
|
|
"request": payload.get("request") or {},
|
|
|
|
|
"result": payload.get("result") or {},
|
|
|
|
|
"error_message": payload.get("error_message"),
|
|
|
|
|
"metadata": payload.get("metadata") or {},
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="create_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task.task_id,
|
|
|
|
|
after_snapshot=task.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish(
|
|
|
|
|
"generation.task_created",
|
|
|
|
|
{
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
"status": task.status,
|
|
|
|
|
"workflow_id": task.workflow_id,
|
|
|
|
|
"node_id": task.node_id,
|
|
|
|
|
},
|
|
|
|
|
)
|
2026-07-05 09:07:34 +08:00
|
|
|
return task
|
|
|
|
|
|
|
|
|
|
def list_jobs(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
status: str | None = None,
|
|
|
|
|
workflow_id: str | None = None,
|
|
|
|
|
profile_id: str | None = None,
|
|
|
|
|
page: int = 1,
|
|
|
|
|
page_size: int = 30,
|
|
|
|
|
) -> JobListResult:
|
|
|
|
|
if status and status not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
normalized_page = max(page, 1)
|
|
|
|
|
normalized_page_size = min(max(page_size, 1), 100)
|
|
|
|
|
items, total = self.repository.list(
|
|
|
|
|
status=status,
|
|
|
|
|
workflow_id=workflow_id,
|
|
|
|
|
profile_id=profile_id,
|
|
|
|
|
page=normalized_page,
|
|
|
|
|
page_size=normalized_page_size,
|
|
|
|
|
)
|
|
|
|
|
return JobListResult(
|
|
|
|
|
items=items,
|
|
|
|
|
total=total,
|
|
|
|
|
page=normalized_page,
|
|
|
|
|
page_size=normalized_page_size,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def get_job(self, task_id: str) -> GenerationTask:
|
|
|
|
|
task = self.repository.get(task_id)
|
|
|
|
|
if task is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
return task
|
|
|
|
|
|
2026-07-05 18:23:35 +08:00
|
|
|
def get_job_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None:
|
|
|
|
|
normalized_key = _optional_text(idempotency_key)
|
|
|
|
|
if normalized_key is None:
|
|
|
|
|
return None
|
|
|
|
|
return self.repository.get_by_idempotency_key(normalized_key)
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def cancel_job(self, task_id: str, ctx: RequestContext, *, reason: str | None = None) -> GenerationTask:
|
2026-07-05 09:07:34 +08:00
|
|
|
before = self.get_job(task_id)
|
2026-07-06 02:23:05 +08:00
|
|
|
if before.status == "cancelled":
|
|
|
|
|
return before
|
|
|
|
|
if before.status in {"completed", "succeeded"}:
|
2026-07-05 09:07:34 +08:00
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_CANCELABLE,
|
|
|
|
|
"task cannot be cancelled from current status",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._validate_task_status_transition(before.status, "cancelled")
|
2026-07-06 02:23:05 +08:00
|
|
|
after = self.repository.update(
|
|
|
|
|
task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "cancelled",
|
|
|
|
|
"error_message": _optional_text(reason) or before.error_message,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**before.metadata,
|
|
|
|
|
"cancellation": {
|
|
|
|
|
"requested_by": ctx.actor_id or "system",
|
|
|
|
|
"requested_at": _now(),
|
|
|
|
|
"reason": _optional_text(reason) or "cancelled",
|
|
|
|
|
"previous_status": before.status,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
)
|
2026-07-05 09:07:34 +08:00
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="cancel_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish_status_change(before, after)
|
2026-07-05 09:07:34 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def is_cancelled(self, task_id: str) -> bool:
|
|
|
|
|
return self.get_job(task_id).status == "cancelled"
|
|
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
def retry_job(self, task_id: str, ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
if before.status not in {"failed", "dead_letter", "cancelled"}:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.TASK_NOT_RETRYABLE,
|
|
|
|
|
"task cannot be retried from current status",
|
|
|
|
|
{"task_id": task_id, "status": before.status},
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._validate_task_status_transition(before.status, "queued")
|
2026-07-05 09:07:34 +08:00
|
|
|
after = self.repository.update(
|
|
|
|
|
task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"error_message": None,
|
|
|
|
|
"metadata": {**before.metadata, "retry_requested": True},
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="retry_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish_status_change(before, after)
|
2026-07-05 09:07:34 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-05 09:28:39 +08:00
|
|
|
def update_task(self, task_id: str, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
|
|
|
|
|
before = self.get_job(task_id)
|
|
|
|
|
status = payload.get("status")
|
2026-07-05 15:54:48 +08:00
|
|
|
if status is not None:
|
|
|
|
|
self._validate_task_status_transition(before.status, str(status))
|
2026-07-05 09:28:39 +08:00
|
|
|
after = self.repository.update(task_id, payload)
|
|
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
|
|
|
|
|
self.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="update_task",
|
|
|
|
|
target_type="generation_task",
|
|
|
|
|
target_id=task_id,
|
|
|
|
|
before_snapshot=before.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
if before.status != after.status:
|
|
|
|
|
self._publish_status_change(before, after)
|
|
|
|
|
self._publish("generation.task_updated", {"task_id": task_id, "status": after.status})
|
2026-07-05 09:28:39 +08:00
|
|
|
return after
|
|
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
def queue_summary(self) -> dict[str, int]:
|
|
|
|
|
counts = self.repository.count_by_status()
|
|
|
|
|
return {
|
|
|
|
|
"pending": counts.get("created", 0) + counts.get("queued", 0),
|
|
|
|
|
"running": counts.get("running", 0),
|
|
|
|
|
"failed": counts.get("failed", 0),
|
|
|
|
|
"dead_letter": counts.get("dead_letter", 0),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
|
|
|
|
|
return [task.to_dict() for task in self.repository.recent(limit=limit)]
|
|
|
|
|
|
2026-07-05 10:56:13 +08:00
|
|
|
def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
2026-07-05 19:01:09 +08:00
|
|
|
jobs = self.list_jobs(status="dead_letter", page=1, page_size=_batch_limit(limit)).items
|
2026-07-05 10:56:13 +08:00
|
|
|
retried = []
|
|
|
|
|
errors = []
|
|
|
|
|
for job in jobs:
|
|
|
|
|
try:
|
|
|
|
|
retried.append(self.retry_job(job.task_id, ctx).to_dict())
|
|
|
|
|
except ApiError as exc:
|
|
|
|
|
errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message})
|
|
|
|
|
return {"retried": retried, "errors": errors, "total": len(retried)}
|
|
|
|
|
|
|
|
|
|
def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
2026-07-05 19:01:09 +08:00
|
|
|
jobs = self.list_jobs(status="failed", page=1, page_size=_batch_limit(limit)).items
|
2026-07-05 10:56:13 +08:00
|
|
|
moved = []
|
|
|
|
|
for job in jobs:
|
|
|
|
|
moved.append(
|
|
|
|
|
self.update_task(
|
|
|
|
|
job.task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "dead_letter",
|
|
|
|
|
"metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
).to_dict()
|
|
|
|
|
)
|
|
|
|
|
return {"moved": moved, "total": len(moved)}
|
|
|
|
|
|
2026-07-05 15:54:48 +08:00
|
|
|
def _validate_task_status_transition(self, current: str, target: str) -> None:
|
|
|
|
|
if target not in TASK_STATUSES:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"unsupported task status",
|
|
|
|
|
{"field": "status", "allowed": sorted(TASK_STATUSES)},
|
|
|
|
|
)
|
|
|
|
|
transition = task_transition(current, target)
|
|
|
|
|
if not transition.allowed:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"task status transition is not allowed",
|
|
|
|
|
{"current": current, "target": target, "reason": transition.reason},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _publish_status_change(self, before: GenerationTask, after: GenerationTask) -> None:
|
|
|
|
|
self._publish(
|
|
|
|
|
"generation.task_status_changed",
|
|
|
|
|
{
|
|
|
|
|
"task_id": after.task_id,
|
|
|
|
|
"workflow_id": after.workflow_id,
|
|
|
|
|
"node_id": after.node_id,
|
|
|
|
|
"before_status": before.status,
|
|
|
|
|
"after_status": after.status,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _publish(self, event_type: str, payload: dict[str, Any]) -> None:
|
|
|
|
|
if self.event_bus:
|
|
|
|
|
self.event_bus.publish(event_type, payload)
|
|
|
|
|
|
2026-07-05 09:07:34 +08:00
|
|
|
|
|
|
|
|
def new_task_id() -> str:
|
|
|
|
|
return f"task_{uuid4().hex}"
|
2026-07-05 18:20:14 +08:00
|
|
|
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def _now() -> str:
|
|
|
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
|
|
|
|
|
2026-07-05 18:20:14 +08:00
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
if value is None:
|
|
|
|
|
return None
|
|
|
|
|
text = str(value).strip()
|
|
|
|
|
return text or None
|
2026-07-05 19:01:09 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _batch_limit(value: int) -> int:
|
|
|
|
|
try:
|
|
|
|
|
limit = int(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
limit = 50
|
|
|
|
|
return min(max(limit, 1), 100)
|