"""Persistent queue worker primitives.""" from __future__ import annotations import threading from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any from backend.dto.job_dto import GenerationTask from backend.services.queue_service import QueueService from core.identity import RequestContext WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"] @dataclass(frozen=True) class WorkerTaskUpdate: status: str = "succeeded" progress: int = 100 result: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) error_message: str | None = None @dataclass(frozen=True) class RetryPolicy: max_attempts: int = 1 base_delay_seconds: float = 0.0 backoff_multiplier: float = 2.0 retryable_error_types: tuple[str, ...] = () def should_retry(self, attempt: int, exc: Exception) -> bool: if attempt >= max(int(self.max_attempts), 1): return False if not self.retryable_error_types: return True return type(exc).__name__ in self.retryable_error_types def next_retry_at(self, attempt: int) -> str: delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0)) return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat() @dataclass(frozen=True) class WorkerRunResult: worker_id: str status: str task_id: str | None = None processed: int = 0 succeeded: int = 0 failed: int = 0 error_message: str | None = None def to_dict(self) -> dict[str, Any]: return { "worker_id": self.worker_id, "status": self.status, "task_id": self.task_id, "processed": self.processed, "succeeded": self.succeeded, "failed": self.failed, "error_message": self.error_message, } class QueueWorker: def __init__( self, queue_service: QueueService, handler: WorkerHandler, *, worker_id: str = "imagestudio-worker", interval_seconds: float = 1.0, claim_batch_size: int = 5, retry_policy: RetryPolicy | None = None, ): self.queue_service = queue_service self.handler = handler self.worker_id = worker_id self.interval_seconds = max(float(interval_seconds), 0.01) self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) self.retry_policy = retry_policy or RetryPolicy() self._stop_event = threading.Event() self._lock = threading.Lock() self._thread: threading.Thread | None = None def run_once(self, ctx: RequestContext) -> WorkerRunResult: task = self._claim_next(ctx) if task is None: return WorkerRunResult(worker_id=self.worker_id, status="idle") try: update = _normalize_update(self.handler(task)) latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) completed = self.queue_service.update_task( task.task_id, { "status": update.status, "progress": _progress(update.progress), "result": {**task.result, **update.result}, "error_message": update.error_message, "metadata": { **task.metadata, **update.metadata, "worker": { "worker_id": self.worker_id, "status": update.status, "finished_at": _now(), }, }, }, ctx, ) return WorkerRunResult( worker_id=self.worker_id, status=completed.status, task_id=completed.task_id, processed=1, succeeded=1 if completed.status in {"completed", "succeeded"} else 0, failed=1 if completed.status == "failed" else 0, error_message=completed.error_message, ) except Exception as exc: return self._handle_failure(task, exc, ctx) def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: limit = min(max(int(max_tasks), 1), 100) results: list[dict[str, Any]] = [] for _ in range(limit): result = self.run_once(ctx) results.append(result.to_dict()) if result.status == "idle": break return { "worker_id": self.worker_id, "processed": sum(item["processed"] for item in results), "succeeded": sum(item["succeeded"] for item in results), "failed": sum(item["failed"] for item in results), "results": results, } def start(self, ctx: RequestContext) -> dict[str, Any]: with self._lock: if self.is_running(): return {"worker_id": self.worker_id, "running": True, "already_running": True} self._stop_event.clear() self._thread = threading.Thread( target=self._run_loop, args=(ctx,), daemon=True, name=f"ImageStudioQueueWorker-{self.worker_id}", ) self._thread.start() return {"worker_id": self.worker_id, "running": True, "already_running": False} def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: self._stop_event.set() thread = self._thread if thread is not None: thread.join(timeout=max(float(timeout_seconds), 0.1)) return {"worker_id": self.worker_id, "running": self.is_running()} def is_running(self) -> bool: thread = self._thread return bool(thread and thread.is_alive()) def _run_loop(self, ctx: RequestContext) -> None: while not self._stop_event.is_set(): result = self.run_once(ctx) if result.status == "idle": self._stop_event.wait(self.interval_seconds) def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items for candidate in queued: if _retry_deferred(candidate): continue claimed = self.queue_service.claim_queued_task( candidate.task_id, ctx, worker_id=self.worker_id, ) if claimed is not None: return claimed return None def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult: latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) message = _safe_error_message(exc) attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1)) failed = self.queue_service.update_task( task.task_id, { "status": "failed", "error_message": message, "metadata": { **task.metadata, "worker": { "worker_id": self.worker_id, "status": "failed", "error_type": type(exc).__name__, "failed_at": _now(), }, }, }, ctx, ) if self.retry_policy.should_retry(attempt, exc): queued = self.queue_service.update_task( failed.task_id, { "status": "queued", "progress": 0, "error_message": None, "metadata": { **failed.metadata, "retry": { "scheduled": True, "attempt": attempt, "max_attempts": max(int(self.retry_policy.max_attempts), 1), "next_retry_at": self.retry_policy.next_retry_at(attempt), "last_error": message, "error_type": type(exc).__name__, }, }, }, ctx, ) return WorkerRunResult( worker_id=self.worker_id, status="retry_scheduled", task_id=queued.task_id, processed=1, error_message=message, ) return WorkerRunResult( worker_id=self.worker_id, status="failed", task_id=failed.task_id, processed=1, failed=1, error_message=failed.error_message, ) def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: if value is None: return WorkerTaskUpdate() if isinstance(value, WorkerTaskUpdate): return value return WorkerTaskUpdate( status=str(value.get("status") or "succeeded"), progress=_progress(value.get("progress", 100)), result=dict(value.get("result") or {}), metadata=dict(value.get("metadata") or {}), error_message=_optional_text(value.get("error_message")), ) def _now() -> str: return datetime.now(timezone.utc).isoformat() def _retry_deferred(task: GenerationTask) -> bool: retry = task.metadata.get("retry") if not isinstance(retry, dict): return False next_retry_at = _optional_text(retry.get("next_retry_at")) if not next_retry_at: return False try: parsed = datetime.fromisoformat(next_retry_at) except ValueError: return False if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed > datetime.now(timezone.utc) def _progress(value: Any) -> int: return min(max(_int(value, 100), 0), 100) def _int(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _safe_error_message(exc: Exception) -> str: text = " ".join(str(exc or type(exc).__name__).split()) return text[:500] if text else type(exc).__name__