"""Persistent queue worker primitives.""" from __future__ import annotations import threading from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any from backend.dto.job_dto import GenerationTask from backend.services.queue_service import QueueService from core.identity import RequestContext WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"] WorkerTaskFilter = Callable[[GenerationTask], bool] WorkerTerminalCallback = Callable[[GenerationTask, GenerationTask, Exception | None], None] @dataclass(frozen=True) class WorkerTaskUpdate: status: str = "succeeded" progress: int = 100 result: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) error_message: str | None = None @dataclass(frozen=True) class RetryPolicy: max_attempts: int = 1 base_delay_seconds: float = 0.0 backoff_multiplier: float = 2.0 retryable_error_types: tuple[str, ...] = () def should_retry(self, attempt: int, exc: Exception) -> bool: if attempt >= max(int(self.max_attempts), 1): return False if not self.retryable_error_types: return True return type(exc).__name__ in self.retryable_error_types def next_retry_at(self, attempt: int) -> str: delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0)) return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat() @dataclass(frozen=True) class WorkerRunResult: worker_id: str status: str task_id: str | None = None processed: int = 0 succeeded: int = 0 failed: int = 0 error_message: str | None = None def to_dict(self) -> dict[str, Any]: return { "worker_id": self.worker_id, "status": self.status, "task_id": self.task_id, "processed": self.processed, "succeeded": self.succeeded, "failed": self.failed, "error_message": self.error_message, } class QueueWorker: def __init__( self, queue_service: QueueService, handler: WorkerHandler, *, worker_id: str = "imagestudio-worker", interval_seconds: float = 1.0, claim_batch_size: int = 5, retry_policy: RetryPolicy | None = None, task_filter: WorkerTaskFilter | None = None, terminal_callback: WorkerTerminalCallback | None = None, ): self.queue_service = queue_service self.handler = handler self.worker_id = worker_id self.interval_seconds = max(float(interval_seconds), 0.01) self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) self.retry_policy = retry_policy or RetryPolicy() self.task_filter = task_filter self.terminal_callback = terminal_callback self._stop_event = threading.Event() self._lock = threading.Lock() self._thread: threading.Thread | None = None def run_once(self, ctx: RequestContext) -> WorkerRunResult: task = self._claim_next(ctx) if task is None: return WorkerRunResult(worker_id=self.worker_id, status="idle") try: update = _normalize_update(self.handler(task)) latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": self._notify_terminal(task, latest, None) return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) completed = self.queue_service.update_task( task.task_id, { "status": update.status, "progress": _progress(update.progress), "result": {**task.result, **update.result}, "error_message": update.error_message, "metadata": { **task.metadata, **update.metadata, "worker": { "worker_id": self.worker_id, "status": update.status, "finished_at": _now(), }, }, }, ctx, ) self._notify_terminal(task, completed, None) return WorkerRunResult( worker_id=self.worker_id, status=completed.status, task_id=completed.task_id, processed=1, succeeded=1 if completed.status in {"completed", "succeeded"} else 0, failed=1 if completed.status == "failed" else 0, error_message=completed.error_message, ) except Exception as exc: return self._handle_failure(task, exc, ctx) def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: limit = min(max(int(max_tasks), 1), 100) results: list[dict[str, Any]] = [] for _ in range(limit): result = self.run_once(ctx) results.append(result.to_dict()) if result.status == "idle": break return { "worker_id": self.worker_id, "processed": sum(item["processed"] for item in results), "succeeded": sum(item["succeeded"] for item in results), "failed": sum(item["failed"] for item in results), "results": results, } def start(self, ctx: RequestContext) -> dict[str, Any]: with self._lock: if self.is_running(): return {"worker_id": self.worker_id, "running": True, "already_running": True} self._stop_event.clear() self._thread = threading.Thread( target=self._run_loop, args=(ctx,), daemon=True, name=f"ImageStudioQueueWorker-{self.worker_id}", ) self._thread.start() return {"worker_id": self.worker_id, "running": True, "already_running": False} def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: self._stop_event.set() thread = self._thread if thread is not None: thread.join(timeout=max(float(timeout_seconds), 0.1)) return {"worker_id": self.worker_id, "running": self.is_running()} def is_running(self) -> bool: thread = self._thread return bool(thread and thread.is_alive()) def _run_loop(self, ctx: RequestContext) -> None: while not self._stop_event.is_set(): result = self.run_once(ctx) if result.status == "idle": self._stop_event.wait(self.interval_seconds) def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items for candidate in queued: if _retry_deferred(candidate): continue if self.task_filter is not None and not self.task_filter(candidate): continue claimed = self.queue_service.claim_queued_task( candidate.task_id, ctx, worker_id=self.worker_id, ) if claimed is not None: return claimed return None def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult: latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": self._notify_terminal(task, latest, exc) return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) message = _safe_error_message(exc) attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1)) failed = self.queue_service.update_task( task.task_id, { "status": "failed", "error_message": message, "metadata": { **task.metadata, "worker": { "worker_id": self.worker_id, "status": "failed", "error_type": type(exc).__name__, "failed_at": _now(), }, }, }, ctx, ) if self.retry_policy.should_retry(attempt, exc): queued = self.queue_service.update_task( failed.task_id, { "status": "queued", "progress": 0, "error_message": None, "metadata": { **failed.metadata, "retry": { "scheduled": True, "attempt": attempt, "max_attempts": max(int(self.retry_policy.max_attempts), 1), "next_retry_at": self.retry_policy.next_retry_at(attempt), "last_error": message, "error_type": type(exc).__name__, }, }, }, ctx, ) self._notify_terminal(task, queued, exc) return WorkerRunResult( worker_id=self.worker_id, status="retry_scheduled", task_id=queued.task_id, processed=1, error_message=message, ) self._notify_terminal(task, failed, exc) return WorkerRunResult( worker_id=self.worker_id, status="failed", task_id=failed.task_id, processed=1, failed=1, error_message=failed.error_message, ) def _notify_terminal( self, claimed_task: GenerationTask, current_task: GenerationTask, exc: Exception | None, ) -> None: if self.terminal_callback is None: return try: self.terminal_callback(claimed_task, current_task, exc) except Exception: # Queue state is authoritative. A coordinator callback may be # replayed from persisted task state after restart, so it must not # turn a successfully completed provider request into a failure. return class QueueWorkerPool: """Bounded parallel workers sharing one persistent queue.""" def __init__( self, queue_service: QueueService, handler: WorkerHandler, *, worker_count: int = 5, worker_id_prefix: str = "imagestudio-round", interval_seconds: float = 0.25, claim_batch_size: int = 25, retry_policy: RetryPolicy | None = None, task_filter: WorkerTaskFilter | None = None, terminal_callback: WorkerTerminalCallback | None = None, per_route_limit: int | None = None, ): count = min(max(int(worker_count), 1), 16) bounded_handler = _BoundedWorkerHandler( handler, global_limit=count, per_route_limit=min(max(int(per_route_limit or count), 1), count), ) self.workers = [ QueueWorker( queue_service, bounded_handler, worker_id=f"{worker_id_prefix}-{index + 1}", interval_seconds=interval_seconds, claim_batch_size=claim_batch_size, retry_policy=retry_policy, task_filter=task_filter, terminal_callback=terminal_callback, ) for index in range(count) ] def start(self, ctx: RequestContext) -> dict[str, Any]: results = [worker.start(ctx) for worker in self.workers] return { "running": any(item["running"] for item in results), "worker_count": len(self.workers), "workers": results, } def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: results = [ worker.stop(timeout_seconds=timeout_seconds) for worker in reversed(self.workers) ] return { "running": any(item["running"] for item in results), "worker_count": len(self.workers), "workers": list(reversed(results)), } def is_running(self) -> bool: return any(worker.is_running() for worker in self.workers) class _BoundedWorkerHandler: def __init__(self, handler: WorkerHandler, *, global_limit: int, per_route_limit: int): self.handler = handler self.global_semaphore = threading.BoundedSemaphore(global_limit) self.per_route_limit = per_route_limit self._route_guard = threading.Lock() self._route_semaphores: dict[str, threading.BoundedSemaphore] = {} def __call__(self, task: GenerationTask) -> WorkerTaskUpdate | dict[str, Any] | None: route_key = _task_route_key(task) with self._route_guard: route_semaphore = self._route_semaphores.setdefault( route_key, threading.BoundedSemaphore(self.per_route_limit), ) with self.global_semaphore, route_semaphore: return self.handler(task) def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: if value is None: return WorkerTaskUpdate() if isinstance(value, WorkerTaskUpdate): return value return WorkerTaskUpdate( status=str(value.get("status") or "succeeded"), progress=_progress(value.get("progress", 100)), result=dict(value.get("result") or {}), metadata=dict(value.get("metadata") or {}), error_message=_optional_text(value.get("error_message")), ) def _now() -> str: return datetime.now(timezone.utc).isoformat() def _retry_deferred(task: GenerationTask) -> bool: retry = task.metadata.get("retry") if not isinstance(retry, dict): return False next_retry_at = _optional_text(retry.get("next_retry_at")) if not next_retry_at: return False try: parsed = datetime.fromisoformat(next_retry_at) except ValueError: return False if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed > datetime.now(timezone.utc) def _progress(value: Any) -> int: return min(max(_int(value, 100), 0), 100) def _int(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _safe_error_message(exc: Exception) -> str: text = " ".join(str(exc or type(exc).__name__).split()) return text[:500] if text else type(exc).__name__ def _task_route_key(task: GenerationTask) -> str: resolution = task.metadata.get("model_route_resolution") if not isinstance(resolution, dict): resolution = task.request.get("resolved_model_route") if not isinstance(resolution, dict): resolution = {} provider_id = _optional_text(resolution.get("provider_id")) or "unresolved" model_id = _optional_text(resolution.get("model_id")) or _optional_text(task.model_route) or "default" return f"{provider_id}:{model_id}"