"""Persistent queue worker primitives.""" from __future__ import annotations import logging import threading from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any from backend.dto.common import ApiError from backend.dto.job_dto import GenerationTask from backend.services.queue_service import QueueService from core.identity import RequestContext logger = logging.getLogger(__name__) WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"] WorkerTaskFilter = Callable[[GenerationTask], bool] WorkerTerminalCallback = Callable[[GenerationTask, GenerationTask, Exception | None], None] @dataclass(frozen=True) class WorkerTaskUpdate: status: str = "succeeded" progress: int = 100 result: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) error_message: str | None = None @dataclass(frozen=True) class RetryPolicy: max_attempts: int = 1 base_delay_seconds: float = 0.0 backoff_multiplier: float = 2.0 retryable_error_types: tuple[str, ...] = () def should_retry(self, attempt: int, exc: Exception) -> bool: if attempt >= max(int(self.max_attempts), 1): return False if not self.retryable_error_types: return True return type(exc).__name__ in self.retryable_error_types def next_retry_at(self, attempt: int) -> str: delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0)) return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat() @dataclass(frozen=True) class WorkerRunResult: worker_id: str status: str task_id: str | None = None processed: int = 0 succeeded: int = 0 failed: int = 0 error_message: str | None = None def to_dict(self) -> dict[str, Any]: return { "worker_id": self.worker_id, "status": self.status, "task_id": self.task_id, "processed": self.processed, "succeeded": self.succeeded, "failed": self.failed, "error_message": self.error_message, } class QueueWorker: def __init__( self, queue_service: QueueService, handler: WorkerHandler, *, worker_id: str = "imagestudio-worker", interval_seconds: float = 1.0, claim_batch_size: int = 5, retry_policy: RetryPolicy | None = None, task_filter: WorkerTaskFilter | None = None, terminal_callback: WorkerTerminalCallback | None = None, ): self.queue_service = queue_service self.handler = handler self.worker_id = worker_id self.interval_seconds = max(float(interval_seconds), 0.01) self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) self.retry_policy = retry_policy or RetryPolicy() self.task_filter = task_filter self.terminal_callback = terminal_callback self._stop_event = threading.Event() self._lock = threading.Lock() self._thread: threading.Thread | None = None def run_once(self, ctx: RequestContext) -> WorkerRunResult: task = self._claim_next(ctx) if task is None: return WorkerRunResult(worker_id=self.worker_id, status="idle") try: update = _normalize_update(self.handler(task)) latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": self._notify_terminal(task, latest, None) return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) completed = self.queue_service.update_task( task.task_id, { "status": update.status, "progress": _progress(update.progress), "result": {**task.result, **update.result}, "error_message": update.error_message, "metadata": { **task.metadata, **update.metadata, "worker": { "worker_id": self.worker_id, "status": update.status, "finished_at": _now(), }, }, }, ctx, ) self._notify_terminal(task, completed, None) return WorkerRunResult( worker_id=self.worker_id, status=completed.status, task_id=completed.task_id, processed=1, succeeded=1 if completed.status in {"completed", "succeeded"} else 0, failed=1 if completed.status == "failed" else 0, error_message=completed.error_message, ) except Exception as exc: return self._handle_failure(task, exc, ctx) def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: limit = min(max(int(max_tasks), 1), 100) results: list[dict[str, Any]] = [] for _ in range(limit): result = self.run_once(ctx) results.append(result.to_dict()) if result.status == "idle": break return { "worker_id": self.worker_id, "processed": sum(item["processed"] for item in results), "succeeded": sum(item["succeeded"] for item in results), "failed": sum(item["failed"] for item in results), "results": results, } def start(self, ctx: RequestContext) -> dict[str, Any]: with self._lock: if self.is_running(): return {"worker_id": self.worker_id, "running": True, "already_running": True} # A previous thread may have died without clearing state. Always # rebuild a fresh stop event + thread so callers can revive workers. self._stop_event = threading.Event() self._thread = threading.Thread( target=self._run_loop, args=(ctx,), daemon=True, name=f"ImageStudioQueueWorker-{self.worker_id}", ) self._thread.start() return {"worker_id": self.worker_id, "running": True, "already_running": False} def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: self._stop_event.set() thread = self._thread if thread is not None: thread.join(timeout=max(float(timeout_seconds), 0.1)) return {"worker_id": self.worker_id, "running": self.is_running()} def is_running(self) -> bool: thread = self._thread return bool(thread and thread.is_alive()) def _run_loop(self, ctx: RequestContext) -> None: while not self._stop_event.is_set(): try: result = self.run_once(ctx) except Exception: # Provider/business failures must never kill the worker thread. # A dead thread permanently stalls claim of later queued tasks. logger.exception( "ImageStudio queue worker %s crashed while processing; continuing", self.worker_id, ) self._stop_event.wait(self.interval_seconds) continue if result.status == "idle": self._stop_event.wait(self.interval_seconds) def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items for candidate in queued: if _retry_deferred(candidate): continue if self.task_filter is not None and not self.task_filter(candidate): continue claimed = self.queue_service.claim_queued_task( candidate.task_id, ctx, worker_id=self.worker_id, ) if claimed is not None: return claimed return None def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult: """Mark the claimed task failed without ever re-raising. Queue progress is more important than perfect failure bookkeeping. Any secondary error while recording failure must leave the worker thread alive so later queued work can still be claimed. """ message = _safe_error_message(exc) latest = task try: latest = self.queue_service.get_job(task.task_id) except Exception: logger.exception( "ImageStudio queue worker %s could not reload task %s during failure handling", self.worker_id, task.task_id, ) latest = task try: if latest.status == "cancelled": self._notify_terminal(task, latest, exc) return WorkerRunResult( worker_id=self.worker_id, status="cancelled", task_id=latest.task_id, processed=1, error_message=latest.error_message, ) except Exception: logger.exception( "ImageStudio queue worker %s failed while handling cancelled task %s", self.worker_id, task.task_id, ) attempt = _int( (latest.metadata or {}).get("worker_attempt"), _int((task.metadata or {}).get("worker_attempt"), 1), ) base_metadata = dict(task.metadata or {}) try: if isinstance(latest.metadata, dict): base_metadata = {**base_metadata, **latest.metadata} except Exception: base_metadata = dict(task.metadata or {}) failed = latest try: failed = self.queue_service.update_task( task.task_id, { "status": "failed", "error_message": message, "metadata": { **base_metadata, "worker": { "worker_id": self.worker_id, "status": "failed", "error_type": type(exc).__name__, "failed_at": _now(), }, }, }, ctx, ) except Exception: logger.exception( "ImageStudio queue worker %s could not mark task %s as failed", self.worker_id, task.task_id, ) self._notify_terminal(task, latest, exc) return WorkerRunResult( worker_id=self.worker_id, status="failed", task_id=task.task_id, processed=1, failed=1, error_message=message, ) try: if self.retry_policy.should_retry(attempt, exc): try: queued = self.queue_service.update_task( failed.task_id, { "status": "queued", "progress": 0, "error_message": None, "metadata": { **(failed.metadata or {}), "retry": { "scheduled": True, "attempt": attempt, "max_attempts": max(int(self.retry_policy.max_attempts), 1), "next_retry_at": self.retry_policy.next_retry_at(attempt), "last_error": message, "error_type": type(exc).__name__, }, }, }, ctx, ) self._notify_terminal(task, queued, exc) return WorkerRunResult( worker_id=self.worker_id, status="retry_scheduled", task_id=queued.task_id, processed=1, error_message=message, ) except Exception: logger.exception( "ImageStudio queue worker %s could not requeue task %s after failure", self.worker_id, task.task_id, ) self._notify_terminal(task, failed, exc) return WorkerRunResult( worker_id=self.worker_id, status="failed", task_id=failed.task_id, processed=1, failed=1, error_message=failed.error_message or message, ) except Exception: logger.exception( "ImageStudio queue worker %s hit a secondary error after failing task %s", self.worker_id, task.task_id, ) return WorkerRunResult( worker_id=self.worker_id, status="failed", task_id=task.task_id, processed=1, failed=1, error_message=message, ) def _notify_terminal( self, claimed_task: GenerationTask, current_task: GenerationTask, exc: Exception | None, ) -> None: if self.terminal_callback is None: return try: self.terminal_callback(claimed_task, current_task, exc) except Exception: # Queue state is authoritative. A coordinator callback may be # replayed from persisted task state after restart, so it must not # turn a successfully completed provider request into a failure. return class QueueWorkerPool: """Bounded parallel workers sharing one persistent queue.""" def __init__( self, queue_service: QueueService, handler: WorkerHandler, *, worker_count: int = 5, worker_id_prefix: str = "imagestudio-round", interval_seconds: float = 0.25, claim_batch_size: int = 25, retry_policy: RetryPolicy | None = None, task_filter: WorkerTaskFilter | None = None, terminal_callback: WorkerTerminalCallback | None = None, per_route_limit: int | None = None, ): count = min(max(int(worker_count), 1), 16) bounded_handler = _BoundedWorkerHandler( handler, global_limit=count, per_route_limit=min(max(int(per_route_limit or count), 1), count), ) self.workers = [ QueueWorker( queue_service, bounded_handler, worker_id=f"{worker_id_prefix}-{index + 1}", interval_seconds=interval_seconds, claim_batch_size=claim_batch_size, retry_policy=retry_policy, task_filter=task_filter, terminal_callback=terminal_callback, ) for index in range(count) ] def start(self, ctx: RequestContext) -> dict[str, Any]: results = [worker.start(ctx) for worker in self.workers] return { "running": any(item["running"] for item in results), "worker_count": len(self.workers), "workers": results, } def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: results = [ worker.stop(timeout_seconds=timeout_seconds) for worker in reversed(self.workers) ] return { "running": any(item["running"] for item in results), "worker_count": len(self.workers), "workers": list(reversed(results)), } def is_running(self) -> bool: return any(worker.is_running() for worker in self.workers) class _BoundedWorkerHandler: def __init__(self, handler: WorkerHandler, *, global_limit: int, per_route_limit: int): self.handler = handler self.global_semaphore = threading.BoundedSemaphore(global_limit) self.per_route_limit = per_route_limit self._route_guard = threading.Lock() self._route_semaphores: dict[str, threading.BoundedSemaphore] = {} def __call__(self, task: GenerationTask) -> WorkerTaskUpdate | dict[str, Any] | None: route_key = _task_route_key(task) with self._route_guard: route_semaphore = self._route_semaphores.setdefault( route_key, threading.BoundedSemaphore(self.per_route_limit), ) with self.global_semaphore, route_semaphore: return self.handler(task) def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: if value is None: return WorkerTaskUpdate() if isinstance(value, WorkerTaskUpdate): return value return WorkerTaskUpdate( status=str(value.get("status") or "succeeded"), progress=_progress(value.get("progress", 100)), result=dict(value.get("result") or {}), metadata=dict(value.get("metadata") or {}), error_message=_optional_text(value.get("error_message")), ) def _now() -> str: return datetime.now(timezone.utc).isoformat() def _retry_deferred(task: GenerationTask) -> bool: retry = task.metadata.get("retry") if not isinstance(retry, dict): return False next_retry_at = _optional_text(retry.get("next_retry_at")) if not next_retry_at: return False try: parsed = datetime.fromisoformat(next_retry_at) except ValueError: return False if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed > datetime.now(timezone.utc) def _progress(value: Any) -> int: return min(max(_int(value, 100), 0), 100) def _int(value: Any, default: int) -> int: try: return int(value) except (TypeError, ValueError): return default def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _safe_error_message(exc: Exception) -> str: if isinstance(exc, ApiError): code = exc.error_code.value if hasattr(exc.error_code, "value") else str(exc.error_code) parts = [str(exc.message or "").strip(), f"[{code}]"] if isinstance(exc.details, dict) and exc.details: # Keep a short, single-line hint without dumping full provider payloads. hint_keys = ("reason", "finish_reason", "provider", "model") hints = [] for key in hint_keys: value = exc.details.get(key) if value is None: continue text_value = " ".join(str(value).split()) if text_value: hints.append(f"{key}={text_value[:80]}") if hints: parts.append("(" + ", ".join(hints) + ")") text = " ".join(part for part in parts if part) return text[:500] if text else f"ApiError [{code}]" text = " ".join(str(exc or type(exc).__name__).split()) return text[:500] if text else type(exc).__name__ def _task_route_key(task: GenerationTask) -> str: resolution = task.metadata.get("model_route_resolution") if not isinstance(resolution, dict): resolution = task.request.get("resolved_model_route") if not isinstance(resolution, dict): resolution = {} provider_id = _optional_text(resolution.get("provider_id")) or "unresolved" model_id = _optional_text(resolution.get("model_id")) or _optional_text(task.model_route) or "default" return f"{provider_id}:{model_id}"