"""Persistent worker pool for advanced-material recognition and validation.""" from __future__ import annotations from datetime import datetime, timezone from typing import Any from backend.dto.job_dto import GenerationTask from backend.services.advanced_material_service import ( AdvancedMaterialService, create_advanced_material_service, ) from backend.services.worker_service import QueueWorkerPool, RetryPolicy, WorkerTaskUpdate from core.identity import RequestContext class AdvancedMaterialWorkerRuntime: def __init__( self, service: AdvancedMaterialService, *, worker_count: int = 8, ctx: RequestContext | None = None, ): self.service = service self.ctx = ctx or RequestContext.system("advanced-material-worker") recognition_concurrency = _recognition_concurrency(getattr(service, "config", {})) self.pool = QueueWorkerPool( service.queue_service, self._execute, worker_count=worker_count, worker_id_prefix="imagestudio-advanced-material", interval_seconds=0.35, claim_batch_size=25, retry_policy=RetryPolicy(max_attempts=1), task_filter=_is_advanced_material_task, terminal_callback=self._on_terminal, # Keep worker threads available for queue progress, but limit # simultaneous calls to the same vision route. Vision providers # and relay services commonly rate-limit image requests more # aggressively than ordinary text requests. per_route_limit=recognition_concurrency, ) def start(self) -> dict[str, Any]: recovery = self._recover_interrupted_tasks() result = self.pool.start(self.ctx) return {**result, "recovery": recovery} def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: return self.pool.stop(timeout_seconds=timeout_seconds) def _execute(self, task: GenerationTask) -> WorkerTaskUpdate: return self.service.execute_task(task, self.ctx) def _on_terminal( self, claimed_task: GenerationTask, current_task: GenerationTask, exc: Exception | None, ) -> None: if current_task.status in {"failed", "dead_letter", "cancelled"}: self.service.handle_task_failure( claimed_task, exc or RuntimeError(current_task.error_message or "advanced material task failed"), self.ctx, ) def _recover_interrupted_tasks(self) -> dict[str, Any]: """Safely recover recognition, but never replay a paid image request. Recognition is read-only and idempotent. A validation request may have reached the remote provider before the plugin process stopped, so it is marked failed and left for an explicit human retry. """ # The pool has not started yet, so every advanced-material task still # in running/retrying belongs to the previous process. Do not apply a # stale-age grace period: a fast container restart must recover tasks # written only milliseconds earlier. Do not cap the snapshot either, # otherwise unrelated queue rows can hide advanced-material work. candidates = self.service.queue_service.repository.list_stale( statuses=("running", "retrying"), updated_before=None, limit=None, ) requeued: list[str] = [] failed: list[str] = [] skipped: list[str] = [] for task in candidates: if not _is_advanced_material_task(task): skipped.append(task.task_id) continue interrupted = self.service.queue_service.update_task_if_status( task.task_id, {task.status}, { "status": "failed", "error_message": "task was interrupted by a previous plugin process", "metadata": { **task.metadata, "recovery": { "source": "advanced_material_startup", "previous_status": task.status, "observed_at": datetime.now(timezone.utc).isoformat(), }, }, }, self.ctx, ) if interrupted is None: skipped.append(task.task_id) continue if task.task_type in { "advanced_material_recognition", "advanced_style_source_recognition", "advanced_style_consensus", }: queued = self.service.queue_service.update_task_if_status( task.task_id, {"failed"}, { "status": "queued", "progress": 0, "error_message": None, "metadata": { **interrupted.metadata, "recovery": {**interrupted.metadata.get("recovery", {}), "status": "requeued"}, }, }, self.ctx, ) if queued is not None: requeued.append(task.task_id) continue self.service.handle_task_failure( task, RuntimeError("task was interrupted by a previous plugin process"), self.ctx, ) failed.append(task.task_id) return {"requeued_task_ids": requeued, "failed_task_ids": failed, "skipped_task_ids": skipped} def create_advanced_material_worker_runtime() -> AdvancedMaterialWorkerRuntime: return AdvancedMaterialWorkerRuntime(create_advanced_material_service()) def _is_advanced_material_task(task: GenerationTask) -> bool: return task.metadata.get("source") == "advanced_material" and task.task_type in { "advanced_material_recognition", "advanced_material_validation_generation", "advanced_style_source_recognition", "advanced_style_consensus", "advanced_style_validation_generation", } def _recognition_concurrency(config: dict[str, Any] | None) -> int: advanced_materials = config.get("advanced_materials") if isinstance(config, dict) else None if not isinstance(advanced_materials, dict): return 2 try: return min(max(int(advanced_materials.get("recognition_concurrency", 2)), 1), 8) except (TypeError, ValueError): return 2