2026-07-18 23:20:06 +08:00
|
|
|
"""Persistent worker pool for advanced-material recognition and validation."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
from backend.dto.job_dto import GenerationTask
|
|
|
|
|
from backend.services.advanced_material_service import (
|
|
|
|
|
AdvancedMaterialService,
|
|
|
|
|
create_advanced_material_service,
|
|
|
|
|
)
|
|
|
|
|
from backend.services.worker_service import QueueWorkerPool, RetryPolicy, WorkerTaskUpdate
|
|
|
|
|
from core.identity import RequestContext
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class AdvancedMaterialWorkerRuntime:
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
service: AdvancedMaterialService,
|
|
|
|
|
*,
|
|
|
|
|
worker_count: int = 8,
|
|
|
|
|
ctx: RequestContext | None = None,
|
|
|
|
|
):
|
|
|
|
|
self.service = service
|
|
|
|
|
self.ctx = ctx or RequestContext.system("advanced-material-worker")
|
2026-07-20 17:57:02 +08:00
|
|
|
recognition_concurrency = _recognition_concurrency(getattr(service, "config", {}))
|
2026-07-18 23:20:06 +08:00
|
|
|
self.pool = QueueWorkerPool(
|
|
|
|
|
service.queue_service,
|
|
|
|
|
self._execute,
|
|
|
|
|
worker_count=worker_count,
|
|
|
|
|
worker_id_prefix="imagestudio-advanced-material",
|
|
|
|
|
interval_seconds=0.35,
|
|
|
|
|
claim_batch_size=25,
|
|
|
|
|
retry_policy=RetryPolicy(max_attempts=1),
|
|
|
|
|
task_filter=_is_advanced_material_task,
|
|
|
|
|
terminal_callback=self._on_terminal,
|
2026-07-20 17:57:02 +08:00
|
|
|
# Keep worker threads available for queue progress, but limit
|
|
|
|
|
# simultaneous calls to the same vision route. Vision providers
|
|
|
|
|
# and relay services commonly rate-limit image requests more
|
|
|
|
|
# aggressively than ordinary text requests.
|
|
|
|
|
per_route_limit=recognition_concurrency,
|
2026-07-18 23:20:06 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def start(self) -> dict[str, Any]:
|
|
|
|
|
recovery = self._recover_interrupted_tasks()
|
|
|
|
|
result = self.pool.start(self.ctx)
|
|
|
|
|
return {**result, "recovery": recovery}
|
|
|
|
|
|
|
|
|
|
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
|
|
|
|
|
return self.pool.stop(timeout_seconds=timeout_seconds)
|
|
|
|
|
|
|
|
|
|
def _execute(self, task: GenerationTask) -> WorkerTaskUpdate:
|
|
|
|
|
return self.service.execute_task(task, self.ctx)
|
|
|
|
|
|
|
|
|
|
def _on_terminal(
|
|
|
|
|
self,
|
|
|
|
|
claimed_task: GenerationTask,
|
|
|
|
|
current_task: GenerationTask,
|
|
|
|
|
exc: Exception | None,
|
|
|
|
|
) -> None:
|
|
|
|
|
if current_task.status in {"failed", "dead_letter", "cancelled"}:
|
|
|
|
|
self.service.handle_task_failure(
|
|
|
|
|
claimed_task,
|
|
|
|
|
exc or RuntimeError(current_task.error_message or "advanced material task failed"),
|
|
|
|
|
self.ctx,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def _recover_interrupted_tasks(self) -> dict[str, Any]:
|
|
|
|
|
"""Safely recover recognition, but never replay a paid image request.
|
|
|
|
|
|
|
|
|
|
Recognition is read-only and idempotent. A validation request may have
|
|
|
|
|
reached the remote provider before the plugin process stopped, so it is
|
|
|
|
|
marked failed and left for an explicit human retry.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# The pool has not started yet, so every advanced-material task still
|
|
|
|
|
# in running/retrying belongs to the previous process. Do not apply a
|
|
|
|
|
# stale-age grace period: a fast container restart must recover tasks
|
|
|
|
|
# written only milliseconds earlier. Do not cap the snapshot either,
|
|
|
|
|
# otherwise unrelated queue rows can hide advanced-material work.
|
|
|
|
|
candidates = self.service.queue_service.repository.list_stale(
|
|
|
|
|
statuses=("running", "retrying"),
|
|
|
|
|
updated_before=None,
|
|
|
|
|
limit=None,
|
|
|
|
|
)
|
|
|
|
|
requeued: list[str] = []
|
|
|
|
|
failed: list[str] = []
|
|
|
|
|
skipped: list[str] = []
|
|
|
|
|
for task in candidates:
|
|
|
|
|
if not _is_advanced_material_task(task):
|
|
|
|
|
skipped.append(task.task_id)
|
|
|
|
|
continue
|
|
|
|
|
interrupted = self.service.queue_service.update_task_if_status(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{task.status},
|
|
|
|
|
{
|
|
|
|
|
"status": "failed",
|
|
|
|
|
"error_message": "task was interrupted by a previous plugin process",
|
|
|
|
|
"metadata": {
|
|
|
|
|
**task.metadata,
|
|
|
|
|
"recovery": {
|
|
|
|
|
"source": "advanced_material_startup",
|
|
|
|
|
"previous_status": task.status,
|
|
|
|
|
"observed_at": datetime.now(timezone.utc).isoformat(),
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
self.ctx,
|
|
|
|
|
)
|
|
|
|
|
if interrupted is None:
|
|
|
|
|
skipped.append(task.task_id)
|
|
|
|
|
continue
|
|
|
|
|
if task.task_type in {
|
|
|
|
|
"advanced_material_recognition",
|
|
|
|
|
"advanced_style_source_recognition",
|
|
|
|
|
"advanced_style_consensus",
|
|
|
|
|
}:
|
|
|
|
|
queued = self.service.queue_service.update_task_if_status(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{"failed"},
|
|
|
|
|
{
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"error_message": None,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**interrupted.metadata,
|
|
|
|
|
"recovery": {**interrupted.metadata.get("recovery", {}), "status": "requeued"},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
self.ctx,
|
|
|
|
|
)
|
|
|
|
|
if queued is not None:
|
|
|
|
|
requeued.append(task.task_id)
|
|
|
|
|
continue
|
|
|
|
|
self.service.handle_task_failure(
|
|
|
|
|
task,
|
|
|
|
|
RuntimeError("task was interrupted by a previous plugin process"),
|
|
|
|
|
self.ctx,
|
|
|
|
|
)
|
|
|
|
|
failed.append(task.task_id)
|
|
|
|
|
return {"requeued_task_ids": requeued, "failed_task_ids": failed, "skipped_task_ids": skipped}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def create_advanced_material_worker_runtime() -> AdvancedMaterialWorkerRuntime:
|
|
|
|
|
return AdvancedMaterialWorkerRuntime(create_advanced_material_service())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _is_advanced_material_task(task: GenerationTask) -> bool:
|
|
|
|
|
return task.metadata.get("source") == "advanced_material" and task.task_type in {
|
|
|
|
|
"advanced_material_recognition",
|
|
|
|
|
"advanced_material_validation_generation",
|
|
|
|
|
"advanced_style_source_recognition",
|
|
|
|
|
"advanced_style_consensus",
|
|
|
|
|
"advanced_style_validation_generation",
|
|
|
|
|
}
|
2026-07-20 17:57:02 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _recognition_concurrency(config: dict[str, Any] | None) -> int:
|
|
|
|
|
advanced_materials = config.get("advanced_materials") if isinstance(config, dict) else None
|
|
|
|
|
if not isinstance(advanced_materials, dict):
|
|
|
|
|
return 2
|
|
|
|
|
try:
|
|
|
|
|
return min(max(int(advanced_materials.get("recognition_concurrency", 2)), 1), 8)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return 2
|