astrbot_plugin_imagestudio/backend/services/advanced_material_worker_runtime.py

167 行
6.6 KiB
Python

"""Persistent worker pool for advanced-material recognition and validation."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from backend.dto.job_dto import GenerationTask
from backend.services.advanced_material_service import (
AdvancedMaterialService,
create_advanced_material_service,
)
from backend.services.worker_service import QueueWorkerPool, RetryPolicy, WorkerTaskUpdate
from core.identity import RequestContext
class AdvancedMaterialWorkerRuntime:
def __init__(
self,
service: AdvancedMaterialService,
*,
worker_count: int = 8,
ctx: RequestContext | None = None,
):
self.service = service
self.ctx = ctx or RequestContext.system("advanced-material-worker")
recognition_concurrency = _recognition_concurrency(getattr(service, "config", {}))
self.pool = QueueWorkerPool(
service.queue_service,
self._execute,
worker_count=worker_count,
worker_id_prefix="imagestudio-advanced-material",
interval_seconds=0.35,
claim_batch_size=25,
retry_policy=RetryPolicy(max_attempts=1),
task_filter=_is_advanced_material_task,
terminal_callback=self._on_terminal,
# Keep worker threads available for queue progress, but limit
# simultaneous calls to the same vision route. Vision providers
# and relay services commonly rate-limit image requests more
# aggressively than ordinary text requests.
per_route_limit=recognition_concurrency,
)
def start(self) -> dict[str, Any]:
recovery = self._recover_interrupted_tasks()
result = self.pool.start(self.ctx)
return {**result, "recovery": recovery}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
return self.pool.stop(timeout_seconds=timeout_seconds)
def _execute(self, task: GenerationTask) -> WorkerTaskUpdate:
return self.service.execute_task(task, self.ctx)
def _on_terminal(
self,
claimed_task: GenerationTask,
current_task: GenerationTask,
exc: Exception | None,
) -> None:
if current_task.status in {"failed", "dead_letter", "cancelled"}:
self.service.handle_task_failure(
claimed_task,
exc or RuntimeError(current_task.error_message or "advanced material task failed"),
self.ctx,
)
def _recover_interrupted_tasks(self) -> dict[str, Any]:
"""Safely recover recognition, but never replay a paid image request.
Recognition is read-only and idempotent. A validation request may have
reached the remote provider before the plugin process stopped, so it is
marked failed and left for an explicit human retry.
"""
# The pool has not started yet, so every advanced-material task still
# in running/retrying belongs to the previous process. Do not apply a
# stale-age grace period: a fast container restart must recover tasks
# written only milliseconds earlier. Do not cap the snapshot either,
# otherwise unrelated queue rows can hide advanced-material work.
candidates = self.service.queue_service.repository.list_stale(
statuses=("running", "retrying"),
updated_before=None,
limit=None,
)
requeued: list[str] = []
failed: list[str] = []
skipped: list[str] = []
for task in candidates:
if not _is_advanced_material_task(task):
skipped.append(task.task_id)
continue
interrupted = self.service.queue_service.update_task_if_status(
task.task_id,
{task.status},
{
"status": "failed",
"error_message": "task was interrupted by a previous plugin process",
"metadata": {
**task.metadata,
"recovery": {
"source": "advanced_material_startup",
"previous_status": task.status,
"observed_at": datetime.now(timezone.utc).isoformat(),
},
},
},
self.ctx,
)
if interrupted is None:
skipped.append(task.task_id)
continue
if task.task_type in {
"advanced_material_recognition",
"advanced_style_source_recognition",
"advanced_style_consensus",
}:
queued = self.service.queue_service.update_task_if_status(
task.task_id,
{"failed"},
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**interrupted.metadata,
"recovery": {**interrupted.metadata.get("recovery", {}), "status": "requeued"},
},
},
self.ctx,
)
if queued is not None:
requeued.append(task.task_id)
continue
self.service.handle_task_failure(
task,
RuntimeError("task was interrupted by a previous plugin process"),
self.ctx,
)
failed.append(task.task_id)
return {"requeued_task_ids": requeued, "failed_task_ids": failed, "skipped_task_ids": skipped}
def create_advanced_material_worker_runtime() -> AdvancedMaterialWorkerRuntime:
return AdvancedMaterialWorkerRuntime(create_advanced_material_service())
def _is_advanced_material_task(task: GenerationTask) -> bool:
return task.metadata.get("source") == "advanced_material" and task.task_type in {
"advanced_material_recognition",
"advanced_material_validation_generation",
"advanced_style_source_recognition",
"advanced_style_consensus",
"advanced_style_validation_generation",
}
def _recognition_concurrency(config: dict[str, Any] | None) -> int:
advanced_materials = config.get("advanced_materials") if isinstance(config, dict) else None
if not isinstance(advanced_materials, dict):
return 2
try:
return min(max(int(advanced_materials.get("recognition_concurrency", 2)), 1), 8)
except (TypeError, ValueError):
return 2