astrbot_plugin_imagestudio/backend/services/worker_service.py

545 行
20 KiB
Python

"""Persistent queue worker primitives."""
from __future__ import annotations
import logging
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Any
from backend.dto.common import ApiError
from backend.dto.job_dto import GenerationTask
from backend.services.queue_service import QueueService
from core.identity import RequestContext
logger = logging.getLogger(__name__)
WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"]
WorkerTaskFilter = Callable[[GenerationTask], bool]
WorkerTerminalCallback = Callable[[GenerationTask, GenerationTask, Exception | None], None]
@dataclass(frozen=True)
class WorkerTaskUpdate:
status: str = "succeeded"
progress: int = 100
result: dict[str, Any] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
error_message: str | None = None
@dataclass(frozen=True)
class RetryPolicy:
max_attempts: int = 1
base_delay_seconds: float = 0.0
backoff_multiplier: float = 2.0
retryable_error_types: tuple[str, ...] = ()
def should_retry(self, attempt: int, exc: Exception) -> bool:
if attempt >= max(int(self.max_attempts), 1):
return False
if not self.retryable_error_types:
return True
return type(exc).__name__ in self.retryable_error_types
def next_retry_at(self, attempt: int) -> str:
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
@dataclass(frozen=True)
class WorkerRunResult:
worker_id: str
status: str
task_id: str | None = None
processed: int = 0
succeeded: int = 0
failed: int = 0
error_message: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"worker_id": self.worker_id,
"status": self.status,
"task_id": self.task_id,
"processed": self.processed,
"succeeded": self.succeeded,
"failed": self.failed,
"error_message": self.error_message,
}
class QueueWorker:
def __init__(
self,
queue_service: QueueService,
handler: WorkerHandler,
*,
worker_id: str = "imagestudio-worker",
interval_seconds: float = 1.0,
claim_batch_size: int = 5,
retry_policy: RetryPolicy | None = None,
task_filter: WorkerTaskFilter | None = None,
terminal_callback: WorkerTerminalCallback | None = None,
):
self.queue_service = queue_service
self.handler = handler
self.worker_id = worker_id
self.interval_seconds = max(float(interval_seconds), 0.01)
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
self.retry_policy = retry_policy or RetryPolicy()
self.task_filter = task_filter
self.terminal_callback = terminal_callback
self._stop_event = threading.Event()
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
def run_once(self, ctx: RequestContext) -> WorkerRunResult:
task = self._claim_next(ctx)
if task is None:
return WorkerRunResult(worker_id=self.worker_id, status="idle")
try:
update = _normalize_update(self.handler(task))
latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled":
self._notify_terminal(task, latest, None)
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
completed = self.queue_service.update_task(
task.task_id,
{
"status": update.status,
"progress": _progress(update.progress),
"result": {**task.result, **update.result},
"error_message": update.error_message,
"metadata": {
**task.metadata,
**update.metadata,
"worker": {
"worker_id": self.worker_id,
"status": update.status,
"finished_at": _now(),
},
},
},
ctx,
)
self._notify_terminal(task, completed, None)
return WorkerRunResult(
worker_id=self.worker_id,
status=completed.status,
task_id=completed.task_id,
processed=1,
succeeded=1 if completed.status in {"completed", "succeeded"} else 0,
failed=1 if completed.status == "failed" else 0,
error_message=completed.error_message,
)
except Exception as exc:
return self._handle_failure(task, exc, ctx)
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
limit = min(max(int(max_tasks), 1), 100)
results: list[dict[str, Any]] = []
for _ in range(limit):
result = self.run_once(ctx)
results.append(result.to_dict())
if result.status == "idle":
break
return {
"worker_id": self.worker_id,
"processed": sum(item["processed"] for item in results),
"succeeded": sum(item["succeeded"] for item in results),
"failed": sum(item["failed"] for item in results),
"results": results,
}
def start(self, ctx: RequestContext) -> dict[str, Any]:
with self._lock:
if self.is_running():
return {"worker_id": self.worker_id, "running": True, "already_running": True}
# A previous thread may have died without clearing state. Always
# rebuild a fresh stop event + thread so callers can revive workers.
self._stop_event = threading.Event()
self._thread = threading.Thread(
target=self._run_loop,
args=(ctx,),
daemon=True,
name=f"ImageStudioQueueWorker-{self.worker_id}",
)
self._thread.start()
return {"worker_id": self.worker_id, "running": True, "already_running": False}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
self._stop_event.set()
thread = self._thread
if thread is not None:
thread.join(timeout=max(float(timeout_seconds), 0.1))
return {"worker_id": self.worker_id, "running": self.is_running()}
def is_running(self) -> bool:
thread = self._thread
return bool(thread and thread.is_alive())
def _run_loop(self, ctx: RequestContext) -> None:
while not self._stop_event.is_set():
try:
result = self.run_once(ctx)
except Exception:
# Provider/business failures must never kill the worker thread.
# A dead thread permanently stalls claim of later queued tasks.
logger.exception(
"ImageStudio queue worker %s crashed while processing; continuing",
self.worker_id,
)
self._stop_event.wait(self.interval_seconds)
continue
if result.status == "idle":
self._stop_event.wait(self.interval_seconds)
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
for candidate in queued:
if _retry_deferred(candidate):
continue
if self.task_filter is not None and not self.task_filter(candidate):
continue
claimed = self.queue_service.claim_queued_task(
candidate.task_id,
ctx,
worker_id=self.worker_id,
)
if claimed is not None:
return claimed
return None
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
"""Mark the claimed task failed without ever re-raising.
Queue progress is more important than perfect failure bookkeeping. Any
secondary error while recording failure must leave the worker thread
alive so later queued work can still be claimed.
"""
message = _safe_error_message(exc)
latest = task
try:
latest = self.queue_service.get_job(task.task_id)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not reload task %s during failure handling",
self.worker_id,
task.task_id,
)
latest = task
try:
if latest.status == "cancelled":
self._notify_terminal(task, latest, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s failed while handling cancelled task %s",
self.worker_id,
task.task_id,
)
attempt = _int(
(latest.metadata or {}).get("worker_attempt"),
_int((task.metadata or {}).get("worker_attempt"), 1),
)
base_metadata = dict(task.metadata or {})
try:
if isinstance(latest.metadata, dict):
base_metadata = {**base_metadata, **latest.metadata}
except Exception:
base_metadata = dict(task.metadata or {})
failed = latest
try:
failed = self.queue_service.update_task(
task.task_id,
{
"status": "failed",
"error_message": message,
"metadata": {
**base_metadata,
"worker": {
"worker_id": self.worker_id,
"status": "failed",
"error_type": type(exc).__name__,
"failed_at": _now(),
},
},
},
ctx,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not mark task %s as failed",
self.worker_id,
task.task_id,
)
self._notify_terminal(task, latest, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=task.task_id,
processed=1,
failed=1,
error_message=message,
)
try:
if self.retry_policy.should_retry(attempt, exc):
try:
queued = self.queue_service.update_task(
failed.task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**(failed.metadata or {}),
"retry": {
"scheduled": True,
"attempt": attempt,
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
"next_retry_at": self.retry_policy.next_retry_at(attempt),
"last_error": message,
"error_type": type(exc).__name__,
},
},
},
ctx,
)
self._notify_terminal(task, queued, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="retry_scheduled",
task_id=queued.task_id,
processed=1,
error_message=message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not requeue task %s after failure",
self.worker_id,
task.task_id,
)
self._notify_terminal(task, failed, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=failed.task_id,
processed=1,
failed=1,
error_message=failed.error_message or message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s hit a secondary error after failing task %s",
self.worker_id,
task.task_id,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=task.task_id,
processed=1,
failed=1,
error_message=message,
)
def _notify_terminal(
self,
claimed_task: GenerationTask,
current_task: GenerationTask,
exc: Exception | None,
) -> None:
if self.terminal_callback is None:
return
try:
self.terminal_callback(claimed_task, current_task, exc)
except Exception:
# Queue state is authoritative. A coordinator callback may be
# replayed from persisted task state after restart, so it must not
# turn a successfully completed provider request into a failure.
return
class QueueWorkerPool:
"""Bounded parallel workers sharing one persistent queue."""
def __init__(
self,
queue_service: QueueService,
handler: WorkerHandler,
*,
worker_count: int = 5,
worker_id_prefix: str = "imagestudio-round",
interval_seconds: float = 0.25,
claim_batch_size: int = 25,
retry_policy: RetryPolicy | None = None,
task_filter: WorkerTaskFilter | None = None,
terminal_callback: WorkerTerminalCallback | None = None,
per_route_limit: int | None = None,
):
count = min(max(int(worker_count), 1), 16)
bounded_handler = _BoundedWorkerHandler(
handler,
global_limit=count,
per_route_limit=min(max(int(per_route_limit or count), 1), count),
)
self.workers = [
QueueWorker(
queue_service,
bounded_handler,
worker_id=f"{worker_id_prefix}-{index + 1}",
interval_seconds=interval_seconds,
claim_batch_size=claim_batch_size,
retry_policy=retry_policy,
task_filter=task_filter,
terminal_callback=terminal_callback,
)
for index in range(count)
]
def start(self, ctx: RequestContext) -> dict[str, Any]:
results = [worker.start(ctx) for worker in self.workers]
return {
"running": any(item["running"] for item in results),
"worker_count": len(self.workers),
"workers": results,
}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
results = [
worker.stop(timeout_seconds=timeout_seconds)
for worker in reversed(self.workers)
]
return {
"running": any(item["running"] for item in results),
"worker_count": len(self.workers),
"workers": list(reversed(results)),
}
def is_running(self) -> bool:
return any(worker.is_running() for worker in self.workers)
class _BoundedWorkerHandler:
def __init__(self, handler: WorkerHandler, *, global_limit: int, per_route_limit: int):
self.handler = handler
self.global_semaphore = threading.BoundedSemaphore(global_limit)
self.per_route_limit = per_route_limit
self._route_guard = threading.Lock()
self._route_semaphores: dict[str, threading.BoundedSemaphore] = {}
def __call__(self, task: GenerationTask) -> WorkerTaskUpdate | dict[str, Any] | None:
route_key = _task_route_key(task)
with self._route_guard:
route_semaphore = self._route_semaphores.setdefault(
route_key,
threading.BoundedSemaphore(self.per_route_limit),
)
with self.global_semaphore, route_semaphore:
return self.handler(task)
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
if value is None:
return WorkerTaskUpdate()
if isinstance(value, WorkerTaskUpdate):
return value
return WorkerTaskUpdate(
status=str(value.get("status") or "succeeded"),
progress=_progress(value.get("progress", 100)),
result=dict(value.get("result") or {}),
metadata=dict(value.get("metadata") or {}),
error_message=_optional_text(value.get("error_message")),
)
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _retry_deferred(task: GenerationTask) -> bool:
retry = task.metadata.get("retry")
if not isinstance(retry, dict):
return False
next_retry_at = _optional_text(retry.get("next_retry_at"))
if not next_retry_at:
return False
try:
parsed = datetime.fromisoformat(next_retry_at)
except ValueError:
return False
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed > datetime.now(timezone.utc)
def _progress(value: Any) -> int:
return min(max(_int(value, 100), 0), 100)
def _int(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _optional_text(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _safe_error_message(exc: Exception) -> str:
if isinstance(exc, ApiError):
code = exc.error_code.value if hasattr(exc.error_code, "value") else str(exc.error_code)
parts = [str(exc.message or "").strip(), f"[{code}]"]
if isinstance(exc.details, dict) and exc.details:
# Keep a short, single-line hint without dumping full provider payloads.
hint_keys = ("reason", "finish_reason", "provider", "model")
hints = []
for key in hint_keys:
value = exc.details.get(key)
if value is None:
continue
text_value = " ".join(str(value).split())
if text_value:
hints.append(f"{key}={text_value[:80]}")
if hints:
parts.append("(" + ", ".join(hints) + ")")
text = " ".join(part for part in parts if part)
return text[:500] if text else f"ApiError [{code}]"
text = " ".join(str(exc or type(exc).__name__).split())
return text[:500] if text else type(exc).__name__
def _task_route_key(task: GenerationTask) -> str:
resolution = task.metadata.get("model_route_resolution")
if not isinstance(resolution, dict):
resolution = task.request.get("resolved_model_route")
if not isinstance(resolution, dict):
resolution = {}
provider_id = _optional_text(resolution.get("provider_id")) or "unresolved"
model_id = _optional_text(resolution.get("model_id")) or _optional_text(task.model_route) or "default"
return f"{provider_id}:{model_id}"