astrbot_plugin_imagestudio/backend/services/worker_service.py

565 行
21 KiB
Python

2026-07-06 02:19:07 +08:00
"""Persistent queue worker primitives."""
from __future__ import annotations
import logging
2026-07-06 02:19:07 +08:00
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
2026-07-06 02:27:07 +08:00
from datetime import datetime, timedelta, timezone
2026-07-06 02:19:07 +08:00
from typing import Any
from backend.dto.common import ApiError
2026-07-06 02:19:07 +08:00
from backend.dto.job_dto import GenerationTask
from backend.services.queue_service import QueueService
from core.identity import RequestContext
from core.text_utils import optional_text as _optional_text
2026-07-06 02:19:07 +08:00
logger = logging.getLogger(__name__)
2026-07-06 02:19:07 +08:00
WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"]
WorkerTaskFilter = Callable[[GenerationTask], bool]
WorkerTerminalCallback = Callable[[GenerationTask, GenerationTask, Exception | None], None]
2026-07-06 02:19:07 +08:00
@dataclass(frozen=True)
class WorkerTaskUpdate:
status: str = "succeeded"
progress: int = 100
result: dict[str, Any] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
error_message: str | None = None
2026-07-06 02:27:07 +08:00
@dataclass(frozen=True)
class RetryPolicy:
max_attempts: int = 1
base_delay_seconds: float = 0.0
backoff_multiplier: float = 2.0
retryable_error_types: tuple[str, ...] = ()
def should_retry(self, attempt: int, exc: Exception) -> bool:
if attempt >= max(int(self.max_attempts), 1):
return False
if not self.retryable_error_types:
return True
return type(exc).__name__ in self.retryable_error_types
def next_retry_at(self, attempt: int) -> str:
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
2026-07-06 02:19:07 +08:00
@dataclass(frozen=True)
class WorkerRunResult:
worker_id: str
status: str
task_id: str | None = None
processed: int = 0
succeeded: int = 0
failed: int = 0
error_message: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"worker_id": self.worker_id,
"status": self.status,
"task_id": self.task_id,
"processed": self.processed,
"succeeded": self.succeeded,
"failed": self.failed,
"error_message": self.error_message,
}
class QueueWorker:
def __init__(
self,
queue_service: QueueService,
handler: WorkerHandler,
*,
worker_id: str = "imagestudio-worker",
interval_seconds: float = 1.0,
claim_batch_size: int = 5,
2026-07-06 02:27:07 +08:00
retry_policy: RetryPolicy | None = None,
task_filter: WorkerTaskFilter | None = None,
terminal_callback: WorkerTerminalCallback | None = None,
2026-07-06 02:19:07 +08:00
):
self.queue_service = queue_service
self.handler = handler
self.worker_id = worker_id
self.interval_seconds = max(float(interval_seconds), 0.01)
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
2026-07-06 02:27:07 +08:00
self.retry_policy = retry_policy or RetryPolicy()
self.task_filter = task_filter
self.terminal_callback = terminal_callback
2026-07-06 02:19:07 +08:00
self._stop_event = threading.Event()
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
def run_once(self, ctx: RequestContext) -> WorkerRunResult:
task = self._claim_next(ctx)
if task is None:
return WorkerRunResult(worker_id=self.worker_id, status="idle")
try:
update = _normalize_update(self.handler(task))
2026-07-06 02:23:05 +08:00
latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled":
self._notify_terminal(task, latest, None)
2026-07-06 02:23:05 +08:00
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
# Compare-and-set on "running" rather than an unconditional UPDATE.
# The read above is a TOCTOU window: a cancel landing between
# get_job() and the write was silently overwritten, resurrecting a
# task the user had cancelled (and still billing them for it).
completed = self.queue_service.update_task_if_status(
2026-07-06 02:19:07 +08:00
task.task_id,
{"running"},
2026-07-06 02:19:07 +08:00
{
"status": update.status,
"progress": _progress(update.progress),
"result": {**task.result, **update.result},
"error_message": update.error_message,
"metadata": {
**task.metadata,
**update.metadata,
"worker": {
"worker_id": self.worker_id,
"status": update.status,
"finished_at": _now(),
},
},
},
ctx,
require_workflow_not_cancelled=True,
2026-07-06 02:19:07 +08:00
)
if completed is None:
# Someone else owns the task now (cancelled, or reclaimed by a
# stale-task sweep). Do not resurrect it.
latest = self.queue_service.get_job(task.task_id)
self._notify_terminal(task, latest, None)
return WorkerRunResult(
worker_id=self.worker_id,
status=latest.status,
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
self._notify_terminal(task, completed, None)
2026-07-06 02:19:07 +08:00
return WorkerRunResult(
worker_id=self.worker_id,
status=completed.status,
task_id=completed.task_id,
processed=1,
succeeded=1 if completed.status in {"completed", "succeeded"} else 0,
failed=1 if completed.status == "failed" else 0,
error_message=completed.error_message,
)
except Exception as exc:
2026-07-06 02:27:07 +08:00
return self._handle_failure(task, exc, ctx)
2026-07-06 02:19:07 +08:00
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
limit = min(max(int(max_tasks), 1), 100)
results: list[dict[str, Any]] = []
for _ in range(limit):
result = self.run_once(ctx)
results.append(result.to_dict())
if result.status == "idle":
break
return {
"worker_id": self.worker_id,
"processed": sum(item["processed"] for item in results),
"succeeded": sum(item["succeeded"] for item in results),
"failed": sum(item["failed"] for item in results),
"results": results,
}
def start(self, ctx: RequestContext) -> dict[str, Any]:
with self._lock:
if self.is_running():
return {"worker_id": self.worker_id, "running": True, "already_running": True}
# A previous thread may have died without clearing state. Always
# rebuild a fresh stop event + thread so callers can revive workers.
self._stop_event = threading.Event()
2026-07-06 02:19:07 +08:00
self._thread = threading.Thread(
target=self._run_loop,
args=(ctx,),
daemon=True,
name=f"ImageStudioQueueWorker-{self.worker_id}",
)
self._thread.start()
return {"worker_id": self.worker_id, "running": True, "already_running": False}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
self._stop_event.set()
thread = self._thread
if thread is not None:
thread.join(timeout=max(float(timeout_seconds), 0.1))
return {"worker_id": self.worker_id, "running": self.is_running()}
def is_running(self) -> bool:
thread = self._thread
return bool(thread and thread.is_alive())
def _run_loop(self, ctx: RequestContext) -> None:
while not self._stop_event.is_set():
try:
result = self.run_once(ctx)
except Exception:
# Provider/business failures must never kill the worker thread.
# A dead thread permanently stalls claim of later queued tasks.
logger.exception(
"ImageStudio queue worker %s crashed while processing; continuing",
self.worker_id,
)
self._stop_event.wait(self.interval_seconds)
continue
2026-07-06 02:19:07 +08:00
if result.status == "idle":
self._stop_event.wait(self.interval_seconds)
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
# FIFO claim. Using the display listing here made the queue LIFO, so a
# backlog deeper than claim_batch_size starved its oldest entries
# indefinitely while newer tasks kept jumping ahead.
queued = self.queue_service.list_claimable_jobs(
status="queued", limit=self.claim_batch_size
)
2026-07-06 02:19:07 +08:00
for candidate in queued:
2026-07-06 02:27:07 +08:00
if _retry_deferred(candidate):
continue
if self.task_filter is not None and not self.task_filter(candidate):
continue
claimed = self.queue_service.claim_queued_task(
candidate.task_id,
ctx,
worker_id=self.worker_id,
)
if claimed is not None:
return claimed
2026-07-06 02:19:07 +08:00
return None
2026-07-06 02:27:07 +08:00
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
"""Mark the claimed task failed without ever re-raising.
Queue progress is more important than perfect failure bookkeeping. Any
secondary error while recording failure must leave the worker thread
alive so later queued work can still be claimed.
"""
2026-07-06 02:19:07 +08:00
message = _safe_error_message(exc)
latest = task
try:
latest = self.queue_service.get_job(task.task_id)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not reload task %s during failure handling",
self.worker_id,
task.task_id,
)
latest = task
try:
if latest.status == "cancelled":
self._notify_terminal(task, latest, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s failed while handling cancelled task %s",
self.worker_id,
task.task_id,
)
attempt = _int(
(latest.metadata or {}).get("worker_attempt"),
_int((task.metadata or {}).get("worker_attempt"), 1),
2026-07-06 02:19:07 +08:00
)
base_metadata = dict(task.metadata or {})
try:
if isinstance(latest.metadata, dict):
base_metadata = {**base_metadata, **latest.metadata}
except Exception:
base_metadata = dict(task.metadata or {})
failed = latest
try:
failed = self.queue_service.update_task(
task.task_id,
2026-07-06 02:27:07 +08:00
{
"status": "failed",
"error_message": message,
2026-07-06 02:27:07 +08:00
"metadata": {
**base_metadata,
"worker": {
"worker_id": self.worker_id,
"status": "failed",
2026-07-06 02:27:07 +08:00
"error_type": type(exc).__name__,
"failed_at": _now(),
2026-07-06 02:27:07 +08:00
},
},
},
ctx,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not mark task %s as failed",
self.worker_id,
task.task_id,
)
self._notify_terminal(task, latest, exc)
2026-07-06 02:27:07 +08:00
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=task.task_id,
2026-07-06 02:27:07 +08:00
processed=1,
failed=1,
error_message=message,
)
try:
if self.retry_policy.should_retry(attempt, exc):
try:
queued = self.queue_service.update_task(
failed.task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**(failed.metadata or {}),
"retry": {
"scheduled": True,
"attempt": attempt,
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
"next_retry_at": self.retry_policy.next_retry_at(attempt),
"last_error": message,
"error_type": type(exc).__name__,
},
},
},
ctx,
)
self._notify_terminal(task, queued, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="retry_scheduled",
task_id=queued.task_id,
processed=1,
error_message=message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s could not requeue task %s after failure",
self.worker_id,
task.task_id,
)
self._notify_terminal(task, failed, exc)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=failed.task_id,
processed=1,
failed=1,
error_message=failed.error_message or message,
)
except Exception:
logger.exception(
"ImageStudio queue worker %s hit a secondary error after failing task %s",
self.worker_id,
task.task_id,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=task.task_id,
processed=1,
failed=1,
2026-07-06 02:27:07 +08:00
error_message=message,
)
2026-07-06 02:19:07 +08:00
def _notify_terminal(
self,
claimed_task: GenerationTask,
current_task: GenerationTask,
exc: Exception | None,
) -> None:
if self.terminal_callback is None:
return
try:
self.terminal_callback(claimed_task, current_task, exc)
except Exception:
# Queue state is authoritative. A coordinator callback may be
# replayed from persisted task state after restart, so it must not
# turn a successfully completed provider request into a failure.
return
class QueueWorkerPool:
"""Bounded parallel workers sharing one persistent queue."""
def __init__(
self,
queue_service: QueueService,
handler: WorkerHandler,
*,
worker_count: int = 5,
worker_id_prefix: str = "imagestudio-round",
interval_seconds: float = 0.25,
claim_batch_size: int = 25,
retry_policy: RetryPolicy | None = None,
task_filter: WorkerTaskFilter | None = None,
terminal_callback: WorkerTerminalCallback | None = None,
per_route_limit: int | None = None,
):
count = min(max(int(worker_count), 1), 16)
bounded_handler = _BoundedWorkerHandler(
handler,
global_limit=count,
per_route_limit=min(max(int(per_route_limit or count), 1), count),
)
self.workers = [
QueueWorker(
queue_service,
bounded_handler,
worker_id=f"{worker_id_prefix}-{index + 1}",
interval_seconds=interval_seconds,
claim_batch_size=claim_batch_size,
retry_policy=retry_policy,
task_filter=task_filter,
terminal_callback=terminal_callback,
)
for index in range(count)
]
def start(self, ctx: RequestContext) -> dict[str, Any]:
results = [worker.start(ctx) for worker in self.workers]
return {
"running": any(item["running"] for item in results),
"worker_count": len(self.workers),
"workers": results,
}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
results = [
worker.stop(timeout_seconds=timeout_seconds)
for worker in reversed(self.workers)
]
return {
"running": any(item["running"] for item in results),
"worker_count": len(self.workers),
"workers": list(reversed(results)),
}
def is_running(self) -> bool:
return any(worker.is_running() for worker in self.workers)
class _BoundedWorkerHandler:
def __init__(self, handler: WorkerHandler, *, global_limit: int, per_route_limit: int):
self.handler = handler
self.global_semaphore = threading.BoundedSemaphore(global_limit)
self.per_route_limit = per_route_limit
self._route_guard = threading.Lock()
self._route_semaphores: dict[str, threading.BoundedSemaphore] = {}
def __call__(self, task: GenerationTask) -> WorkerTaskUpdate | dict[str, Any] | None:
route_key = _task_route_key(task)
with self._route_guard:
route_semaphore = self._route_semaphores.setdefault(
route_key,
threading.BoundedSemaphore(self.per_route_limit),
)
with self.global_semaphore, route_semaphore:
return self.handler(task)
2026-07-06 02:19:07 +08:00
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
if value is None:
return WorkerTaskUpdate()
if isinstance(value, WorkerTaskUpdate):
return value
return WorkerTaskUpdate(
status=str(value.get("status") or "succeeded"),
progress=_progress(value.get("progress", 100)),
result=dict(value.get("result") or {}),
metadata=dict(value.get("metadata") or {}),
error_message=_optional_text(value.get("error_message")),
)
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
2026-07-06 02:27:07 +08:00
def _retry_deferred(task: GenerationTask) -> bool:
retry = task.metadata.get("retry")
if not isinstance(retry, dict):
return False
next_retry_at = _optional_text(retry.get("next_retry_at"))
if not next_retry_at:
return False
try:
parsed = datetime.fromisoformat(next_retry_at)
except ValueError:
return False
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed > datetime.now(timezone.utc)
2026-07-06 02:19:07 +08:00
def _progress(value: Any) -> int:
return min(max(_int(value, 100), 0), 100)
def _int(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _safe_error_message(exc: Exception) -> str:
if isinstance(exc, ApiError):
code = exc.error_code.value if hasattr(exc.error_code, "value") else str(exc.error_code)
parts = [str(exc.message or "").strip(), f"[{code}]"]
if isinstance(exc.details, dict) and exc.details:
# Keep a short, single-line hint without dumping full provider payloads.
hint_keys = ("reason", "finish_reason", "provider", "model")
hints = []
for key in hint_keys:
value = exc.details.get(key)
if value is None:
continue
text_value = " ".join(str(value).split())
if text_value:
hints.append(f"{key}={text_value[:80]}")
if hints:
parts.append("(" + ", ".join(hints) + ")")
text = " ".join(part for part in parts if part)
return text[:500] if text else f"ApiError [{code}]"
2026-07-06 02:19:07 +08:00
text = " ".join(str(exc or type(exc).__name__).split())
return text[:500] if text else type(exc).__name__
def _task_route_key(task: GenerationTask) -> str:
resolution = task.metadata.get("model_route_resolution")
if not isinstance(resolution, dict):
resolution = task.request.get("resolved_model_route")
if not isinstance(resolution, dict):
resolution = {}
provider_id = _optional_text(resolution.get("provider_id")) or "unresolved"
model_id = _optional_text(resolution.get("model_id")) or _optional_text(task.model_route) or "default"
return f"{provider_id}:{model_id}"