astrbot_plugin_imagestudio/backend/services/worker_service.py

319 行
11 KiB
Python

2026-07-06 02:19:07 +08:00
"""Persistent queue worker primitives."""
from __future__ import annotations
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
2026-07-06 02:27:07 +08:00
from datetime import datetime, timedelta, timezone
2026-07-06 02:19:07 +08:00
from typing import Any
from backend.dto.job_dto import GenerationTask
from backend.services.queue_service import QueueService
from core.identity import RequestContext
WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"]
@dataclass(frozen=True)
class WorkerTaskUpdate:
status: str = "succeeded"
progress: int = 100
result: dict[str, Any] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
error_message: str | None = None
2026-07-06 02:27:07 +08:00
@dataclass(frozen=True)
class RetryPolicy:
max_attempts: int = 1
base_delay_seconds: float = 0.0
backoff_multiplier: float = 2.0
retryable_error_types: tuple[str, ...] = ()
def should_retry(self, attempt: int, exc: Exception) -> bool:
if attempt >= max(int(self.max_attempts), 1):
return False
if not self.retryable_error_types:
return True
return type(exc).__name__ in self.retryable_error_types
def next_retry_at(self, attempt: int) -> str:
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
2026-07-06 02:19:07 +08:00
@dataclass(frozen=True)
class WorkerRunResult:
worker_id: str
status: str
task_id: str | None = None
processed: int = 0
succeeded: int = 0
failed: int = 0
error_message: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"worker_id": self.worker_id,
"status": self.status,
"task_id": self.task_id,
"processed": self.processed,
"succeeded": self.succeeded,
"failed": self.failed,
"error_message": self.error_message,
}
class QueueWorker:
def __init__(
self,
queue_service: QueueService,
handler: WorkerHandler,
*,
worker_id: str = "imagestudio-worker",
interval_seconds: float = 1.0,
claim_batch_size: int = 5,
2026-07-06 02:27:07 +08:00
retry_policy: RetryPolicy | None = None,
2026-07-06 02:19:07 +08:00
):
self.queue_service = queue_service
self.handler = handler
self.worker_id = worker_id
self.interval_seconds = max(float(interval_seconds), 0.01)
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
2026-07-06 02:27:07 +08:00
self.retry_policy = retry_policy or RetryPolicy()
2026-07-06 02:19:07 +08:00
self._stop_event = threading.Event()
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
def run_once(self, ctx: RequestContext) -> WorkerRunResult:
task = self._claim_next(ctx)
if task is None:
return WorkerRunResult(worker_id=self.worker_id, status="idle")
try:
update = _normalize_update(self.handler(task))
2026-07-06 02:23:05 +08:00
latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled":
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
2026-07-06 02:19:07 +08:00
completed = self.queue_service.update_task(
task.task_id,
{
"status": update.status,
"progress": _progress(update.progress),
"result": {**task.result, **update.result},
"error_message": update.error_message,
"metadata": {
**task.metadata,
**update.metadata,
"worker": {
"worker_id": self.worker_id,
"status": update.status,
"finished_at": _now(),
},
},
},
ctx,
)
return WorkerRunResult(
worker_id=self.worker_id,
status=completed.status,
task_id=completed.task_id,
processed=1,
succeeded=1 if completed.status in {"completed", "succeeded"} else 0,
failed=1 if completed.status == "failed" else 0,
error_message=completed.error_message,
)
except Exception as exc:
2026-07-06 02:27:07 +08:00
return self._handle_failure(task, exc, ctx)
2026-07-06 02:19:07 +08:00
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
limit = min(max(int(max_tasks), 1), 100)
results: list[dict[str, Any]] = []
for _ in range(limit):
result = self.run_once(ctx)
results.append(result.to_dict())
if result.status == "idle":
break
return {
"worker_id": self.worker_id,
"processed": sum(item["processed"] for item in results),
"succeeded": sum(item["succeeded"] for item in results),
"failed": sum(item["failed"] for item in results),
"results": results,
}
def start(self, ctx: RequestContext) -> dict[str, Any]:
with self._lock:
if self.is_running():
return {"worker_id": self.worker_id, "running": True, "already_running": True}
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run_loop,
args=(ctx,),
daemon=True,
name=f"ImageStudioQueueWorker-{self.worker_id}",
)
self._thread.start()
return {"worker_id": self.worker_id, "running": True, "already_running": False}
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
self._stop_event.set()
thread = self._thread
if thread is not None:
thread.join(timeout=max(float(timeout_seconds), 0.1))
return {"worker_id": self.worker_id, "running": self.is_running()}
def is_running(self) -> bool:
thread = self._thread
return bool(thread and thread.is_alive())
def _run_loop(self, ctx: RequestContext) -> None:
while not self._stop_event.is_set():
result = self.run_once(ctx)
if result.status == "idle":
self._stop_event.wait(self.interval_seconds)
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
for candidate in queued:
2026-07-06 02:27:07 +08:00
if _retry_deferred(candidate):
continue
claimed = self.queue_service.claim_queued_task(
candidate.task_id,
ctx,
worker_id=self.worker_id,
)
if claimed is not None:
return claimed
2026-07-06 02:19:07 +08:00
return None
2026-07-06 02:27:07 +08:00
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
2026-07-06 02:23:05 +08:00
latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled":
2026-07-06 02:27:07 +08:00
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
2026-07-06 02:19:07 +08:00
message = _safe_error_message(exc)
2026-07-06 02:27:07 +08:00
attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1))
failed = self.queue_service.update_task(
2026-07-06 02:19:07 +08:00
task.task_id,
{
"status": "failed",
"error_message": message,
"metadata": {
**task.metadata,
"worker": {
"worker_id": self.worker_id,
"status": "failed",
"error_type": type(exc).__name__,
"failed_at": _now(),
},
},
},
ctx,
)
2026-07-06 02:27:07 +08:00
if self.retry_policy.should_retry(attempt, exc):
queued = self.queue_service.update_task(
failed.task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**failed.metadata,
"retry": {
"scheduled": True,
"attempt": attempt,
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
"next_retry_at": self.retry_policy.next_retry_at(attempt),
"last_error": message,
"error_type": type(exc).__name__,
},
},
},
ctx,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="retry_scheduled",
task_id=queued.task_id,
processed=1,
error_message=message,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=failed.task_id,
processed=1,
failed=1,
error_message=failed.error_message,
)
2026-07-06 02:19:07 +08:00
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
if value is None:
return WorkerTaskUpdate()
if isinstance(value, WorkerTaskUpdate):
return value
return WorkerTaskUpdate(
status=str(value.get("status") or "succeeded"),
progress=_progress(value.get("progress", 100)),
result=dict(value.get("result") or {}),
metadata=dict(value.get("metadata") or {}),
error_message=_optional_text(value.get("error_message")),
)
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
2026-07-06 02:27:07 +08:00
def _retry_deferred(task: GenerationTask) -> bool:
retry = task.metadata.get("retry")
if not isinstance(retry, dict):
return False
next_retry_at = _optional_text(retry.get("next_retry_at"))
if not next_retry_at:
return False
try:
parsed = datetime.fromisoformat(next_retry_at)
except ValueError:
return False
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed > datetime.now(timezone.utc)
2026-07-06 02:19:07 +08:00
def _progress(value: Any) -> int:
return min(max(_int(value, 100), 0), 100)
def _int(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default
def _optional_text(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _safe_error_message(exc: Exception) -> str:
text = " ".join(str(exc or type(exc).__name__).split())
return text[:500] if text else type(exc).__name__