diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py new file mode 100644 index 0000000..82bf8dd --- /dev/null +++ b/backend/services/worker_service.py @@ -0,0 +1,247 @@ +"""Persistent queue worker primitives.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +from backend.dto.common import ApiError +from backend.dto.job_dto import GenerationTask +from backend.services.queue_service import QueueService +from core.identity import RequestContext + + +WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"] + + +@dataclass(frozen=True) +class WorkerTaskUpdate: + status: str = "succeeded" + progress: int = 100 + result: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) + error_message: str | None = None + + +@dataclass(frozen=True) +class WorkerRunResult: + worker_id: str + status: str + task_id: str | None = None + processed: int = 0 + succeeded: int = 0 + failed: int = 0 + error_message: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "worker_id": self.worker_id, + "status": self.status, + "task_id": self.task_id, + "processed": self.processed, + "succeeded": self.succeeded, + "failed": self.failed, + "error_message": self.error_message, + } + + +class QueueWorker: + def __init__( + self, + queue_service: QueueService, + handler: WorkerHandler, + *, + worker_id: str = "imagestudio-worker", + interval_seconds: float = 1.0, + claim_batch_size: int = 5, + ): + self.queue_service = queue_service + self.handler = handler + self.worker_id = worker_id + self.interval_seconds = max(float(interval_seconds), 0.01) + self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) + self._stop_event = threading.Event() + self._lock = threading.Lock() + self._thread: threading.Thread | None = None + + def run_once(self, ctx: RequestContext) -> WorkerRunResult: + task = self._claim_next(ctx) + if task is None: + return WorkerRunResult(worker_id=self.worker_id, status="idle") + try: + update = _normalize_update(self.handler(task)) + completed = self.queue_service.update_task( + task.task_id, + { + "status": update.status, + "progress": _progress(update.progress), + "result": {**task.result, **update.result}, + "error_message": update.error_message, + "metadata": { + **task.metadata, + **update.metadata, + "worker": { + "worker_id": self.worker_id, + "status": update.status, + "finished_at": _now(), + }, + }, + }, + ctx, + ) + return WorkerRunResult( + worker_id=self.worker_id, + status=completed.status, + task_id=completed.task_id, + processed=1, + succeeded=1 if completed.status in {"completed", "succeeded"} else 0, + failed=1 if completed.status == "failed" else 0, + error_message=completed.error_message, + ) + except Exception as exc: + failed = self._mark_failed(task, exc, ctx) + return WorkerRunResult( + worker_id=self.worker_id, + status="failed", + task_id=failed.task_id, + processed=1, + failed=1, + error_message=failed.error_message, + ) + + def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: + limit = min(max(int(max_tasks), 1), 100) + results: list[dict[str, Any]] = [] + for _ in range(limit): + result = self.run_once(ctx) + results.append(result.to_dict()) + if result.status == "idle": + break + return { + "worker_id": self.worker_id, + "processed": sum(item["processed"] for item in results), + "succeeded": sum(item["succeeded"] for item in results), + "failed": sum(item["failed"] for item in results), + "results": results, + } + + def start(self, ctx: RequestContext) -> dict[str, Any]: + with self._lock: + if self.is_running(): + return {"worker_id": self.worker_id, "running": True, "already_running": True} + self._stop_event.clear() + self._thread = threading.Thread( + target=self._run_loop, + args=(ctx,), + daemon=True, + name=f"ImageStudioQueueWorker-{self.worker_id}", + ) + self._thread.start() + return {"worker_id": self.worker_id, "running": True, "already_running": False} + + def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]: + self._stop_event.set() + thread = self._thread + if thread is not None: + thread.join(timeout=max(float(timeout_seconds), 0.1)) + return {"worker_id": self.worker_id, "running": self.is_running()} + + def is_running(self) -> bool: + thread = self._thread + return bool(thread and thread.is_alive()) + + def _run_loop(self, ctx: RequestContext) -> None: + while not self._stop_event.is_set(): + result = self.run_once(ctx) + if result.status == "idle": + self._stop_event.wait(self.interval_seconds) + + def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: + queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items + for candidate in queued: + try: + attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1 + return self.queue_service.update_task( + candidate.task_id, + { + "status": "running", + "progress": max(candidate.progress, 1), + "metadata": { + **candidate.metadata, + "worker_attempt": attempt, + "worker": { + "worker_id": self.worker_id, + "status": "running", + "claimed_at": _now(), + }, + }, + }, + ctx, + ) + except ApiError: + continue + return None + + def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask: + message = _safe_error_message(exc) + return self.queue_service.update_task( + task.task_id, + { + "status": "failed", + "error_message": message, + "metadata": { + **task.metadata, + "worker": { + "worker_id": self.worker_id, + "status": "failed", + "error_type": type(exc).__name__, + "failed_at": _now(), + }, + }, + }, + ctx, + ) + + +def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: + if value is None: + return WorkerTaskUpdate() + if isinstance(value, WorkerTaskUpdate): + return value + return WorkerTaskUpdate( + status=str(value.get("status") or "succeeded"), + progress=_progress(value.get("progress", 100)), + result=dict(value.get("result") or {}), + metadata=dict(value.get("metadata") or {}), + error_message=_optional_text(value.get("error_message")), + ) + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _progress(value: Any) -> int: + return min(max(_int(value, 100), 0), 100) + + +def _int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _optional_text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _safe_error_message(exc: Exception) -> str: + text = " ".join(str(exc or type(exc).__name__).split()) + return text[:500] if text else type(exc).__name__ diff --git a/tests/test_worker_service.py b/tests/test_worker_service.py new file mode 100644 index 0000000..530d680 --- /dev/null +++ b/tests/test_worker_service.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import tempfile +import threading +import unittest +from pathlib import Path + +from backend.services.queue_service import QueueService +from backend.services.worker_service import QueueWorker, WorkerTaskUpdate +from core.audit_service import AuditService +from core.event_bus import EventBus +from core.identity import RequestContext +from database.db_helper import Database, DatabaseSettings, SqliteSettings +from database.repositories.job_repository import JobRepository + + +class QueueWorkerTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.database = Database( + DatabaseSettings(sqlite=SqliteSettings(path=str(Path(self.tmp.name) / "imagestudio.db"))) + ) + self.database.initialize() + self.audit_service = AuditService(self.database) + self.event_bus = EventBus() + self.queue_service = QueueService(JobRepository(self.database), self.audit_service, self.event_bus) + self.ctx = RequestContext.from_astrbot( + username="admin", + user_id="user_001", + permissions={"can_write"}, + ) + + def tearDown(self): + self.tmp.cleanup() + + def test_run_once_claims_and_completes_queued_task(self): + task = self.queue_service.create_task({"status": "queued", "request": {"prompt": "portrait"}}, self.ctx) + worker = QueueWorker( + self.queue_service, + lambda claimed: WorkerTaskUpdate( + result={"handled_task_id": claimed.task_id}, + metadata={"handler": "unit"}, + ), + worker_id="unit-worker", + ) + + result = worker.run_once(self.ctx) + + self.assertEqual(result.status, "succeeded") + self.assertEqual(result.task_id, task.task_id) + completed = self.queue_service.get_job(task.task_id) + self.assertEqual(completed.status, "succeeded") + self.assertEqual(completed.progress, 100) + self.assertEqual(completed.result["handled_task_id"], task.task_id) + self.assertEqual(completed.metadata["handler"], "unit") + self.assertEqual(completed.metadata["worker"]["worker_id"], "unit-worker") + self.assertEqual(completed.metadata["worker"]["status"], "succeeded") + self.assertEqual(completed.metadata["worker_attempt"], 1) + + def test_run_once_marks_task_failed_when_handler_raises(self): + task = self.queue_service.create_task({"status": "queued"}, self.ctx) + + def fail_handler(claimed): + raise RuntimeError(f"boom {claimed.task_id}") + + worker = QueueWorker(self.queue_service, fail_handler, worker_id="failing-worker") + + result = worker.run_once(self.ctx) + + self.assertEqual(result.status, "failed") + failed = self.queue_service.get_job(task.task_id) + self.assertEqual(failed.status, "failed") + self.assertIn("boom", failed.error_message) + self.assertEqual(failed.metadata["worker"]["status"], "failed") + self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError") + + def test_background_worker_consumes_task_asynchronously(self): + handled = threading.Event() + task = self.queue_service.create_task({"status": "queued"}, self.ctx) + + def handler(claimed): + handled.set() + return {"result": {"async": True}} + + worker = QueueWorker(self.queue_service, handler, worker_id="async-worker", interval_seconds=0.01) + try: + start_result = worker.start(self.ctx) + self.assertTrue(start_result["running"]) + self.assertTrue(handled.wait(2.0)) + finally: + stop_result = worker.stop(timeout_seconds=2.0) + + self.assertFalse(stop_result["running"]) + completed = self.queue_service.get_job(task.task_id) + self.assertEqual(completed.status, "succeeded") + self.assertTrue(completed.result["async"]) + + def test_run_batch_stops_after_idle(self): + for index in range(2): + self.queue_service.create_task({"status": "queued", "request": {"index": index}}, self.ctx) + worker = QueueWorker(self.queue_service, lambda task: {"result": {"ok": True}}, worker_id="batch-worker") + + summary = worker.run_batch(self.ctx, max_tasks=10) + + self.assertEqual(summary["processed"], 2) + self.assertEqual(summary["succeeded"], 2) + self.assertEqual(summary["failed"], 0) + self.assertEqual(summary["results"][-1]["status"], "idle") + + +if __name__ == "__main__": + unittest.main()