S83 异步worker基础设施
这个提交存在于:
父节点
1aeb290737
当前提交
3c92a566f8
共有 2 个文件被更改,包括 359 次插入 和 0 次删除
247
backend/services/worker_service.py
普通文件
247
backend/services/worker_service.py
普通文件
|
|
@ -0,0 +1,247 @@
|
|||
"""Persistent queue worker primitives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from backend.dto.common import ApiError
|
||||
from backend.dto.job_dto import GenerationTask
|
||||
from backend.services.queue_service import QueueService
|
||||
from core.identity import RequestContext
|
||||
|
||||
|
||||
WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerTaskUpdate:
|
||||
status: str = "succeeded"
|
||||
progress: int = 100
|
||||
result: dict[str, Any] = field(default_factory=dict)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerRunResult:
|
||||
worker_id: str
|
||||
status: str
|
||||
task_id: str | None = None
|
||||
processed: int = 0
|
||||
succeeded: int = 0
|
||||
failed: int = 0
|
||||
error_message: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"worker_id": self.worker_id,
|
||||
"status": self.status,
|
||||
"task_id": self.task_id,
|
||||
"processed": self.processed,
|
||||
"succeeded": self.succeeded,
|
||||
"failed": self.failed,
|
||||
"error_message": self.error_message,
|
||||
}
|
||||
|
||||
|
||||
class QueueWorker:
|
||||
def __init__(
|
||||
self,
|
||||
queue_service: QueueService,
|
||||
handler: WorkerHandler,
|
||||
*,
|
||||
worker_id: str = "imagestudio-worker",
|
||||
interval_seconds: float = 1.0,
|
||||
claim_batch_size: int = 5,
|
||||
):
|
||||
self.queue_service = queue_service
|
||||
self.handler = handler
|
||||
self.worker_id = worker_id
|
||||
self.interval_seconds = max(float(interval_seconds), 0.01)
|
||||
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
|
||||
self._stop_event = threading.Event()
|
||||
self._lock = threading.Lock()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def run_once(self, ctx: RequestContext) -> WorkerRunResult:
|
||||
task = self._claim_next(ctx)
|
||||
if task is None:
|
||||
return WorkerRunResult(worker_id=self.worker_id, status="idle")
|
||||
try:
|
||||
update = _normalize_update(self.handler(task))
|
||||
completed = self.queue_service.update_task(
|
||||
task.task_id,
|
||||
{
|
||||
"status": update.status,
|
||||
"progress": _progress(update.progress),
|
||||
"result": {**task.result, **update.result},
|
||||
"error_message": update.error_message,
|
||||
"metadata": {
|
||||
**task.metadata,
|
||||
**update.metadata,
|
||||
"worker": {
|
||||
"worker_id": self.worker_id,
|
||||
"status": update.status,
|
||||
"finished_at": _now(),
|
||||
},
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status=completed.status,
|
||||
task_id=completed.task_id,
|
||||
processed=1,
|
||||
succeeded=1 if completed.status in {"completed", "succeeded"} else 0,
|
||||
failed=1 if completed.status == "failed" else 0,
|
||||
error_message=completed.error_message,
|
||||
)
|
||||
except Exception as exc:
|
||||
failed = self._mark_failed(task, exc, ctx)
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status="failed",
|
||||
task_id=failed.task_id,
|
||||
processed=1,
|
||||
failed=1,
|
||||
error_message=failed.error_message,
|
||||
)
|
||||
|
||||
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
|
||||
limit = min(max(int(max_tasks), 1), 100)
|
||||
results: list[dict[str, Any]] = []
|
||||
for _ in range(limit):
|
||||
result = self.run_once(ctx)
|
||||
results.append(result.to_dict())
|
||||
if result.status == "idle":
|
||||
break
|
||||
return {
|
||||
"worker_id": self.worker_id,
|
||||
"processed": sum(item["processed"] for item in results),
|
||||
"succeeded": sum(item["succeeded"] for item in results),
|
||||
"failed": sum(item["failed"] for item in results),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
def start(self, ctx: RequestContext) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
if self.is_running():
|
||||
return {"worker_id": self.worker_id, "running": True, "already_running": True}
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop,
|
||||
args=(ctx,),
|
||||
daemon=True,
|
||||
name=f"ImageStudioQueueWorker-{self.worker_id}",
|
||||
)
|
||||
self._thread.start()
|
||||
return {"worker_id": self.worker_id, "running": True, "already_running": False}
|
||||
|
||||
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
|
||||
self._stop_event.set()
|
||||
thread = self._thread
|
||||
if thread is not None:
|
||||
thread.join(timeout=max(float(timeout_seconds), 0.1))
|
||||
return {"worker_id": self.worker_id, "running": self.is_running()}
|
||||
|
||||
def is_running(self) -> bool:
|
||||
thread = self._thread
|
||||
return bool(thread and thread.is_alive())
|
||||
|
||||
def _run_loop(self, ctx: RequestContext) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
result = self.run_once(ctx)
|
||||
if result.status == "idle":
|
||||
self._stop_event.wait(self.interval_seconds)
|
||||
|
||||
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
|
||||
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
|
||||
for candidate in queued:
|
||||
try:
|
||||
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
|
||||
return self.queue_service.update_task(
|
||||
candidate.task_id,
|
||||
{
|
||||
"status": "running",
|
||||
"progress": max(candidate.progress, 1),
|
||||
"metadata": {
|
||||
**candidate.metadata,
|
||||
"worker_attempt": attempt,
|
||||
"worker": {
|
||||
"worker_id": self.worker_id,
|
||||
"status": "running",
|
||||
"claimed_at": _now(),
|
||||
},
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
except ApiError:
|
||||
continue
|
||||
return None
|
||||
|
||||
def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask:
|
||||
message = _safe_error_message(exc)
|
||||
return self.queue_service.update_task(
|
||||
task.task_id,
|
||||
{
|
||||
"status": "failed",
|
||||
"error_message": message,
|
||||
"metadata": {
|
||||
**task.metadata,
|
||||
"worker": {
|
||||
"worker_id": self.worker_id,
|
||||
"status": "failed",
|
||||
"error_type": type(exc).__name__,
|
||||
"failed_at": _now(),
|
||||
},
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
|
||||
if value is None:
|
||||
return WorkerTaskUpdate()
|
||||
if isinstance(value, WorkerTaskUpdate):
|
||||
return value
|
||||
return WorkerTaskUpdate(
|
||||
status=str(value.get("status") or "succeeded"),
|
||||
progress=_progress(value.get("progress", 100)),
|
||||
result=dict(value.get("result") or {}),
|
||||
metadata=dict(value.get("metadata") or {}),
|
||||
error_message=_optional_text(value.get("error_message")),
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _progress(value: Any) -> int:
|
||||
return min(max(_int(value, 100), 0), 100)
|
||||
|
||||
|
||||
def _int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _optional_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _safe_error_message(exc: Exception) -> str:
|
||||
text = " ".join(str(exc or type(exc).__name__).split())
|
||||
return text[:500] if text else type(exc).__name__
|
||||
112
tests/test_worker_service.py
普通文件
112
tests/test_worker_service.py
普通文件
|
|
@ -0,0 +1,112 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from backend.services.queue_service import QueueService
|
||||
from backend.services.worker_service import QueueWorker, WorkerTaskUpdate
|
||||
from core.audit_service import AuditService
|
||||
from core.event_bus import EventBus
|
||||
from core.identity import RequestContext
|
||||
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
||||
from database.repositories.job_repository import JobRepository
|
||||
|
||||
|
||||
class QueueWorkerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.database = Database(
|
||||
DatabaseSettings(sqlite=SqliteSettings(path=str(Path(self.tmp.name) / "imagestudio.db")))
|
||||
)
|
||||
self.database.initialize()
|
||||
self.audit_service = AuditService(self.database)
|
||||
self.event_bus = EventBus()
|
||||
self.queue_service = QueueService(JobRepository(self.database), self.audit_service, self.event_bus)
|
||||
self.ctx = RequestContext.from_astrbot(
|
||||
username="admin",
|
||||
user_id="user_001",
|
||||
permissions={"can_write"},
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_run_once_claims_and_completes_queued_task(self):
|
||||
task = self.queue_service.create_task({"status": "queued", "request": {"prompt": "portrait"}}, self.ctx)
|
||||
worker = QueueWorker(
|
||||
self.queue_service,
|
||||
lambda claimed: WorkerTaskUpdate(
|
||||
result={"handled_task_id": claimed.task_id},
|
||||
metadata={"handler": "unit"},
|
||||
),
|
||||
worker_id="unit-worker",
|
||||
)
|
||||
|
||||
result = worker.run_once(self.ctx)
|
||||
|
||||
self.assertEqual(result.status, "succeeded")
|
||||
self.assertEqual(result.task_id, task.task_id)
|
||||
completed = self.queue_service.get_job(task.task_id)
|
||||
self.assertEqual(completed.status, "succeeded")
|
||||
self.assertEqual(completed.progress, 100)
|
||||
self.assertEqual(completed.result["handled_task_id"], task.task_id)
|
||||
self.assertEqual(completed.metadata["handler"], "unit")
|
||||
self.assertEqual(completed.metadata["worker"]["worker_id"], "unit-worker")
|
||||
self.assertEqual(completed.metadata["worker"]["status"], "succeeded")
|
||||
self.assertEqual(completed.metadata["worker_attempt"], 1)
|
||||
|
||||
def test_run_once_marks_task_failed_when_handler_raises(self):
|
||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||
|
||||
def fail_handler(claimed):
|
||||
raise RuntimeError(f"boom {claimed.task_id}")
|
||||
|
||||
worker = QueueWorker(self.queue_service, fail_handler, worker_id="failing-worker")
|
||||
|
||||
result = worker.run_once(self.ctx)
|
||||
|
||||
self.assertEqual(result.status, "failed")
|
||||
failed = self.queue_service.get_job(task.task_id)
|
||||
self.assertEqual(failed.status, "failed")
|
||||
self.assertIn("boom", failed.error_message)
|
||||
self.assertEqual(failed.metadata["worker"]["status"], "failed")
|
||||
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
|
||||
|
||||
def test_background_worker_consumes_task_asynchronously(self):
|
||||
handled = threading.Event()
|
||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||
|
||||
def handler(claimed):
|
||||
handled.set()
|
||||
return {"result": {"async": True}}
|
||||
|
||||
worker = QueueWorker(self.queue_service, handler, worker_id="async-worker", interval_seconds=0.01)
|
||||
try:
|
||||
start_result = worker.start(self.ctx)
|
||||
self.assertTrue(start_result["running"])
|
||||
self.assertTrue(handled.wait(2.0))
|
||||
finally:
|
||||
stop_result = worker.stop(timeout_seconds=2.0)
|
||||
|
||||
self.assertFalse(stop_result["running"])
|
||||
completed = self.queue_service.get_job(task.task_id)
|
||||
self.assertEqual(completed.status, "succeeded")
|
||||
self.assertTrue(completed.result["async"])
|
||||
|
||||
def test_run_batch_stops_after_idle(self):
|
||||
for index in range(2):
|
||||
self.queue_service.create_task({"status": "queued", "request": {"index": index}}, self.ctx)
|
||||
worker = QueueWorker(self.queue_service, lambda task: {"result": {"ok": True}}, worker_id="batch-worker")
|
||||
|
||||
summary = worker.run_batch(self.ctx, max_tasks=10)
|
||||
|
||||
self.assertEqual(summary["processed"], 2)
|
||||
self.assertEqual(summary["succeeded"], 2)
|
||||
self.assertEqual(summary["failed"], 0)
|
||||
self.assertEqual(summary["results"][-1]["status"], "idle")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
正在加载…
添加表格
在新工单中引用