2026-07-06 02:19:07 +08:00
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import tempfile
|
|
|
|
|
import threading
|
|
|
|
|
import unittest
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
from backend.services.queue_service import QueueService
|
2026-07-06 02:27:07 +08:00
|
|
|
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate
|
2026-07-06 02:19:07 +08:00
|
|
|
from core.audit_service import AuditService
|
|
|
|
|
from core.event_bus import EventBus
|
|
|
|
|
from core.identity import RequestContext
|
|
|
|
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
|
|
|
|
from database.repositories.job_repository import JobRepository
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class QueueWorkerTests(unittest.TestCase):
|
|
|
|
|
def setUp(self):
|
|
|
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
|
|
|
self.database = Database(
|
|
|
|
|
DatabaseSettings(sqlite=SqliteSettings(path=str(Path(self.tmp.name) / "imagestudio.db")))
|
|
|
|
|
)
|
|
|
|
|
self.database.initialize()
|
|
|
|
|
self.audit_service = AuditService(self.database)
|
|
|
|
|
self.event_bus = EventBus()
|
|
|
|
|
self.queue_service = QueueService(JobRepository(self.database), self.audit_service, self.event_bus)
|
|
|
|
|
self.ctx = RequestContext.from_astrbot(
|
|
|
|
|
username="admin",
|
|
|
|
|
user_id="user_001",
|
|
|
|
|
permissions={"can_write"},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
def tearDown(self):
|
|
|
|
|
self.tmp.cleanup()
|
|
|
|
|
|
|
|
|
|
def test_run_once_claims_and_completes_queued_task(self):
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued", "request": {"prompt": "portrait"}}, self.ctx)
|
|
|
|
|
worker = QueueWorker(
|
|
|
|
|
self.queue_service,
|
|
|
|
|
lambda claimed: WorkerTaskUpdate(
|
|
|
|
|
result={"handled_task_id": claimed.task_id},
|
|
|
|
|
metadata={"handler": "unit"},
|
|
|
|
|
),
|
|
|
|
|
worker_id="unit-worker",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
result = worker.run_once(self.ctx)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(result.status, "succeeded")
|
|
|
|
|
self.assertEqual(result.task_id, task.task_id)
|
|
|
|
|
completed = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(completed.status, "succeeded")
|
|
|
|
|
self.assertEqual(completed.progress, 100)
|
|
|
|
|
self.assertEqual(completed.result["handled_task_id"], task.task_id)
|
|
|
|
|
self.assertEqual(completed.metadata["handler"], "unit")
|
|
|
|
|
self.assertEqual(completed.metadata["worker"]["worker_id"], "unit-worker")
|
|
|
|
|
self.assertEqual(completed.metadata["worker"]["status"], "succeeded")
|
|
|
|
|
self.assertEqual(completed.metadata["worker_attempt"], 1)
|
|
|
|
|
|
|
|
|
|
def test_run_once_marks_task_failed_when_handler_raises(self):
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
|
|
|
|
|
|
|
|
|
def fail_handler(claimed):
|
|
|
|
|
raise RuntimeError(f"boom {claimed.task_id}")
|
|
|
|
|
|
|
|
|
|
worker = QueueWorker(self.queue_service, fail_handler, worker_id="failing-worker")
|
|
|
|
|
|
|
|
|
|
result = worker.run_once(self.ctx)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(result.status, "failed")
|
|
|
|
|
failed = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(failed.status, "failed")
|
|
|
|
|
self.assertIn("boom", failed.error_message)
|
|
|
|
|
self.assertEqual(failed.metadata["worker"]["status"], "failed")
|
|
|
|
|
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
|
|
|
|
|
|
2026-07-06 02:27:07 +08:00
|
|
|
def test_retry_policy_reschedules_retryable_failure(self):
|
|
|
|
|
calls = []
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
|
|
|
|
|
|
|
|
|
def fail_handler(claimed):
|
|
|
|
|
calls.append(claimed.task_id)
|
|
|
|
|
raise RuntimeError("temporary provider outage")
|
|
|
|
|
|
|
|
|
|
worker = QueueWorker(
|
|
|
|
|
self.queue_service,
|
|
|
|
|
fail_handler,
|
|
|
|
|
worker_id="retry-worker",
|
|
|
|
|
retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
result = worker.run_once(self.ctx)
|
|
|
|
|
deferred = worker.run_once(self.ctx)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(result.status, "retry_scheduled")
|
|
|
|
|
self.assertEqual(deferred.status, "idle")
|
|
|
|
|
self.assertEqual(calls, [task.task_id])
|
|
|
|
|
queued = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(queued.status, "queued")
|
|
|
|
|
self.assertIsNone(queued.error_message)
|
|
|
|
|
self.assertEqual(queued.metadata["retry"]["attempt"], 1)
|
|
|
|
|
self.assertEqual(queued.metadata["retry"]["max_attempts"], 2)
|
|
|
|
|
self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"])
|
|
|
|
|
|
|
|
|
|
def test_retry_policy_exhausts_to_failed(self):
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
|
|
|
|
worker = QueueWorker(
|
|
|
|
|
self.queue_service,
|
|
|
|
|
lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")),
|
|
|
|
|
worker_id="retry-exhaust-worker",
|
|
|
|
|
retry_policy=RetryPolicy(max_attempts=2),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
first = worker.run_once(self.ctx)
|
|
|
|
|
second = worker.run_once(self.ctx)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(first.status, "retry_scheduled")
|
|
|
|
|
self.assertEqual(second.status, "failed")
|
|
|
|
|
failed = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(failed.status, "failed")
|
|
|
|
|
self.assertEqual(failed.metadata["worker_attempt"], 2)
|
|
|
|
|
self.assertIn("still down", failed.error_message)
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
def test_background_worker_consumes_task_asynchronously(self):
|
|
|
|
|
handled = threading.Event()
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
|
|
|
|
|
|
|
|
|
def handler(claimed):
|
|
|
|
|
handled.set()
|
|
|
|
|
return {"result": {"async": True}}
|
|
|
|
|
|
|
|
|
|
worker = QueueWorker(self.queue_service, handler, worker_id="async-worker", interval_seconds=0.01)
|
|
|
|
|
try:
|
|
|
|
|
start_result = worker.start(self.ctx)
|
|
|
|
|
self.assertTrue(start_result["running"])
|
|
|
|
|
self.assertTrue(handled.wait(2.0))
|
|
|
|
|
finally:
|
|
|
|
|
stop_result = worker.stop(timeout_seconds=2.0)
|
|
|
|
|
|
|
|
|
|
self.assertFalse(stop_result["running"])
|
|
|
|
|
completed = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(completed.status, "succeeded")
|
|
|
|
|
self.assertTrue(completed.result["async"])
|
|
|
|
|
|
2026-07-06 02:23:05 +08:00
|
|
|
def test_worker_does_not_overwrite_task_cancelled_during_handler(self):
|
|
|
|
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
|
|
|
|
|
|
|
|
|
def handler(claimed):
|
|
|
|
|
self.queue_service.cancel_job(claimed.task_id, self.ctx, reason="stop during run")
|
|
|
|
|
return {"result": {"late_result": True}}
|
|
|
|
|
|
|
|
|
|
worker = QueueWorker(self.queue_service, handler, worker_id="cancel-aware-worker")
|
|
|
|
|
|
|
|
|
|
result = worker.run_once(self.ctx)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(result.status, "cancelled")
|
|
|
|
|
cancelled = self.queue_service.get_job(task.task_id)
|
|
|
|
|
self.assertEqual(cancelled.status, "cancelled")
|
|
|
|
|
self.assertNotIn("late_result", cancelled.result)
|
|
|
|
|
self.assertEqual(cancelled.metadata["cancellation"]["reason"], "stop during run")
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
def test_run_batch_stops_after_idle(self):
|
|
|
|
|
for index in range(2):
|
|
|
|
|
self.queue_service.create_task({"status": "queued", "request": {"index": index}}, self.ctx)
|
|
|
|
|
worker = QueueWorker(self.queue_service, lambda task: {"result": {"ok": True}}, worker_id="batch-worker")
|
|
|
|
|
|
|
|
|
|
summary = worker.run_batch(self.ctx, max_tasks=10)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(summary["processed"], 2)
|
|
|
|
|
self.assertEqual(summary["succeeded"], 2)
|
|
|
|
|
self.assertEqual(summary["failed"], 0)
|
|
|
|
|
self.assertEqual(summary["results"][-1]["status"], "idle")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
unittest.main()
|