from __future__ import annotations import tempfile import threading import unittest from concurrent.futures import ThreadPoolExecutor from pathlib import Path from backend.services.queue_service import QueueService from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate from core.audit_service import AuditService from core.event_bus import EventBus from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.job_repository import JobRepository class QueueWorkerTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.database = Database( DatabaseSettings(sqlite=SqliteSettings(path=str(Path(self.tmp.name) / "imagestudio.db"))) ) self.database.initialize() self.audit_service = AuditService(self.database) self.event_bus = EventBus() self.queue_service = QueueService(JobRepository(self.database), self.audit_service, self.event_bus) self.ctx = RequestContext.from_astrbot( username="admin", user_id="user_001", permissions={"can_write"}, ) def tearDown(self): self.tmp.cleanup() def test_run_once_claims_and_completes_queued_task(self): task = self.queue_service.create_task({"status": "queued", "request": {"prompt": "portrait"}}, self.ctx) worker = QueueWorker( self.queue_service, lambda claimed: WorkerTaskUpdate( result={"handled_task_id": claimed.task_id}, metadata={"handler": "unit"}, ), worker_id="unit-worker", ) result = worker.run_once(self.ctx) self.assertEqual(result.status, "succeeded") self.assertEqual(result.task_id, task.task_id) completed = self.queue_service.get_job(task.task_id) self.assertEqual(completed.status, "succeeded") self.assertEqual(completed.progress, 100) self.assertEqual(completed.result["handled_task_id"], task.task_id) self.assertEqual(completed.metadata["handler"], "unit") self.assertEqual(completed.metadata["worker"]["worker_id"], "unit-worker") self.assertEqual(completed.metadata["worker"]["status"], "succeeded") self.assertEqual(completed.metadata["worker_attempt"], 1) def test_run_once_marks_task_failed_when_handler_raises(self): task = self.queue_service.create_task({"status": "queued"}, self.ctx) def fail_handler(claimed): raise RuntimeError(f"boom {claimed.task_id}") worker = QueueWorker(self.queue_service, fail_handler, worker_id="failing-worker") result = worker.run_once(self.ctx) self.assertEqual(result.status, "failed") failed = self.queue_service.get_job(task.task_id) self.assertEqual(failed.status, "failed") self.assertIn("boom", failed.error_message) self.assertEqual(failed.metadata["worker"]["status"], "failed") self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError") def test_retry_policy_reschedules_retryable_failure(self): calls = [] task = self.queue_service.create_task({"status": "queued"}, self.ctx) def fail_handler(claimed): calls.append(claimed.task_id) raise RuntimeError("temporary provider outage") worker = QueueWorker( self.queue_service, fail_handler, worker_id="retry-worker", retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60), ) result = worker.run_once(self.ctx) deferred = worker.run_once(self.ctx) self.assertEqual(result.status, "retry_scheduled") self.assertEqual(deferred.status, "idle") self.assertEqual(calls, [task.task_id]) queued = self.queue_service.get_job(task.task_id) self.assertEqual(queued.status, "queued") self.assertIsNone(queued.error_message) self.assertEqual(queued.metadata["retry"]["attempt"], 1) self.assertEqual(queued.metadata["retry"]["max_attempts"], 2) self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"]) def test_retry_policy_exhausts_to_failed(self): task = self.queue_service.create_task({"status": "queued"}, self.ctx) worker = QueueWorker( self.queue_service, lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")), worker_id="retry-exhaust-worker", retry_policy=RetryPolicy(max_attempts=2), ) first = worker.run_once(self.ctx) second = worker.run_once(self.ctx) self.assertEqual(first.status, "retry_scheduled") self.assertEqual(second.status, "failed") failed = self.queue_service.get_job(task.task_id) self.assertEqual(failed.status, "failed") self.assertEqual(failed.metadata["worker_attempt"], 2) self.assertIn("still down", failed.error_message) def test_background_worker_consumes_task_asynchronously(self): handled = threading.Event() task = self.queue_service.create_task({"status": "queued"}, self.ctx) def handler(claimed): handled.set() return {"result": {"async": True}} worker = QueueWorker(self.queue_service, handler, worker_id="async-worker", interval_seconds=0.01) try: start_result = worker.start(self.ctx) self.assertTrue(start_result["running"]) self.assertTrue(handled.wait(2.0)) finally: stop_result = worker.stop(timeout_seconds=2.0) self.assertFalse(stop_result["running"]) completed = self.queue_service.get_job(task.task_id) self.assertEqual(completed.status, "succeeded") self.assertTrue(completed.result["async"]) def test_worker_does_not_overwrite_task_cancelled_during_handler(self): task = self.queue_service.create_task({"status": "queued"}, self.ctx) def handler(claimed): self.queue_service.cancel_job(claimed.task_id, self.ctx, reason="stop during run") return {"result": {"late_result": True}} worker = QueueWorker(self.queue_service, handler, worker_id="cancel-aware-worker") result = worker.run_once(self.ctx) self.assertEqual(result.status, "cancelled") cancelled = self.queue_service.get_job(task.task_id) self.assertEqual(cancelled.status, "cancelled") self.assertNotIn("late_result", cancelled.result) self.assertEqual(cancelled.metadata["cancellation"]["reason"], "stop during run") def test_run_batch_stops_after_idle(self): for index in range(2): self.queue_service.create_task({"status": "queued", "request": {"index": index}}, self.ctx) worker = QueueWorker(self.queue_service, lambda task: {"result": {"ok": True}}, worker_id="batch-worker") summary = worker.run_batch(self.ctx, max_tasks=10) self.assertEqual(summary["processed"], 2) self.assertEqual(summary["succeeded"], 2) self.assertEqual(summary["failed"], 0) self.assertEqual(summary["results"][-1]["status"], "idle") def test_worker_claim_is_compare_and_set(self): task = self.queue_service.create_task({"status": "queued"}, self.ctx) barrier = threading.Barrier(2) def claim(worker_id: str): barrier.wait(timeout=2) return self.queue_service.claim_queued_task(task.task_id, self.ctx, worker_id=worker_id) with ThreadPoolExecutor(max_workers=2) as executor: results = list(executor.map(claim, ("worker-a", "worker-b"))) claimed = [result for result in results if result is not None] self.assertEqual(len(claimed), 1) self.assertEqual(claimed[0].status, "running") self.assertEqual(claimed[0].metadata["worker_attempt"], 1) if __name__ == "__main__": unittest.main()