from __future__ import annotations import threading import time from typing import Any from backend.dto.common import ApiError, ErrorCode from backend.services.queue_service import QueueService from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate, _safe_error_message from core.audit_service import AuditService from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.job_repository import JobRepository def _database(tmp_path) -> Database: database = Database( DatabaseSettings( backend="sqlite", sqlite=SqliteSettings(path=str(tmp_path / "worker-service.sqlite3")), ) ) database.initialize() return database def _queue(tmp_path) -> tuple[QueueService, JobRepository, RequestContext]: database = _database(tmp_path) repository = JobRepository(database) queue = QueueService(repository, AuditService(database)) return queue, repository, RequestContext.system("worker-test") def _enqueue( queue: QueueService, ctx: RequestContext, *, task_id: str = "task-1", task_type: str = "advanced_style_source_recognition", metadata: dict[str, Any] | None = None, ): return queue.create_task( { "task_id": task_id, "task_type": task_type, "status": "queued", "progress": 0, "request": {}, "metadata": metadata or { "source": "advanced_material", "advanced_material_import_id": "import-1", }, }, ctx, ) def test_safe_error_message_prefers_api_error_fields() -> None: exc = ApiError( ErrorCode.GENERATION_FAILED, "advanced style source recognition failed", {"finish_reason": "prohibited_content", "reason": "no usable output"}, ) message = _safe_error_message(exc) assert "advanced style source recognition failed" in message assert "GENERATION_FAILED" in message assert "prohibited_content" in message def test_handler_api_error_marks_task_failed_without_raising(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) _enqueue(queue, ctx) def boom(_task): raise ApiError( ErrorCode.GENERATION_FAILED, "advanced style source recognition failed", {"finish_reason": "prohibited_content"}, ) worker = QueueWorker(queue, boom, worker_id="imagestudio-advanced-material-1") result = worker.run_once(ctx) assert result.status == "failed" assert result.failed == 1 assert result.task_id == "task-1" assert "advanced style source recognition failed" in (result.error_message or "") assert "GENERATION_FAILED" in (result.error_message or "") job = queue.get_job("task-1") assert job.status == "failed" assert job.metadata.get("worker", {}).get("worker_id") == "imagestudio-advanced-material-1" def test_update_task_secondary_failure_does_not_raise(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) _enqueue(queue, ctx, task_id="task-secondary") def boom(_task): raise RuntimeError("provider exploded") worker = QueueWorker(queue, boom, worker_id="worker-secondary") original_update = queue.update_task def flaky_update(task_id, payload, request_ctx): if payload.get("status") == "failed": raise RuntimeError("sqlite is busy") return original_update(task_id, payload, request_ctx) queue.update_task = flaky_update # type: ignore[method-assign] result = worker.run_once(ctx) assert result.status == "failed" assert result.failed == 1 assert result.task_id == "task-secondary" assert "provider exploded" in (result.error_message or "") def test_terminal_callback_exception_is_swallowed(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) _enqueue(queue, ctx, task_id="task-callback") seen: list[str] = [] def ok(_task): return WorkerTaskUpdate(status="succeeded", progress=100, result={"ok": True}) def terminal(claimed, current, exc): seen.append(current.task_id) raise RuntimeError("coordinator exploded") worker = QueueWorker( queue, ok, worker_id="worker-callback", terminal_callback=terminal, ) result = worker.run_once(ctx) assert result.status == "succeeded" assert result.succeeded == 1 assert seen == ["task-callback"] assert queue.get_job("task-callback").status == "succeeded" def test_worker_thread_survives_repeated_handler_failures(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) for index in range(3): _enqueue(queue, ctx, task_id=f"task-loop-{index}") def boom(_task): raise ApiError(ErrorCode.GENERATION_FAILED, "vision blocked", {"finish_reason": "prohibited_content"}) worker = QueueWorker( queue, boom, worker_id="imagestudio-advanced-material-loop", interval_seconds=0.05, claim_batch_size=10, ) start = worker.start(ctx) assert start["running"] is True deadline = time.time() + 5.0 failed_ids: set[str] = set() while time.time() < deadline and len(failed_ids) < 3: for task_id in ("task-loop-0", "task-loop-1", "task-loop-2"): job = queue.get_job(task_id) if job.status == "failed": failed_ids.add(task_id) if len(failed_ids) < 3: time.sleep(0.05) assert worker.is_running() is True assert failed_ids == {"task-loop-0", "task-loop-1", "task-loop-2"} stop = worker.stop(timeout_seconds=2.0) assert stop["running"] is False def test_start_can_revive_after_thread_reference_is_dead(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) worker = QueueWorker( queue, lambda _task: WorkerTaskUpdate(), worker_id="worker-revive", interval_seconds=0.05, ) first = worker.start(ctx) assert first["running"] is True thread = worker._thread assert thread is not None worker._stop_event.set() thread.join(timeout=2.0) assert worker.is_running() is False second = worker.start(ctx) assert second["running"] is True assert second["already_running"] is False assert worker.is_running() is True worker.stop(timeout_seconds=2.0) def test_run_loop_swallows_run_once_crash(tmp_path) -> None: queue, _repository, ctx = _queue(tmp_path) worker = QueueWorker( queue, lambda _task: WorkerTaskUpdate(), worker_id="worker-loop-guard", interval_seconds=0.05, ) calls = {"n": 0} original = worker.run_once def exploding_once(request_ctx): calls["n"] += 1 if calls["n"] == 1: raise RuntimeError("unexpected run_once crash") return original(request_ctx) worker.run_once = exploding_once # type: ignore[method-assign] worker.start(ctx) deadline = time.time() + 2.0 while time.time() < deadline and calls["n"] < 2: time.sleep(0.05) assert worker.is_running() is True assert calls["n"] >= 2 worker.stop(timeout_seconds=2.0)