228 行
7.1 KiB
Python
228 行
7.1 KiB
Python
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import threading
|
|||
|
|
import time
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
from backend.dto.common import ApiError, ErrorCode
|
|||
|
|
from backend.services.queue_service import QueueService
|
|||
|
|
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate, _safe_error_message
|
|||
|
|
from core.audit_service import AuditService
|
|||
|
|
from core.identity import RequestContext
|
|||
|
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
|||
|
|
from database.repositories.job_repository import JobRepository
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _database(tmp_path) -> Database:
|
|||
|
|
database = Database(
|
|||
|
|
DatabaseSettings(
|
|||
|
|
backend="sqlite",
|
|||
|
|
sqlite=SqliteSettings(path=str(tmp_path / "worker-service.sqlite3")),
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
database.initialize()
|
|||
|
|
return database
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _queue(tmp_path) -> tuple[QueueService, JobRepository, RequestContext]:
|
|||
|
|
database = _database(tmp_path)
|
|||
|
|
repository = JobRepository(database)
|
|||
|
|
queue = QueueService(repository, AuditService(database))
|
|||
|
|
return queue, repository, RequestContext.system("worker-test")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _enqueue(
|
|||
|
|
queue: QueueService,
|
|||
|
|
ctx: RequestContext,
|
|||
|
|
*,
|
|||
|
|
task_id: str = "task-1",
|
|||
|
|
task_type: str = "advanced_style_source_recognition",
|
|||
|
|
metadata: dict[str, Any] | None = None,
|
|||
|
|
):
|
|||
|
|
return queue.create_task(
|
|||
|
|
{
|
|||
|
|
"task_id": task_id,
|
|||
|
|
"task_type": task_type,
|
|||
|
|
"status": "queued",
|
|||
|
|
"progress": 0,
|
|||
|
|
"request": {},
|
|||
|
|
"metadata": metadata
|
|||
|
|
or {
|
|||
|
|
"source": "advanced_material",
|
|||
|
|
"advanced_material_import_id": "import-1",
|
|||
|
|
},
|
|||
|
|
},
|
|||
|
|
ctx,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_safe_error_message_prefers_api_error_fields() -> None:
|
|||
|
|
exc = ApiError(
|
|||
|
|
ErrorCode.GENERATION_FAILED,
|
|||
|
|
"advanced style source recognition failed",
|
|||
|
|
{"finish_reason": "prohibited_content", "reason": "no usable output"},
|
|||
|
|
)
|
|||
|
|
message = _safe_error_message(exc)
|
|||
|
|
assert "advanced style source recognition failed" in message
|
|||
|
|
assert "GENERATION_FAILED" in message
|
|||
|
|
assert "prohibited_content" in message
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_handler_api_error_marks_task_failed_without_raising(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
_enqueue(queue, ctx)
|
|||
|
|
|
|||
|
|
def boom(_task):
|
|||
|
|
raise ApiError(
|
|||
|
|
ErrorCode.GENERATION_FAILED,
|
|||
|
|
"advanced style source recognition failed",
|
|||
|
|
{"finish_reason": "prohibited_content"},
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
worker = QueueWorker(queue, boom, worker_id="imagestudio-advanced-material-1")
|
|||
|
|
result = worker.run_once(ctx)
|
|||
|
|
|
|||
|
|
assert result.status == "failed"
|
|||
|
|
assert result.failed == 1
|
|||
|
|
assert result.task_id == "task-1"
|
|||
|
|
assert "advanced style source recognition failed" in (result.error_message or "")
|
|||
|
|
assert "GENERATION_FAILED" in (result.error_message or "")
|
|||
|
|
|
|||
|
|
job = queue.get_job("task-1")
|
|||
|
|
assert job.status == "failed"
|
|||
|
|
assert job.metadata.get("worker", {}).get("worker_id") == "imagestudio-advanced-material-1"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_update_task_secondary_failure_does_not_raise(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
_enqueue(queue, ctx, task_id="task-secondary")
|
|||
|
|
|
|||
|
|
def boom(_task):
|
|||
|
|
raise RuntimeError("provider exploded")
|
|||
|
|
|
|||
|
|
worker = QueueWorker(queue, boom, worker_id="worker-secondary")
|
|||
|
|
original_update = queue.update_task
|
|||
|
|
|
|||
|
|
def flaky_update(task_id, payload, request_ctx):
|
|||
|
|
if payload.get("status") == "failed":
|
|||
|
|
raise RuntimeError("sqlite is busy")
|
|||
|
|
return original_update(task_id, payload, request_ctx)
|
|||
|
|
|
|||
|
|
queue.update_task = flaky_update # type: ignore[method-assign]
|
|||
|
|
|
|||
|
|
result = worker.run_once(ctx)
|
|||
|
|
assert result.status == "failed"
|
|||
|
|
assert result.failed == 1
|
|||
|
|
assert result.task_id == "task-secondary"
|
|||
|
|
assert "provider exploded" in (result.error_message or "")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_terminal_callback_exception_is_swallowed(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
_enqueue(queue, ctx, task_id="task-callback")
|
|||
|
|
seen: list[str] = []
|
|||
|
|
|
|||
|
|
def ok(_task):
|
|||
|
|
return WorkerTaskUpdate(status="succeeded", progress=100, result={"ok": True})
|
|||
|
|
|
|||
|
|
def terminal(claimed, current, exc):
|
|||
|
|
seen.append(current.task_id)
|
|||
|
|
raise RuntimeError("coordinator exploded")
|
|||
|
|
|
|||
|
|
worker = QueueWorker(
|
|||
|
|
queue,
|
|||
|
|
ok,
|
|||
|
|
worker_id="worker-callback",
|
|||
|
|
terminal_callback=terminal,
|
|||
|
|
)
|
|||
|
|
result = worker.run_once(ctx)
|
|||
|
|
assert result.status == "succeeded"
|
|||
|
|
assert result.succeeded == 1
|
|||
|
|
assert seen == ["task-callback"]
|
|||
|
|
assert queue.get_job("task-callback").status == "succeeded"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_worker_thread_survives_repeated_handler_failures(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
for index in range(3):
|
|||
|
|
_enqueue(queue, ctx, task_id=f"task-loop-{index}")
|
|||
|
|
|
|||
|
|
def boom(_task):
|
|||
|
|
raise ApiError(ErrorCode.GENERATION_FAILED, "vision blocked", {"finish_reason": "prohibited_content"})
|
|||
|
|
|
|||
|
|
worker = QueueWorker(
|
|||
|
|
queue,
|
|||
|
|
boom,
|
|||
|
|
worker_id="imagestudio-advanced-material-loop",
|
|||
|
|
interval_seconds=0.05,
|
|||
|
|
claim_batch_size=10,
|
|||
|
|
)
|
|||
|
|
start = worker.start(ctx)
|
|||
|
|
assert start["running"] is True
|
|||
|
|
|
|||
|
|
deadline = time.time() + 5.0
|
|||
|
|
failed_ids: set[str] = set()
|
|||
|
|
while time.time() < deadline and len(failed_ids) < 3:
|
|||
|
|
for task_id in ("task-loop-0", "task-loop-1", "task-loop-2"):
|
|||
|
|
job = queue.get_job(task_id)
|
|||
|
|
if job.status == "failed":
|
|||
|
|
failed_ids.add(task_id)
|
|||
|
|
if len(failed_ids) < 3:
|
|||
|
|
time.sleep(0.05)
|
|||
|
|
|
|||
|
|
assert worker.is_running() is True
|
|||
|
|
assert failed_ids == {"task-loop-0", "task-loop-1", "task-loop-2"}
|
|||
|
|
|
|||
|
|
stop = worker.stop(timeout_seconds=2.0)
|
|||
|
|
assert stop["running"] is False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_start_can_revive_after_thread_reference_is_dead(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
worker = QueueWorker(
|
|||
|
|
queue,
|
|||
|
|
lambda _task: WorkerTaskUpdate(),
|
|||
|
|
worker_id="worker-revive",
|
|||
|
|
interval_seconds=0.05,
|
|||
|
|
)
|
|||
|
|
first = worker.start(ctx)
|
|||
|
|
assert first["running"] is True
|
|||
|
|
thread = worker._thread
|
|||
|
|
assert thread is not None
|
|||
|
|
worker._stop_event.set()
|
|||
|
|
thread.join(timeout=2.0)
|
|||
|
|
assert worker.is_running() is False
|
|||
|
|
|
|||
|
|
second = worker.start(ctx)
|
|||
|
|
assert second["running"] is True
|
|||
|
|
assert second["already_running"] is False
|
|||
|
|
assert worker.is_running() is True
|
|||
|
|
worker.stop(timeout_seconds=2.0)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_run_loop_swallows_run_once_crash(tmp_path) -> None:
|
|||
|
|
queue, _repository, ctx = _queue(tmp_path)
|
|||
|
|
worker = QueueWorker(
|
|||
|
|
queue,
|
|||
|
|
lambda _task: WorkerTaskUpdate(),
|
|||
|
|
worker_id="worker-loop-guard",
|
|||
|
|
interval_seconds=0.05,
|
|||
|
|
)
|
|||
|
|
calls = {"n": 0}
|
|||
|
|
original = worker.run_once
|
|||
|
|
|
|||
|
|
def exploding_once(request_ctx):
|
|||
|
|
calls["n"] += 1
|
|||
|
|
if calls["n"] == 1:
|
|||
|
|
raise RuntimeError("unexpected run_once crash")
|
|||
|
|
return original(request_ctx)
|
|||
|
|
|
|||
|
|
worker.run_once = exploding_once # type: ignore[method-assign]
|
|||
|
|
worker.start(ctx)
|
|||
|
|
deadline = time.time() + 2.0
|
|||
|
|
while time.time() < deadline and calls["n"] < 2:
|
|||
|
|
time.sleep(0.05)
|
|||
|
|
assert worker.is_running() is True
|
|||
|
|
assert calls["n"] >= 2
|
|||
|
|
worker.stop(timeout_seconds=2.0)
|