astrbot_plugin_imagestudio/tests/test_worker_service.py

228 行
7.1 KiB
Python

from __future__ import annotations
import threading
import time
from typing import Any
from backend.dto.common import ApiError, ErrorCode
from backend.services.queue_service import QueueService
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate, _safe_error_message
from core.audit_service import AuditService
from core.identity import RequestContext
from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.job_repository import JobRepository
def _database(tmp_path) -> Database:
database = Database(
DatabaseSettings(
backend="sqlite",
sqlite=SqliteSettings(path=str(tmp_path / "worker-service.sqlite3")),
)
)
database.initialize()
return database
def _queue(tmp_path) -> tuple[QueueService, JobRepository, RequestContext]:
database = _database(tmp_path)
repository = JobRepository(database)
queue = QueueService(repository, AuditService(database))
return queue, repository, RequestContext.system("worker-test")
def _enqueue(
queue: QueueService,
ctx: RequestContext,
*,
task_id: str = "task-1",
task_type: str = "advanced_style_source_recognition",
metadata: dict[str, Any] | None = None,
):
return queue.create_task(
{
"task_id": task_id,
"task_type": task_type,
"status": "queued",
"progress": 0,
"request": {},
"metadata": metadata
or {
"source": "advanced_material",
"advanced_material_import_id": "import-1",
},
},
ctx,
)
def test_safe_error_message_prefers_api_error_fields() -> None:
exc = ApiError(
ErrorCode.GENERATION_FAILED,
"advanced style source recognition failed",
{"finish_reason": "prohibited_content", "reason": "no usable output"},
)
message = _safe_error_message(exc)
assert "advanced style source recognition failed" in message
assert "GENERATION_FAILED" in message
assert "prohibited_content" in message
def test_handler_api_error_marks_task_failed_without_raising(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
_enqueue(queue, ctx)
def boom(_task):
raise ApiError(
ErrorCode.GENERATION_FAILED,
"advanced style source recognition failed",
{"finish_reason": "prohibited_content"},
)
worker = QueueWorker(queue, boom, worker_id="imagestudio-advanced-material-1")
result = worker.run_once(ctx)
assert result.status == "failed"
assert result.failed == 1
assert result.task_id == "task-1"
assert "advanced style source recognition failed" in (result.error_message or "")
assert "GENERATION_FAILED" in (result.error_message or "")
job = queue.get_job("task-1")
assert job.status == "failed"
assert job.metadata.get("worker", {}).get("worker_id") == "imagestudio-advanced-material-1"
def test_update_task_secondary_failure_does_not_raise(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
_enqueue(queue, ctx, task_id="task-secondary")
def boom(_task):
raise RuntimeError("provider exploded")
worker = QueueWorker(queue, boom, worker_id="worker-secondary")
original_update = queue.update_task
def flaky_update(task_id, payload, request_ctx):
if payload.get("status") == "failed":
raise RuntimeError("sqlite is busy")
return original_update(task_id, payload, request_ctx)
queue.update_task = flaky_update # type: ignore[method-assign]
result = worker.run_once(ctx)
assert result.status == "failed"
assert result.failed == 1
assert result.task_id == "task-secondary"
assert "provider exploded" in (result.error_message or "")
def test_terminal_callback_exception_is_swallowed(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
_enqueue(queue, ctx, task_id="task-callback")
seen: list[str] = []
def ok(_task):
return WorkerTaskUpdate(status="succeeded", progress=100, result={"ok": True})
def terminal(claimed, current, exc):
seen.append(current.task_id)
raise RuntimeError("coordinator exploded")
worker = QueueWorker(
queue,
ok,
worker_id="worker-callback",
terminal_callback=terminal,
)
result = worker.run_once(ctx)
assert result.status == "succeeded"
assert result.succeeded == 1
assert seen == ["task-callback"]
assert queue.get_job("task-callback").status == "succeeded"
def test_worker_thread_survives_repeated_handler_failures(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
for index in range(3):
_enqueue(queue, ctx, task_id=f"task-loop-{index}")
def boom(_task):
raise ApiError(ErrorCode.GENERATION_FAILED, "vision blocked", {"finish_reason": "prohibited_content"})
worker = QueueWorker(
queue,
boom,
worker_id="imagestudio-advanced-material-loop",
interval_seconds=0.05,
claim_batch_size=10,
)
start = worker.start(ctx)
assert start["running"] is True
deadline = time.time() + 5.0
failed_ids: set[str] = set()
while time.time() < deadline and len(failed_ids) < 3:
for task_id in ("task-loop-0", "task-loop-1", "task-loop-2"):
job = queue.get_job(task_id)
if job.status == "failed":
failed_ids.add(task_id)
if len(failed_ids) < 3:
time.sleep(0.05)
assert worker.is_running() is True
assert failed_ids == {"task-loop-0", "task-loop-1", "task-loop-2"}
stop = worker.stop(timeout_seconds=2.0)
assert stop["running"] is False
def test_start_can_revive_after_thread_reference_is_dead(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
worker = QueueWorker(
queue,
lambda _task: WorkerTaskUpdate(),
worker_id="worker-revive",
interval_seconds=0.05,
)
first = worker.start(ctx)
assert first["running"] is True
thread = worker._thread
assert thread is not None
worker._stop_event.set()
thread.join(timeout=2.0)
assert worker.is_running() is False
second = worker.start(ctx)
assert second["running"] is True
assert second["already_running"] is False
assert worker.is_running() is True
worker.stop(timeout_seconds=2.0)
def test_run_loop_swallows_run_once_crash(tmp_path) -> None:
queue, _repository, ctx = _queue(tmp_path)
worker = QueueWorker(
queue,
lambda _task: WorkerTaskUpdate(),
worker_id="worker-loop-guard",
interval_seconds=0.05,
)
calls = {"n": 0}
original = worker.run_once
def exploding_once(request_ctx):
calls["n"] += 1
if calls["n"] == 1:
raise RuntimeError("unexpected run_once crash")
return original(request_ctx)
worker.run_once = exploding_once # type: ignore[method-assign]
worker.start(ctx)
deadline = time.time() + 2.0
while time.time() < deadline and calls["n"] < 2:
time.sleep(0.05)
assert worker.is_running() is True
assert calls["n"] >= 2
worker.stop(timeout_seconds=2.0)