astrbot_plugin_imagestudio/tests/test_advanced_material_worker_recovery.py

193 行
6.5 KiB
Python

from __future__ import annotations
from backend.dto.job_dto import GenerationTask
from backend.services.advanced_material_worker_runtime import AdvancedMaterialWorkerRuntime
from backend.services.queue_service import QueueService
from core.audit_service import AuditService
from core.identity import RequestContext
from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.job_repository import JobRepository
class _RecoveryService:
def __init__(self, queue_service: QueueService):
self.queue_service = queue_service
self.failures: list[tuple[GenerationTask, Exception]] = []
def execute_task(self, _task: GenerationTask, _ctx: RequestContext):
raise AssertionError("startup recovery must run before worker execution")
def handle_task_failure(
self,
task: GenerationTask,
exc: Exception,
_ctx: RequestContext,
) -> None:
self.failures.append((task, exc))
def _runtime(tmp_path) -> tuple[AdvancedMaterialWorkerRuntime, JobRepository, _RecoveryService]:
database = Database(
DatabaseSettings(
backend="sqlite",
sqlite=SqliteSettings(path=str(tmp_path / "advanced-worker-recovery.sqlite3")),
)
)
database.initialize()
repository = JobRepository(database)
queue = QueueService(repository, AuditService(database))
service = _RecoveryService(queue)
runtime = AdvancedMaterialWorkerRuntime(service, worker_count=1) # type: ignore[arg-type]
return runtime, repository, service
def _create_task(
repository: JobRepository,
task_id: str,
*,
task_type: str,
status: str,
source: str = "advanced_material",
progress: int = 60,
) -> GenerationTask:
return repository.create(
{
"task_id": task_id,
"task_type": task_type,
"status": status,
"progress": progress,
"request": {},
"metadata": {
"source": source,
"advanced_material_import_id": f"import-{task_id}",
},
}
)
def test_fast_restart_recovers_every_fresh_advanced_running_state(tmp_path) -> None:
runtime, repository, service = _runtime(tmp_path)
recognition_running = _create_task(
repository,
"advanced-recognition-running",
task_type="advanced_material_recognition",
status="running",
)
recognition_retrying = _create_task(
repository,
"advanced-recognition-retrying",
task_type="advanced_material_recognition",
status="retrying",
)
validation_running = _create_task(
repository,
"advanced-validation-running",
task_type="advanced_material_validation_generation",
status="running",
)
validation_retrying = _create_task(
repository,
"advanced-validation-retrying",
task_type="advanced_material_validation_generation",
status="retrying",
)
unrelated = _create_task(
repository,
"workflow-round-running",
task_type="workflow_round_generation",
status="running",
source="workflow_round",
)
# No timestamp aging: this simulates a container/plugin restart in the
# same second that the previous process last wrote the queue rows.
recovery = runtime._recover_interrupted_tasks()
assert set(recovery["requeued_task_ids"]) == {
recognition_running.task_id,
recognition_retrying.task_id,
}
assert set(recovery["failed_task_ids"]) == {
validation_running.task_id,
validation_retrying.task_id,
}
assert recovery["skipped_task_ids"] == [unrelated.task_id]
for task in (recognition_running, recognition_retrying):
current = repository.get(task.task_id)
assert current is not None
assert current.status == "queued"
assert current.progress == 0
assert current.error_message is None
assert current.metadata["recovery"]["previous_status"] == task.status
assert current.metadata["recovery"]["status"] == "requeued"
for task in (validation_running, validation_retrying):
current = repository.get(task.task_id)
assert current is not None
assert current.status == "failed"
assert current.progress == task.progress
assert current.error_message == "task was interrupted by a previous plugin process"
assert current.metadata["recovery"]["previous_status"] == task.status
assert "status" not in current.metadata["recovery"]
current_unrelated = repository.get(unrelated.task_id)
assert current_unrelated is not None
assert current_unrelated.status == "running"
assert {task.task_id for task, _exc in service.failures} == {
validation_running.task_id,
validation_retrying.task_id,
}
def test_startup_snapshot_is_not_truncated_by_unrelated_tasks(tmp_path) -> None:
runtime, repository, _service = _runtime(tmp_path)
database = repository.database
conn = database.connect()
try:
for index in range(105):
conn.execute(
"""
INSERT INTO generation_tasks (
task_id, task_type, status, progress, request_json, metadata_json,
created_at, updated_at
) VALUES (?, ?, 'running', 0, '{}', ?, ?, ?)
""",
(
f"a-unrelated-{index:03d}",
"workflow_round_generation",
'{"source":"workflow_round"}',
"2020-01-01 00:00:00",
"2020-01-01 00:00:00",
),
)
conn.commit()
finally:
conn.close()
advanced = _create_task(
repository,
"z-advanced-after-first-page",
task_type="advanced_material_recognition",
status="running",
)
conn = database.connect()
try:
conn.execute(
"""
UPDATE generation_tasks
SET created_at = ?, updated_at = ?
WHERE task_id = ?
""",
("2020-01-01 00:00:00", "2020-01-01 00:00:00", advanced.task_id),
)
conn.commit()
finally:
conn.close()
recovery = runtime._recover_interrupted_tasks()
assert recovery["requeued_task_ids"] == [advanced.task_id]
assert len(recovery["skipped_task_ids"]) == 105
current = repository.get(advanced.task_id)
assert current is not None
assert current.status == "queued"