193 行
6.5 KiB
Python
193 行
6.5 KiB
Python
from __future__ import annotations
|
|
|
|
from backend.dto.job_dto import GenerationTask
|
|
from backend.services.advanced_material_worker_runtime import AdvancedMaterialWorkerRuntime
|
|
from backend.services.queue_service import QueueService
|
|
from core.audit_service import AuditService
|
|
from core.identity import RequestContext
|
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
|
from database.repositories.job_repository import JobRepository
|
|
|
|
|
|
class _RecoveryService:
|
|
def __init__(self, queue_service: QueueService):
|
|
self.queue_service = queue_service
|
|
self.failures: list[tuple[GenerationTask, Exception]] = []
|
|
|
|
def execute_task(self, _task: GenerationTask, _ctx: RequestContext):
|
|
raise AssertionError("startup recovery must run before worker execution")
|
|
|
|
def handle_task_failure(
|
|
self,
|
|
task: GenerationTask,
|
|
exc: Exception,
|
|
_ctx: RequestContext,
|
|
) -> None:
|
|
self.failures.append((task, exc))
|
|
|
|
|
|
def _runtime(tmp_path) -> tuple[AdvancedMaterialWorkerRuntime, JobRepository, _RecoveryService]:
|
|
database = Database(
|
|
DatabaseSettings(
|
|
backend="sqlite",
|
|
sqlite=SqliteSettings(path=str(tmp_path / "advanced-worker-recovery.sqlite3")),
|
|
)
|
|
)
|
|
database.initialize()
|
|
repository = JobRepository(database)
|
|
queue = QueueService(repository, AuditService(database))
|
|
service = _RecoveryService(queue)
|
|
runtime = AdvancedMaterialWorkerRuntime(service, worker_count=1) # type: ignore[arg-type]
|
|
return runtime, repository, service
|
|
|
|
|
|
def _create_task(
|
|
repository: JobRepository,
|
|
task_id: str,
|
|
*,
|
|
task_type: str,
|
|
status: str,
|
|
source: str = "advanced_material",
|
|
progress: int = 60,
|
|
) -> GenerationTask:
|
|
return repository.create(
|
|
{
|
|
"task_id": task_id,
|
|
"task_type": task_type,
|
|
"status": status,
|
|
"progress": progress,
|
|
"request": {},
|
|
"metadata": {
|
|
"source": source,
|
|
"advanced_material_import_id": f"import-{task_id}",
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
def test_fast_restart_recovers_every_fresh_advanced_running_state(tmp_path) -> None:
|
|
runtime, repository, service = _runtime(tmp_path)
|
|
recognition_running = _create_task(
|
|
repository,
|
|
"advanced-recognition-running",
|
|
task_type="advanced_material_recognition",
|
|
status="running",
|
|
)
|
|
recognition_retrying = _create_task(
|
|
repository,
|
|
"advanced-recognition-retrying",
|
|
task_type="advanced_material_recognition",
|
|
status="retrying",
|
|
)
|
|
validation_running = _create_task(
|
|
repository,
|
|
"advanced-validation-running",
|
|
task_type="advanced_material_validation_generation",
|
|
status="running",
|
|
)
|
|
validation_retrying = _create_task(
|
|
repository,
|
|
"advanced-validation-retrying",
|
|
task_type="advanced_material_validation_generation",
|
|
status="retrying",
|
|
)
|
|
unrelated = _create_task(
|
|
repository,
|
|
"workflow-round-running",
|
|
task_type="workflow_round_generation",
|
|
status="running",
|
|
source="workflow_round",
|
|
)
|
|
|
|
# No timestamp aging: this simulates a container/plugin restart in the
|
|
# same second that the previous process last wrote the queue rows.
|
|
recovery = runtime._recover_interrupted_tasks()
|
|
|
|
assert set(recovery["requeued_task_ids"]) == {
|
|
recognition_running.task_id,
|
|
recognition_retrying.task_id,
|
|
}
|
|
assert set(recovery["failed_task_ids"]) == {
|
|
validation_running.task_id,
|
|
validation_retrying.task_id,
|
|
}
|
|
assert recovery["skipped_task_ids"] == [unrelated.task_id]
|
|
|
|
for task in (recognition_running, recognition_retrying):
|
|
current = repository.get(task.task_id)
|
|
assert current is not None
|
|
assert current.status == "queued"
|
|
assert current.progress == 0
|
|
assert current.error_message is None
|
|
assert current.metadata["recovery"]["previous_status"] == task.status
|
|
assert current.metadata["recovery"]["status"] == "requeued"
|
|
|
|
for task in (validation_running, validation_retrying):
|
|
current = repository.get(task.task_id)
|
|
assert current is not None
|
|
assert current.status == "failed"
|
|
assert current.progress == task.progress
|
|
assert current.error_message == "task was interrupted by a previous plugin process"
|
|
assert current.metadata["recovery"]["previous_status"] == task.status
|
|
assert "status" not in current.metadata["recovery"]
|
|
|
|
current_unrelated = repository.get(unrelated.task_id)
|
|
assert current_unrelated is not None
|
|
assert current_unrelated.status == "running"
|
|
assert {task.task_id for task, _exc in service.failures} == {
|
|
validation_running.task_id,
|
|
validation_retrying.task_id,
|
|
}
|
|
|
|
|
|
def test_startup_snapshot_is_not_truncated_by_unrelated_tasks(tmp_path) -> None:
|
|
runtime, repository, _service = _runtime(tmp_path)
|
|
database = repository.database
|
|
conn = database.connect()
|
|
try:
|
|
for index in range(105):
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO generation_tasks (
|
|
task_id, task_type, status, progress, request_json, metadata_json,
|
|
created_at, updated_at
|
|
) VALUES (?, ?, 'running', 0, '{}', ?, ?, ?)
|
|
""",
|
|
(
|
|
f"a-unrelated-{index:03d}",
|
|
"workflow_round_generation",
|
|
'{"source":"workflow_round"}',
|
|
"2020-01-01 00:00:00",
|
|
"2020-01-01 00:00:00",
|
|
),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
advanced = _create_task(
|
|
repository,
|
|
"z-advanced-after-first-page",
|
|
task_type="advanced_material_recognition",
|
|
status="running",
|
|
)
|
|
conn = database.connect()
|
|
try:
|
|
conn.execute(
|
|
"""
|
|
UPDATE generation_tasks
|
|
SET created_at = ?, updated_at = ?
|
|
WHERE task_id = ?
|
|
""",
|
|
("2020-01-01 00:00:00", "2020-01-01 00:00:00", advanced.task_id),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
recovery = runtime._recover_interrupted_tasks()
|
|
|
|
assert recovery["requeued_task_ids"] == [advanced.task_id]
|
|
assert len(recovery["skipped_task_ids"]) == 105
|
|
current = repository.get(advanced.task_id)
|
|
assert current is not None
|
|
assert current.status == "queued"
|