from __future__ import annotations from backend.dto.job_dto import GenerationTask from backend.services.advanced_material_worker_runtime import AdvancedMaterialWorkerRuntime from backend.services.queue_service import QueueService from core.audit_service import AuditService from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.job_repository import JobRepository class _RecoveryService: def __init__(self, queue_service: QueueService): self.queue_service = queue_service self.failures: list[tuple[GenerationTask, Exception]] = [] def execute_task(self, _task: GenerationTask, _ctx: RequestContext): raise AssertionError("startup recovery must run before worker execution") def handle_task_failure( self, task: GenerationTask, exc: Exception, _ctx: RequestContext, ) -> None: self.failures.append((task, exc)) def _runtime(tmp_path) -> tuple[AdvancedMaterialWorkerRuntime, JobRepository, _RecoveryService]: database = Database( DatabaseSettings( backend="sqlite", sqlite=SqliteSettings(path=str(tmp_path / "advanced-worker-recovery.sqlite3")), ) ) database.initialize() repository = JobRepository(database) queue = QueueService(repository, AuditService(database)) service = _RecoveryService(queue) runtime = AdvancedMaterialWorkerRuntime(service, worker_count=1) # type: ignore[arg-type] return runtime, repository, service def _create_task( repository: JobRepository, task_id: str, *, task_type: str, status: str, source: str = "advanced_material", progress: int = 60, ) -> GenerationTask: return repository.create( { "task_id": task_id, "task_type": task_type, "status": status, "progress": progress, "request": {}, "metadata": { "source": source, "advanced_material_import_id": f"import-{task_id}", }, } ) def test_fast_restart_recovers_every_fresh_advanced_running_state(tmp_path) -> None: runtime, repository, service = _runtime(tmp_path) recognition_running = _create_task( repository, "advanced-recognition-running", task_type="advanced_material_recognition", status="running", ) recognition_retrying = _create_task( repository, "advanced-recognition-retrying", task_type="advanced_material_recognition", status="retrying", ) validation_running = _create_task( repository, "advanced-validation-running", task_type="advanced_material_validation_generation", status="running", ) validation_retrying = _create_task( repository, "advanced-validation-retrying", task_type="advanced_material_validation_generation", status="retrying", ) unrelated = _create_task( repository, "workflow-round-running", task_type="workflow_round_generation", status="running", source="workflow_round", ) # No timestamp aging: this simulates a container/plugin restart in the # same second that the previous process last wrote the queue rows. recovery = runtime._recover_interrupted_tasks() assert set(recovery["requeued_task_ids"]) == { recognition_running.task_id, recognition_retrying.task_id, } assert set(recovery["failed_task_ids"]) == { validation_running.task_id, validation_retrying.task_id, } assert recovery["skipped_task_ids"] == [unrelated.task_id] for task in (recognition_running, recognition_retrying): current = repository.get(task.task_id) assert current is not None assert current.status == "queued" assert current.progress == 0 assert current.error_message is None assert current.metadata["recovery"]["previous_status"] == task.status assert current.metadata["recovery"]["status"] == "requeued" for task in (validation_running, validation_retrying): current = repository.get(task.task_id) assert current is not None assert current.status == "failed" assert current.progress == task.progress assert current.error_message == "task was interrupted by a previous plugin process" assert current.metadata["recovery"]["previous_status"] == task.status assert "status" not in current.metadata["recovery"] current_unrelated = repository.get(unrelated.task_id) assert current_unrelated is not None assert current_unrelated.status == "running" assert {task.task_id for task, _exc in service.failures} == { validation_running.task_id, validation_retrying.task_id, } def test_startup_snapshot_is_not_truncated_by_unrelated_tasks(tmp_path) -> None: runtime, repository, _service = _runtime(tmp_path) database = repository.database conn = database.connect() try: for index in range(105): conn.execute( """ INSERT INTO generation_tasks ( task_id, task_type, status, progress, request_json, metadata_json, created_at, updated_at ) VALUES (?, ?, 'running', 0, '{}', ?, ?, ?) """, ( f"a-unrelated-{index:03d}", "workflow_round_generation", '{"source":"workflow_round"}', "2020-01-01 00:00:00", "2020-01-01 00:00:00", ), ) conn.commit() finally: conn.close() advanced = _create_task( repository, "z-advanced-after-first-page", task_type="advanced_material_recognition", status="running", ) conn = database.connect() try: conn.execute( """ UPDATE generation_tasks SET created_at = ?, updated_at = ? WHERE task_id = ? """, ("2020-01-01 00:00:00", "2020-01-01 00:00:00", advanced.task_id), ) conn.commit() finally: conn.close() recovery = runtime._recover_interrupted_tasks() assert recovery["requeued_task_ids"] == [advanced.task_id] assert len(recovery["skipped_task_ids"]) == 105 current = repository.get(advanced.task_id) assert current is not None assert current.status == "queued" from backend.services.advanced_material_worker_runtime import _recognition_concurrency def test_recognition_concurrency_defaults_to_two_for_vision_relays() -> None: assert _recognition_concurrency({}) == 2 assert _recognition_concurrency({"advanced_materials": {}}) == 2 def test_recognition_concurrency_is_bounded() -> None: assert _recognition_concurrency({"advanced_materials": {"recognition_concurrency": 1}}) == 1 assert _recognition_concurrency({"advanced_materials": {"recognition_concurrency": 99}}) == 8 assert _recognition_concurrency({"advanced_materials": {"recognition_concurrency": "invalid"}}) == 2