from __future__ import annotations import json from dataclasses import replace from typing import Any import pytest from adapters.astrbot_adapter import resolve_route from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask from backend.dto.workflow_dto import ProfileWorkflow, WorkflowDetail, WorkflowNode from backend.services.generation_job_retry_service import GenerationJobRetryService from backend.services.queue_service import QueueService from core.audit_service import AuditService from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.job_repository import JobRepository def _database(tmp_path) -> Database: database = Database( DatabaseSettings( backend="sqlite", sqlite=SqliteSettings(path=str(tmp_path / "queue-recovery.sqlite3")), ) ) database.initialize() return database def _insert_workflow_fixture( database: Database, *, task_id: str = "task-failed", task_status: str = "failed", node_status: str = "failed", workflow_status: str = "failed", ) -> JobRepository: conn = database.connect() try: conn.execute( """ INSERT INTO character_profiles ( profile_id, name, owner_platform, owner_user_id, status, metadata_json ) VALUES (?, ?, ?, ?, ?, ?) """, ("profile-1", "Queue test", "webui", "system", "active", "{}"), ) conn.execute( """ INSERT INTO profile_workflows ( workflow_id, profile_id, workflow_type, display_name, status, current_branch, metadata_json ) VALUES (?, ?, ?, ?, ?, ?, ?) """, ("workflow-1", "profile-1", "character_fast", "Fast character", workflow_status, "main", "{}"), ) conn.execute( """ INSERT INTO workflow_nodes ( node_id, workflow_id, node_key, title, node_type, status, branch_name, metadata_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, ( "node-1", "workflow-1", "quick_avatar", "Quick avatar", "generation", node_status, "main", json.dumps( { "last_task_id": task_id, "last_generate_request": {"prompt": "test"}, "generation_failure": {"message": "upstream failed"}, "retry_preparation": {"task_id": task_id}, "keep": "node metadata", } ), ), ) conn.commit() finally: conn.close() repository = JobRepository(database) repository.create( { "task_id": task_id, "profile_id": "profile-1", "workflow_id": "workflow-1", "node_id": "node-1", "status": task_status, "request": {"prompt": "test"}, "metadata": {}, } ) return repository def test_delete_failed_task_releases_owned_node_reference(tmp_path) -> None: database = _database(tmp_path) repository = _insert_workflow_fixture(database) service = QueueService(repository, AuditService(database)) result = service.delete_job("task-failed", RequestContext.system("test")) assert result["deleted"] is True assert result["node_cleanup"]["released"] is True assert result["workflow_cleanup"]["recovered"] is True assert repository.get("task-failed") is None conn = database.connect() try: row = conn.execute( "SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?", ("node-1",), ).fetchone() workflow_row = conn.execute( "SELECT status FROM profile_workflows WHERE workflow_id = ?", ("workflow-1",), ).fetchone() finally: conn.close() metadata = json.loads(row["metadata_json"]) assert row["status"] == "draft" assert metadata == {"keep": "node metadata"} assert workflow_row["status"] == "running" def test_delete_cancelled_task_does_not_reopen_cancelled_workflow(tmp_path) -> None: database = _database(tmp_path) repository = _insert_workflow_fixture( database, task_id="task-cancelled", task_status="cancelled", node_status="cancelled", workflow_status="cancelled", ) service = QueueService(repository, AuditService(database)) result = service.delete_job("task-cancelled", RequestContext.system("test")) assert result["deleted"] is True assert result["workflow_cleanup"]["recovered"] is False conn = database.connect() try: workflow_row = conn.execute( "SELECT status FROM profile_workflows WHERE workflow_id = ?", ("workflow-1",), ).fetchone() finally: conn.close() assert workflow_row["status"] == "cancelled" def test_delete_completed_task_keeps_completed_node_and_workflow(tmp_path) -> None: database = _database(tmp_path) repository = _insert_workflow_fixture( database, task_id="task-succeeded", task_status="succeeded", node_status="completed", workflow_status="completed", ) service = QueueService(repository, AuditService(database)) result = service.delete_job("task-succeeded", RequestContext.system("test")) assert result["deleted"] is True assert result["workflow_cleanup"]["recovered"] is False conn = database.connect() try: node_row = conn.execute( "SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?", ("node-1",), ).fetchone() workflow_row = conn.execute( "SELECT status FROM profile_workflows WHERE workflow_id = ?", ("workflow-1",), ).fetchone() finally: conn.close() assert node_row["status"] == "completed" assert json.loads(node_row["metadata_json"]) == {"keep": "node metadata"} assert workflow_row["status"] == "completed" def test_delete_running_task_is_rejected_without_mutation(tmp_path) -> None: database = _database(tmp_path) repository = _insert_workflow_fixture( database, task_id="task-running", task_status="running", node_status="running", ) service = QueueService(repository, AuditService(database)) with pytest.raises(ApiError) as captured: service.delete_job("task-running", RequestContext.system("test")) assert captured.value.error_code == ErrorCode.VALIDATION_FAILED assert repository.get("task-running") is not None def test_clear_failed_jobs_counts_errors_toward_batch_limit(tmp_path, monkeypatch) -> None: database = _database(tmp_path) repository = JobRepository(database) for status in ("failed", "dead_letter", "cancelled"): for index in range(2): repository.create( { "task_id": f"task-{status}-{index}", "status": status, "request": {}, "metadata": {}, } ) service = QueueService(repository, AuditService(database)) def fail_delete(task_id: str, _ctx: RequestContext) -> dict[str, Any]: raise ApiError(ErrorCode.VALIDATION_FAILED, "simulated compare-and-delete race", {"task_id": task_id}) monkeypatch.setattr(service, "delete_job", fail_delete) result = service.clear_failed_jobs(RequestContext.system("test"), limit=2) assert result["processed"] == 2 assert len(result["errors"]) == 2 assert result["deleted"] == [] def test_clear_terminal_jobs_can_target_completed_records_without_touching_active_tasks(tmp_path) -> None: database = _database(tmp_path) repository = JobRepository(database) statuses = ("completed", "succeeded", "superseded", "failed", "queued", "running", "retrying") for status in statuses: repository.create( { "task_id": f"task-{status}", "status": status, "request": {}, "metadata": {}, } ) service = QueueService(repository, AuditService(database)) result = service.clear_terminal_jobs( RequestContext.system("test"), statuses=["completed", "succeeded", "superseded"], limit=100, ) assert result["scope"] == "terminal" assert result["requested_statuses"] == ["completed", "succeeded", "superseded"] assert result["total"] == 3 assert result["remaining_by_status"] == {"completed": 0, "succeeded": 0, "superseded": 0} for status in ("completed", "succeeded", "superseded"): assert repository.get(f"task-{status}") is None for status in ("failed", "queued", "running", "retrying"): assert repository.get(f"task-{status}") is not None all_terminal_result = service.clear_terminal_jobs(RequestContext.system("test"), limit=100) assert all_terminal_result["requested_statuses"] == [ "failed", "dead_letter", "cancelled", "completed", "succeeded", "superseded", ] assert all_terminal_result["total"] == 1 assert repository.get("task-failed") is None for status in ("queued", "running", "retrying"): assert repository.get(f"task-{status}") is not None def test_clear_terminal_jobs_rejects_active_statuses(tmp_path) -> None: database = _database(tmp_path) service = QueueService(JobRepository(database), AuditService(database)) with pytest.raises(ApiError) as captured: service.clear_terminal_jobs( RequestContext.system("test"), statuses=["completed", "running"], ) assert captured.value.error_code == ErrorCode.VALIDATION_FAILED assert captured.value.details["unsupported_statuses"] == ["running"] def test_static_failed_clear_route_is_not_shadowed_by_task_route() -> None: match = resolve_route("POST", "jobs/failed/clear") assert match is not None assert match.endpoint == "jobs/failed/clear" assert match.handler_path.endswith(".clear_failed_jobs") def test_static_terminal_clear_route_is_not_shadowed_by_task_route() -> None: match = resolve_route("POST", "jobs/terminal/clear") assert match is not None assert match.endpoint == "jobs/terminal/clear" assert match.handler_path.endswith(".clear_terminal_jobs") def _task(*, status: str = "failed", metadata: dict[str, Any] | None = None) -> GenerationTask: return GenerationTask( task_id="task-original", profile_id="profile-1", workflow_id="workflow-1", node_id="node-1", task_type="generation", status=status, model_route="image-route", progress=100, idempotency_key="original-key", request={"prompt": "test"}, result={}, error_message="provider failed", metadata=metadata or {"keep": "task metadata"}, created_at="2026-07-14T00:00:00Z", updated_at="2026-07-14T00:00:00Z", ) def _workflow() -> ProfileWorkflow: return ProfileWorkflow( workflow_id="workflow-1", profile_id="profile-1", workflow_type="character_fast", display_name="Fast character", mode=None, status="failed", current_node_id="node-1", current_branch="main", session_key=None, metadata={"keep": "workflow metadata"}, created_at="2026-07-14T00:00:00Z", updated_at="2026-07-14T00:00:00Z", ) def _node() -> WorkflowNode: return WorkflowNode( node_id="node-1", workflow_id="workflow-1", node_key="quick_avatar", title="Quick avatar", description=None, step_name="quick_avatar", node_type="generation", status="queued", branch_name="main", prompt="test", negative_prompt=None, model_route="image-route", selected_image_id=None, sort_order=10, metadata={"last_task_id": "task-original", "keep": "node metadata"}, created_at="2026-07-14T00:00:00Z", updated_at="2026-07-14T00:00:00Z", ) class _QueueStub: def __init__(self) -> None: self.tasks = {"task-original": _task()} def get_job(self, task_id: str) -> GenerationTask: return self.tasks[task_id] def get_job_by_idempotency_key(self, _idempotency_key: str) -> GenerationTask | None: return None def begin_retry(self, task_id: str, _ctx: RequestContext, *, operation_id: str) -> GenerationTask: before = self.tasks[task_id] after = replace( before, status="retrying", progress=0, metadata={ **before.metadata, "retry_execution": { "operation_id": operation_id, "status": "running", "previous_status": before.status, }, }, ) self.tasks[task_id] = after return after def update_task(self, task_id: str, payload: dict[str, Any], _ctx: RequestContext) -> GenerationTask: self.tasks[task_id] = replace(self.tasks[task_id], **payload) return self.tasks[task_id] def update_task_if_status( self, task_id: str, expected_statuses, payload: dict[str, Any], ctx: RequestContext, **_kwargs: Any, ) -> GenerationTask | None: if self.tasks[task_id].status not in set(expected_statuses): return None return self.update_task(task_id, payload, ctx) class _AuditStub: def record(self, **_kwargs: Any) -> None: return None class _WorkflowRepositoryStub: def __init__(self, service: "_WorkflowServiceStub") -> None: self.service = service def update_node(self, node_id: str, payload: dict[str, Any]) -> WorkflowNode | None: if node_id != self.service.node.node_id: return None self.service.node = replace(self.service.node, **payload) return self.service.node class _WorkflowServiceStub: def __init__(self) -> None: self.workflow = _workflow() self.node = _node() self.workflow_repository = _WorkflowRepositoryStub(self) self.audit_service = _AuditStub() def get_workflow(self, _workflow_id: str) -> WorkflowDetail: return WorkflowDetail(self.workflow, [self.node]) def get_node(self, _node_id: str) -> WorkflowNode: return self.node def update_workflow( self, _workflow_id: str, payload: dict[str, Any], _ctx: RequestContext, ) -> ProfileWorkflow: self.workflow = replace(self.workflow, **payload) return self.workflow def update_node(self, _node_id: str, payload: dict[str, Any], _ctx: RequestContext) -> WorkflowNode: self.node = replace(self.node, **payload) return self.node def _publish(self, _event_type: str, _payload: dict[str, Any]) -> None: return None class _FailingGenerationService: def __init__(self, workflow_service: _WorkflowServiceStub) -> None: self.workflow_service = workflow_service def generate_node(self, _node_id: str, _request: dict[str, Any], _ctx: RequestContext) -> dict[str, Any]: # Simulate unrelated metadata arriving while the retry is prepared. # Rollback must remove only its own marker, not restore stale snapshots. self.workflow_service.workflow = replace( self.workflow_service.workflow, metadata={**self.workflow_service.workflow.metadata, "concurrent": "workflow update"}, ) self.workflow_service.node = replace( self.workflow_service.node, metadata={**self.workflow_service.node.metadata, "concurrent": "node update"}, ) raise RuntimeError("provider failed before replacement task creation") def test_retry_failure_restores_task_node_and_workflow_without_losing_metadata() -> None: queue = _QueueStub() workflow_service = _WorkflowServiceStub() service = GenerationJobRetryService(queue, _FailingGenerationService(workflow_service)) with pytest.raises(ApiError) as captured: service.retry_job("task-original", RequestContext.system("test")) assert captured.value.error_code == ErrorCode.GENERATION_FAILED assert queue.get_job("task-original").status == "failed" assert queue.get_job("task-original").metadata["retry_execution"]["status"] == "rejected" assert workflow_service.workflow.status == "failed" assert workflow_service.workflow.metadata == { "keep": "workflow metadata", "concurrent": "workflow update", } assert workflow_service.node.status == "queued" assert workflow_service.node.metadata == { "last_task_id": "task-original", "keep": "node metadata", "concurrent": "node update", }