astrbot_plugin_imagestudio/tests/test_queue_retry_recovery.py

498 行
17 KiB
Python

from __future__ import annotations
import json
from dataclasses import replace
from typing import Any
import pytest
from adapters.astrbot_adapter import resolve_route
from backend.dto.common import ApiError, ErrorCode
from backend.dto.job_dto import GenerationTask
from backend.dto.workflow_dto import ProfileWorkflow, WorkflowDetail, WorkflowNode
from backend.services.generation_job_retry_service import GenerationJobRetryService
from backend.services.queue_service import QueueService
from core.audit_service import AuditService
from core.identity import RequestContext
from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.job_repository import JobRepository
def _database(tmp_path) -> Database:
database = Database(
DatabaseSettings(
backend="sqlite",
sqlite=SqliteSettings(path=str(tmp_path / "queue-recovery.sqlite3")),
)
)
database.initialize()
return database
def _insert_workflow_fixture(
database: Database,
*,
task_id: str = "task-failed",
task_status: str = "failed",
node_status: str = "failed",
workflow_status: str = "failed",
) -> JobRepository:
conn = database.connect()
try:
conn.execute(
"""
INSERT INTO character_profiles (
profile_id, name, owner_platform, owner_user_id, status, metadata_json
) VALUES (?, ?, ?, ?, ?, ?)
""",
("profile-1", "Queue test", "webui", "system", "active", "{}"),
)
conn.execute(
"""
INSERT INTO profile_workflows (
workflow_id, profile_id, workflow_type, display_name, status, current_branch, metadata_json
) VALUES (?, ?, ?, ?, ?, ?, ?)
""",
("workflow-1", "profile-1", "character_fast", "Fast character", workflow_status, "main", "{}"),
)
conn.execute(
"""
INSERT INTO workflow_nodes (
node_id, workflow_id, node_key, title, node_type, status, branch_name, metadata_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
"node-1",
"workflow-1",
"quick_avatar",
"Quick avatar",
"generation",
node_status,
"main",
json.dumps(
{
"last_task_id": task_id,
"last_generate_request": {"prompt": "test"},
"generation_failure": {"message": "upstream failed"},
"retry_preparation": {"task_id": task_id},
"keep": "node metadata",
}
),
),
)
conn.commit()
finally:
conn.close()
repository = JobRepository(database)
repository.create(
{
"task_id": task_id,
"profile_id": "profile-1",
"workflow_id": "workflow-1",
"node_id": "node-1",
"status": task_status,
"request": {"prompt": "test"},
"metadata": {},
}
)
return repository
def test_delete_failed_task_releases_owned_node_reference(tmp_path) -> None:
database = _database(tmp_path)
repository = _insert_workflow_fixture(database)
service = QueueService(repository, AuditService(database))
result = service.delete_job("task-failed", RequestContext.system("test"))
assert result["deleted"] is True
assert result["node_cleanup"]["released"] is True
assert result["workflow_cleanup"]["recovered"] is True
assert repository.get("task-failed") is None
conn = database.connect()
try:
row = conn.execute(
"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?",
("node-1",),
).fetchone()
workflow_row = conn.execute(
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
("workflow-1",),
).fetchone()
finally:
conn.close()
metadata = json.loads(row["metadata_json"])
assert row["status"] == "draft"
assert metadata == {"keep": "node metadata"}
assert workflow_row["status"] == "running"
def test_delete_cancelled_task_does_not_reopen_cancelled_workflow(tmp_path) -> None:
database = _database(tmp_path)
repository = _insert_workflow_fixture(
database,
task_id="task-cancelled",
task_status="cancelled",
node_status="cancelled",
workflow_status="cancelled",
)
service = QueueService(repository, AuditService(database))
result = service.delete_job("task-cancelled", RequestContext.system("test"))
assert result["deleted"] is True
assert result["workflow_cleanup"]["recovered"] is False
conn = database.connect()
try:
workflow_row = conn.execute(
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
("workflow-1",),
).fetchone()
finally:
conn.close()
assert workflow_row["status"] == "cancelled"
def test_delete_completed_task_keeps_completed_node_and_workflow(tmp_path) -> None:
database = _database(tmp_path)
repository = _insert_workflow_fixture(
database,
task_id="task-succeeded",
task_status="succeeded",
node_status="completed",
workflow_status="completed",
)
service = QueueService(repository, AuditService(database))
result = service.delete_job("task-succeeded", RequestContext.system("test"))
assert result["deleted"] is True
assert result["workflow_cleanup"]["recovered"] is False
conn = database.connect()
try:
node_row = conn.execute(
"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?",
("node-1",),
).fetchone()
workflow_row = conn.execute(
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
("workflow-1",),
).fetchone()
finally:
conn.close()
assert node_row["status"] == "completed"
assert json.loads(node_row["metadata_json"]) == {"keep": "node metadata"}
assert workflow_row["status"] == "completed"
def test_delete_running_task_is_rejected_without_mutation(tmp_path) -> None:
database = _database(tmp_path)
repository = _insert_workflow_fixture(
database,
task_id="task-running",
task_status="running",
node_status="running",
)
service = QueueService(repository, AuditService(database))
with pytest.raises(ApiError) as captured:
service.delete_job("task-running", RequestContext.system("test"))
assert captured.value.error_code == ErrorCode.VALIDATION_FAILED
assert repository.get("task-running") is not None
def test_clear_failed_jobs_counts_errors_toward_batch_limit(tmp_path, monkeypatch) -> None:
database = _database(tmp_path)
repository = JobRepository(database)
for status in ("failed", "dead_letter", "cancelled"):
for index in range(2):
repository.create(
{
"task_id": f"task-{status}-{index}",
"status": status,
"request": {},
"metadata": {},
}
)
service = QueueService(repository, AuditService(database))
def fail_delete(task_id: str, _ctx: RequestContext) -> dict[str, Any]:
raise ApiError(ErrorCode.VALIDATION_FAILED, "simulated compare-and-delete race", {"task_id": task_id})
monkeypatch.setattr(service, "delete_job", fail_delete)
result = service.clear_failed_jobs(RequestContext.system("test"), limit=2)
assert result["processed"] == 2
assert len(result["errors"]) == 2
assert result["deleted"] == []
def test_clear_terminal_jobs_can_target_completed_records_without_touching_active_tasks(tmp_path) -> None:
database = _database(tmp_path)
repository = JobRepository(database)
statuses = ("completed", "succeeded", "superseded", "failed", "queued", "running", "retrying")
for status in statuses:
repository.create(
{
"task_id": f"task-{status}",
"status": status,
"request": {},
"metadata": {},
}
)
service = QueueService(repository, AuditService(database))
result = service.clear_terminal_jobs(
RequestContext.system("test"),
statuses=["completed", "succeeded", "superseded"],
limit=100,
)
assert result["scope"] == "terminal"
assert result["requested_statuses"] == ["completed", "succeeded", "superseded"]
assert result["total"] == 3
assert result["remaining_by_status"] == {"completed": 0, "succeeded": 0, "superseded": 0}
for status in ("completed", "succeeded", "superseded"):
assert repository.get(f"task-{status}") is None
for status in ("failed", "queued", "running", "retrying"):
assert repository.get(f"task-{status}") is not None
all_terminal_result = service.clear_terminal_jobs(RequestContext.system("test"), limit=100)
assert all_terminal_result["requested_statuses"] == [
"failed",
"dead_letter",
"cancelled",
"completed",
"succeeded",
"superseded",
]
assert all_terminal_result["total"] == 1
assert repository.get("task-failed") is None
for status in ("queued", "running", "retrying"):
assert repository.get(f"task-{status}") is not None
def test_clear_terminal_jobs_rejects_active_statuses(tmp_path) -> None:
database = _database(tmp_path)
service = QueueService(JobRepository(database), AuditService(database))
with pytest.raises(ApiError) as captured:
service.clear_terminal_jobs(
RequestContext.system("test"),
statuses=["completed", "running"],
)
assert captured.value.error_code == ErrorCode.VALIDATION_FAILED
assert captured.value.details["unsupported_statuses"] == ["running"]
def test_static_failed_clear_route_is_not_shadowed_by_task_route() -> None:
match = resolve_route("POST", "jobs/failed/clear")
assert match is not None
assert match.endpoint == "jobs/failed/clear"
assert match.handler_path.endswith(".clear_failed_jobs")
def test_static_terminal_clear_route_is_not_shadowed_by_task_route() -> None:
match = resolve_route("POST", "jobs/terminal/clear")
assert match is not None
assert match.endpoint == "jobs/terminal/clear"
assert match.handler_path.endswith(".clear_terminal_jobs")
def _task(*, status: str = "failed", metadata: dict[str, Any] | None = None) -> GenerationTask:
return GenerationTask(
task_id="task-original",
profile_id="profile-1",
workflow_id="workflow-1",
node_id="node-1",
task_type="generation",
status=status,
model_route="image-route",
progress=100,
idempotency_key="original-key",
request={"prompt": "test"},
result={},
error_message="provider failed",
metadata=metadata or {"keep": "task metadata"},
created_at="2026-07-14T00:00:00Z",
updated_at="2026-07-14T00:00:00Z",
)
def _workflow() -> ProfileWorkflow:
return ProfileWorkflow(
workflow_id="workflow-1",
profile_id="profile-1",
workflow_type="character_fast",
display_name="Fast character",
mode=None,
status="failed",
current_node_id="node-1",
current_branch="main",
session_key=None,
metadata={"keep": "workflow metadata"},
created_at="2026-07-14T00:00:00Z",
updated_at="2026-07-14T00:00:00Z",
)
def _node() -> WorkflowNode:
return WorkflowNode(
node_id="node-1",
workflow_id="workflow-1",
node_key="quick_avatar",
title="Quick avatar",
description=None,
step_name="quick_avatar",
node_type="generation",
status="queued",
branch_name="main",
prompt="test",
negative_prompt=None,
model_route="image-route",
selected_image_id=None,
sort_order=10,
metadata={"last_task_id": "task-original", "keep": "node metadata"},
created_at="2026-07-14T00:00:00Z",
updated_at="2026-07-14T00:00:00Z",
)
class _QueueStub:
def __init__(self) -> None:
self.tasks = {"task-original": _task()}
def get_job(self, task_id: str) -> GenerationTask:
return self.tasks[task_id]
def get_job_by_idempotency_key(self, _idempotency_key: str) -> GenerationTask | None:
return None
def begin_retry(self, task_id: str, _ctx: RequestContext, *, operation_id: str) -> GenerationTask:
before = self.tasks[task_id]
after = replace(
before,
status="retrying",
progress=0,
metadata={
**before.metadata,
"retry_execution": {
"operation_id": operation_id,
"status": "running",
"previous_status": before.status,
},
},
)
self.tasks[task_id] = after
return after
def update_task(self, task_id: str, payload: dict[str, Any], _ctx: RequestContext) -> GenerationTask:
self.tasks[task_id] = replace(self.tasks[task_id], **payload)
return self.tasks[task_id]
def update_task_if_status(
self,
task_id: str,
expected_statuses,
payload: dict[str, Any],
ctx: RequestContext,
**_kwargs: Any,
) -> GenerationTask | None:
if self.tasks[task_id].status not in set(expected_statuses):
return None
return self.update_task(task_id, payload, ctx)
class _AuditStub:
def record(self, **_kwargs: Any) -> None:
return None
class _WorkflowRepositoryStub:
def __init__(self, service: "_WorkflowServiceStub") -> None:
self.service = service
def update_node(self, node_id: str, payload: dict[str, Any]) -> WorkflowNode | None:
if node_id != self.service.node.node_id:
return None
self.service.node = replace(self.service.node, **payload)
return self.service.node
class _WorkflowServiceStub:
def __init__(self) -> None:
self.workflow = _workflow()
self.node = _node()
self.workflow_repository = _WorkflowRepositoryStub(self)
self.audit_service = _AuditStub()
def get_workflow(self, _workflow_id: str) -> WorkflowDetail:
return WorkflowDetail(self.workflow, [self.node])
def get_node(self, _node_id: str) -> WorkflowNode:
return self.node
def update_workflow(
self,
_workflow_id: str,
payload: dict[str, Any],
_ctx: RequestContext,
) -> ProfileWorkflow:
self.workflow = replace(self.workflow, **payload)
return self.workflow
def update_node(self, _node_id: str, payload: dict[str, Any], _ctx: RequestContext) -> WorkflowNode:
self.node = replace(self.node, **payload)
return self.node
def _publish(self, _event_type: str, _payload: dict[str, Any]) -> None:
return None
class _FailingGenerationService:
def __init__(self, workflow_service: _WorkflowServiceStub) -> None:
self.workflow_service = workflow_service
def generate_node(self, _node_id: str, _request: dict[str, Any], _ctx: RequestContext) -> dict[str, Any]:
# Simulate unrelated metadata arriving while the retry is prepared.
# Rollback must remove only its own marker, not restore stale snapshots.
self.workflow_service.workflow = replace(
self.workflow_service.workflow,
metadata={**self.workflow_service.workflow.metadata, "concurrent": "workflow update"},
)
self.workflow_service.node = replace(
self.workflow_service.node,
metadata={**self.workflow_service.node.metadata, "concurrent": "node update"},
)
raise RuntimeError("provider failed before replacement task creation")
def test_retry_failure_restores_task_node_and_workflow_without_losing_metadata() -> None:
queue = _QueueStub()
workflow_service = _WorkflowServiceStub()
service = GenerationJobRetryService(queue, _FailingGenerationService(workflow_service))
with pytest.raises(ApiError) as captured:
service.retry_job("task-original", RequestContext.system("test"))
assert captured.value.error_code == ErrorCode.GENERATION_FAILED
assert queue.get_job("task-original").status == "failed"
assert queue.get_job("task-original").metadata["retry_execution"]["status"] == "rejected"
assert workflow_service.workflow.status == "failed"
assert workflow_service.workflow.metadata == {
"keep": "workflow metadata",
"concurrent": "workflow update",
}
assert workflow_service.node.status == "queued"
assert workflow_service.node.metadata == {
"last_task_id": "task-original",
"keep": "node metadata",
"concurrent": "node update",
}