498 行
17 KiB
Python
498 行
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import replace
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from adapters.astrbot_adapter import resolve_route
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
from backend.dto.job_dto import GenerationTask
|
|
from backend.dto.workflow_dto import ProfileWorkflow, WorkflowDetail, WorkflowNode
|
|
from backend.services.generation_job_retry_service import GenerationJobRetryService
|
|
from backend.services.queue_service import QueueService
|
|
from core.audit_service import AuditService
|
|
from core.identity import RequestContext
|
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
|
from database.repositories.job_repository import JobRepository
|
|
|
|
|
|
def _database(tmp_path) -> Database:
|
|
database = Database(
|
|
DatabaseSettings(
|
|
backend="sqlite",
|
|
sqlite=SqliteSettings(path=str(tmp_path / "queue-recovery.sqlite3")),
|
|
)
|
|
)
|
|
database.initialize()
|
|
return database
|
|
|
|
|
|
def _insert_workflow_fixture(
|
|
database: Database,
|
|
*,
|
|
task_id: str = "task-failed",
|
|
task_status: str = "failed",
|
|
node_status: str = "failed",
|
|
workflow_status: str = "failed",
|
|
) -> JobRepository:
|
|
conn = database.connect()
|
|
try:
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO character_profiles (
|
|
profile_id, name, owner_platform, owner_user_id, status, metadata_json
|
|
) VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
("profile-1", "Queue test", "webui", "system", "active", "{}"),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO profile_workflows (
|
|
workflow_id, profile_id, workflow_type, display_name, status, current_branch, metadata_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
("workflow-1", "profile-1", "character_fast", "Fast character", workflow_status, "main", "{}"),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO workflow_nodes (
|
|
node_id, workflow_id, node_key, title, node_type, status, branch_name, metadata_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
"node-1",
|
|
"workflow-1",
|
|
"quick_avatar",
|
|
"Quick avatar",
|
|
"generation",
|
|
node_status,
|
|
"main",
|
|
json.dumps(
|
|
{
|
|
"last_task_id": task_id,
|
|
"last_generate_request": {"prompt": "test"},
|
|
"generation_failure": {"message": "upstream failed"},
|
|
"retry_preparation": {"task_id": task_id},
|
|
"keep": "node metadata",
|
|
}
|
|
),
|
|
),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
repository = JobRepository(database)
|
|
repository.create(
|
|
{
|
|
"task_id": task_id,
|
|
"profile_id": "profile-1",
|
|
"workflow_id": "workflow-1",
|
|
"node_id": "node-1",
|
|
"status": task_status,
|
|
"request": {"prompt": "test"},
|
|
"metadata": {},
|
|
}
|
|
)
|
|
return repository
|
|
|
|
|
|
def test_delete_failed_task_releases_owned_node_reference(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
repository = _insert_workflow_fixture(database)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
result = service.delete_job("task-failed", RequestContext.system("test"))
|
|
|
|
assert result["deleted"] is True
|
|
assert result["node_cleanup"]["released"] is True
|
|
assert result["workflow_cleanup"]["recovered"] is True
|
|
assert repository.get("task-failed") is None
|
|
conn = database.connect()
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?",
|
|
("node-1",),
|
|
).fetchone()
|
|
workflow_row = conn.execute(
|
|
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
|
|
("workflow-1",),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
metadata = json.loads(row["metadata_json"])
|
|
assert row["status"] == "draft"
|
|
assert metadata == {"keep": "node metadata"}
|
|
assert workflow_row["status"] == "running"
|
|
|
|
|
|
def test_delete_cancelled_task_does_not_reopen_cancelled_workflow(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
repository = _insert_workflow_fixture(
|
|
database,
|
|
task_id="task-cancelled",
|
|
task_status="cancelled",
|
|
node_status="cancelled",
|
|
workflow_status="cancelled",
|
|
)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
result = service.delete_job("task-cancelled", RequestContext.system("test"))
|
|
|
|
assert result["deleted"] is True
|
|
assert result["workflow_cleanup"]["recovered"] is False
|
|
conn = database.connect()
|
|
try:
|
|
workflow_row = conn.execute(
|
|
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
|
|
("workflow-1",),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
assert workflow_row["status"] == "cancelled"
|
|
|
|
|
|
def test_delete_completed_task_keeps_completed_node_and_workflow(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
repository = _insert_workflow_fixture(
|
|
database,
|
|
task_id="task-succeeded",
|
|
task_status="succeeded",
|
|
node_status="completed",
|
|
workflow_status="completed",
|
|
)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
result = service.delete_job("task-succeeded", RequestContext.system("test"))
|
|
|
|
assert result["deleted"] is True
|
|
assert result["workflow_cleanup"]["recovered"] is False
|
|
conn = database.connect()
|
|
try:
|
|
node_row = conn.execute(
|
|
"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = ?",
|
|
("node-1",),
|
|
).fetchone()
|
|
workflow_row = conn.execute(
|
|
"SELECT status FROM profile_workflows WHERE workflow_id = ?",
|
|
("workflow-1",),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
assert node_row["status"] == "completed"
|
|
assert json.loads(node_row["metadata_json"]) == {"keep": "node metadata"}
|
|
assert workflow_row["status"] == "completed"
|
|
|
|
|
|
def test_delete_running_task_is_rejected_without_mutation(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
repository = _insert_workflow_fixture(
|
|
database,
|
|
task_id="task-running",
|
|
task_status="running",
|
|
node_status="running",
|
|
)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
with pytest.raises(ApiError) as captured:
|
|
service.delete_job("task-running", RequestContext.system("test"))
|
|
|
|
assert captured.value.error_code == ErrorCode.VALIDATION_FAILED
|
|
assert repository.get("task-running") is not None
|
|
|
|
|
|
def test_clear_failed_jobs_counts_errors_toward_batch_limit(tmp_path, monkeypatch) -> None:
|
|
database = _database(tmp_path)
|
|
repository = JobRepository(database)
|
|
for status in ("failed", "dead_letter", "cancelled"):
|
|
for index in range(2):
|
|
repository.create(
|
|
{
|
|
"task_id": f"task-{status}-{index}",
|
|
"status": status,
|
|
"request": {},
|
|
"metadata": {},
|
|
}
|
|
)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
def fail_delete(task_id: str, _ctx: RequestContext) -> dict[str, Any]:
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "simulated compare-and-delete race", {"task_id": task_id})
|
|
|
|
monkeypatch.setattr(service, "delete_job", fail_delete)
|
|
|
|
result = service.clear_failed_jobs(RequestContext.system("test"), limit=2)
|
|
|
|
assert result["processed"] == 2
|
|
assert len(result["errors"]) == 2
|
|
assert result["deleted"] == []
|
|
|
|
|
|
def test_clear_terminal_jobs_can_target_completed_records_without_touching_active_tasks(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
repository = JobRepository(database)
|
|
statuses = ("completed", "succeeded", "superseded", "failed", "queued", "running", "retrying")
|
|
for status in statuses:
|
|
repository.create(
|
|
{
|
|
"task_id": f"task-{status}",
|
|
"status": status,
|
|
"request": {},
|
|
"metadata": {},
|
|
}
|
|
)
|
|
service = QueueService(repository, AuditService(database))
|
|
|
|
result = service.clear_terminal_jobs(
|
|
RequestContext.system("test"),
|
|
statuses=["completed", "succeeded", "superseded"],
|
|
limit=100,
|
|
)
|
|
|
|
assert result["scope"] == "terminal"
|
|
assert result["requested_statuses"] == ["completed", "succeeded", "superseded"]
|
|
assert result["total"] == 3
|
|
assert result["remaining_by_status"] == {"completed": 0, "succeeded": 0, "superseded": 0}
|
|
for status in ("completed", "succeeded", "superseded"):
|
|
assert repository.get(f"task-{status}") is None
|
|
for status in ("failed", "queued", "running", "retrying"):
|
|
assert repository.get(f"task-{status}") is not None
|
|
|
|
all_terminal_result = service.clear_terminal_jobs(RequestContext.system("test"), limit=100)
|
|
|
|
assert all_terminal_result["requested_statuses"] == [
|
|
"failed",
|
|
"dead_letter",
|
|
"cancelled",
|
|
"completed",
|
|
"succeeded",
|
|
"superseded",
|
|
]
|
|
assert all_terminal_result["total"] == 1
|
|
assert repository.get("task-failed") is None
|
|
for status in ("queued", "running", "retrying"):
|
|
assert repository.get(f"task-{status}") is not None
|
|
|
|
|
|
def test_clear_terminal_jobs_rejects_active_statuses(tmp_path) -> None:
|
|
database = _database(tmp_path)
|
|
service = QueueService(JobRepository(database), AuditService(database))
|
|
|
|
with pytest.raises(ApiError) as captured:
|
|
service.clear_terminal_jobs(
|
|
RequestContext.system("test"),
|
|
statuses=["completed", "running"],
|
|
)
|
|
|
|
assert captured.value.error_code == ErrorCode.VALIDATION_FAILED
|
|
assert captured.value.details["unsupported_statuses"] == ["running"]
|
|
|
|
|
|
def test_static_failed_clear_route_is_not_shadowed_by_task_route() -> None:
|
|
match = resolve_route("POST", "jobs/failed/clear")
|
|
|
|
assert match is not None
|
|
assert match.endpoint == "jobs/failed/clear"
|
|
assert match.handler_path.endswith(".clear_failed_jobs")
|
|
|
|
|
|
def test_static_terminal_clear_route_is_not_shadowed_by_task_route() -> None:
|
|
match = resolve_route("POST", "jobs/terminal/clear")
|
|
|
|
assert match is not None
|
|
assert match.endpoint == "jobs/terminal/clear"
|
|
assert match.handler_path.endswith(".clear_terminal_jobs")
|
|
|
|
|
|
def _task(*, status: str = "failed", metadata: dict[str, Any] | None = None) -> GenerationTask:
|
|
return GenerationTask(
|
|
task_id="task-original",
|
|
profile_id="profile-1",
|
|
workflow_id="workflow-1",
|
|
node_id="node-1",
|
|
task_type="generation",
|
|
status=status,
|
|
model_route="image-route",
|
|
progress=100,
|
|
idempotency_key="original-key",
|
|
request={"prompt": "test"},
|
|
result={},
|
|
error_message="provider failed",
|
|
metadata=metadata or {"keep": "task metadata"},
|
|
created_at="2026-07-14T00:00:00Z",
|
|
updated_at="2026-07-14T00:00:00Z",
|
|
)
|
|
|
|
|
|
def _workflow() -> ProfileWorkflow:
|
|
return ProfileWorkflow(
|
|
workflow_id="workflow-1",
|
|
profile_id="profile-1",
|
|
workflow_type="character_fast",
|
|
display_name="Fast character",
|
|
mode=None,
|
|
status="failed",
|
|
current_node_id="node-1",
|
|
current_branch="main",
|
|
session_key=None,
|
|
metadata={"keep": "workflow metadata"},
|
|
created_at="2026-07-14T00:00:00Z",
|
|
updated_at="2026-07-14T00:00:00Z",
|
|
)
|
|
|
|
|
|
def _node() -> WorkflowNode:
|
|
return WorkflowNode(
|
|
node_id="node-1",
|
|
workflow_id="workflow-1",
|
|
node_key="quick_avatar",
|
|
title="Quick avatar",
|
|
description=None,
|
|
step_name="quick_avatar",
|
|
node_type="generation",
|
|
status="queued",
|
|
branch_name="main",
|
|
prompt="test",
|
|
negative_prompt=None,
|
|
model_route="image-route",
|
|
selected_image_id=None,
|
|
sort_order=10,
|
|
metadata={"last_task_id": "task-original", "keep": "node metadata"},
|
|
created_at="2026-07-14T00:00:00Z",
|
|
updated_at="2026-07-14T00:00:00Z",
|
|
)
|
|
|
|
|
|
class _QueueStub:
|
|
def __init__(self) -> None:
|
|
self.tasks = {"task-original": _task()}
|
|
|
|
def get_job(self, task_id: str) -> GenerationTask:
|
|
return self.tasks[task_id]
|
|
|
|
def get_job_by_idempotency_key(self, _idempotency_key: str) -> GenerationTask | None:
|
|
return None
|
|
|
|
def begin_retry(self, task_id: str, _ctx: RequestContext, *, operation_id: str) -> GenerationTask:
|
|
before = self.tasks[task_id]
|
|
after = replace(
|
|
before,
|
|
status="retrying",
|
|
progress=0,
|
|
metadata={
|
|
**before.metadata,
|
|
"retry_execution": {
|
|
"operation_id": operation_id,
|
|
"status": "running",
|
|
"previous_status": before.status,
|
|
},
|
|
},
|
|
)
|
|
self.tasks[task_id] = after
|
|
return after
|
|
|
|
def update_task(self, task_id: str, payload: dict[str, Any], _ctx: RequestContext) -> GenerationTask:
|
|
self.tasks[task_id] = replace(self.tasks[task_id], **payload)
|
|
return self.tasks[task_id]
|
|
|
|
def update_task_if_status(
|
|
self,
|
|
task_id: str,
|
|
expected_statuses,
|
|
payload: dict[str, Any],
|
|
ctx: RequestContext,
|
|
**_kwargs: Any,
|
|
) -> GenerationTask | None:
|
|
if self.tasks[task_id].status not in set(expected_statuses):
|
|
return None
|
|
return self.update_task(task_id, payload, ctx)
|
|
|
|
|
|
class _AuditStub:
|
|
def record(self, **_kwargs: Any) -> None:
|
|
return None
|
|
|
|
|
|
class _WorkflowRepositoryStub:
|
|
def __init__(self, service: "_WorkflowServiceStub") -> None:
|
|
self.service = service
|
|
|
|
def update_node(self, node_id: str, payload: dict[str, Any]) -> WorkflowNode | None:
|
|
if node_id != self.service.node.node_id:
|
|
return None
|
|
self.service.node = replace(self.service.node, **payload)
|
|
return self.service.node
|
|
|
|
|
|
class _WorkflowServiceStub:
|
|
def __init__(self) -> None:
|
|
self.workflow = _workflow()
|
|
self.node = _node()
|
|
self.workflow_repository = _WorkflowRepositoryStub(self)
|
|
self.audit_service = _AuditStub()
|
|
|
|
def get_workflow(self, _workflow_id: str) -> WorkflowDetail:
|
|
return WorkflowDetail(self.workflow, [self.node])
|
|
|
|
def get_node(self, _node_id: str) -> WorkflowNode:
|
|
return self.node
|
|
|
|
def update_workflow(
|
|
self,
|
|
_workflow_id: str,
|
|
payload: dict[str, Any],
|
|
_ctx: RequestContext,
|
|
) -> ProfileWorkflow:
|
|
self.workflow = replace(self.workflow, **payload)
|
|
return self.workflow
|
|
|
|
def update_node(self, _node_id: str, payload: dict[str, Any], _ctx: RequestContext) -> WorkflowNode:
|
|
self.node = replace(self.node, **payload)
|
|
return self.node
|
|
|
|
def _publish(self, _event_type: str, _payload: dict[str, Any]) -> None:
|
|
return None
|
|
|
|
|
|
class _FailingGenerationService:
|
|
def __init__(self, workflow_service: _WorkflowServiceStub) -> None:
|
|
self.workflow_service = workflow_service
|
|
|
|
def generate_node(self, _node_id: str, _request: dict[str, Any], _ctx: RequestContext) -> dict[str, Any]:
|
|
# Simulate unrelated metadata arriving while the retry is prepared.
|
|
# Rollback must remove only its own marker, not restore stale snapshots.
|
|
self.workflow_service.workflow = replace(
|
|
self.workflow_service.workflow,
|
|
metadata={**self.workflow_service.workflow.metadata, "concurrent": "workflow update"},
|
|
)
|
|
self.workflow_service.node = replace(
|
|
self.workflow_service.node,
|
|
metadata={**self.workflow_service.node.metadata, "concurrent": "node update"},
|
|
)
|
|
raise RuntimeError("provider failed before replacement task creation")
|
|
|
|
|
|
def test_retry_failure_restores_task_node_and_workflow_without_losing_metadata() -> None:
|
|
queue = _QueueStub()
|
|
workflow_service = _WorkflowServiceStub()
|
|
service = GenerationJobRetryService(queue, _FailingGenerationService(workflow_service))
|
|
|
|
with pytest.raises(ApiError) as captured:
|
|
service.retry_job("task-original", RequestContext.system("test"))
|
|
|
|
assert captured.value.error_code == ErrorCode.GENERATION_FAILED
|
|
assert queue.get_job("task-original").status == "failed"
|
|
assert queue.get_job("task-original").metadata["retry_execution"]["status"] == "rejected"
|
|
assert workflow_service.workflow.status == "failed"
|
|
assert workflow_service.workflow.metadata == {
|
|
"keep": "workflow metadata",
|
|
"concurrent": "workflow update",
|
|
}
|
|
assert workflow_service.node.status == "queued"
|
|
assert workflow_service.node.metadata == {
|
|
"last_task_id": "task-original",
|
|
"keep": "node metadata",
|
|
"concurrent": "node update",
|
|
}
|