astrbot_plugin_imagestudio/tests/test_generation_job_retry_service.py

339 行
14 KiB
Python

from __future__ import annotations
import tempfile
import threading
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from backend.dto.common import ApiError, ErrorCode
from backend.routes.job_routes import retry_dead_letter_jobs, retry_job
from backend.services.generation_job_retry_service import (
MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES,
GenerationJobRetryService,
)
from backend.services.generation_service import GenerationService
from backend.services.image_service import ImageService
from backend.services.profile_service import ProfileService
from backend.services.queue_service import QueueService
from backend.services.workflow_service import WorkflowService
from core.audit_service import AuditService
from core.config_service import ConfigService
from core.identity import RequestContext
from core.storage import VfsStorage
from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.image_repository import ImageRepository
from database.repositories.job_repository import JobRepository
from database.repositories.profile_repository import ProfileRepository
from database.repositories.workflow_repository import WorkflowRepository
from engines.single_flow import SingleFlowEngine
class GenerationJobRetryServiceTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
self.config_path = self.root / "config.yaml"
self.config_path.write_text(
f"""
plugin:
name: astrbot_plugin_imagestudio
storage:
images_path: "{(self.root / 'images').as_posix()}"
thumbnails_path: "{(self.root / 'thumbnails').as_posix()}"
materials_path: "{(self.root / 'materials').as_posix()}"
styles_path: "{(self.root / 'styles').as_posix()}"
archives_path: "{(self.root / 'archives').as_posix()}"
temp_path: "{(self.root / 'temp').as_posix()}"
""",
encoding="utf-8",
)
for folder in ("images", "thumbnails", "materials", "styles", "archives", "temp"):
(self.root / folder).mkdir(parents=True, exist_ok=True)
self.database = Database(
DatabaseSettings(sqlite=SqliteSettings(path=str(self.root / "imagestudio.db")))
)
self.database.initialize()
self.audit_service = AuditService(self.database)
self.profile_repository = ProfileRepository(self.database)
self.workflow_repository = WorkflowRepository(self.database)
self.queue_service = QueueService(JobRepository(self.database), self.audit_service)
self.profile_service = ProfileService(self.profile_repository, self.audit_service)
self.workflow_service = WorkflowService(
self.workflow_repository,
self.profile_repository,
self.audit_service,
)
self.image_service = ImageService(
ImageRepository(self.database),
self.workflow_repository,
self.audit_service,
)
self.generation_service = GenerationService(
workflow_service=self.workflow_service,
queue_service=self.queue_service,
image_service=self.image_service,
single_flow_engine=SingleFlowEngine(
VfsStorage(ConfigService(self.config_path)),
allow_mock_fallback=True,
),
)
self.retry_service = GenerationJobRetryService(self.queue_service, self.generation_service)
self.ctx = RequestContext.from_astrbot(
username="admin",
user_id="user_001",
permissions={"can_write", "can_retry_tasks"},
)
def tearDown(self):
self.tmp.cleanup()
def test_retry_executes_replacement_and_supersedes_original(self):
original = self._failed_generation_task(status="failed")
response = retry_job(self.ctx, original.task_id, service=self.retry_service)
self.assertTrue(response["ok"])
replacement = response["data"]
self.assertNotEqual(replacement["task_id"], original.task_id)
self.assertEqual(replacement["status"], "succeeded")
self.assertEqual(replacement["metadata"]["retry_lineage"]["retry_of_task_id"], original.task_id)
superseded = self.queue_service.get_job(original.task_id)
self.assertEqual(superseded.status, "superseded")
self.assertEqual(superseded.result["replacement_task_id"], replacement["task_id"])
self.assertEqual(self.queue_service.queue_summary()["pending"], 0)
repeated = retry_job(self.ctx, original.task_id, service=self.retry_service)
self.assertFalse(repeated["ok"])
self.assertEqual(repeated["error_code"], "TASK_NOT_RETRYABLE")
def test_failed_replacement_is_returned_in_error_lineage(self):
original = self._failed_generation_task(status="failed")
service = GenerationJobRetryService(
self.queue_service,
_CreatesFailedReplacement(self.queue_service, self.workflow_service),
)
response = retry_job(self.ctx, original.task_id, service=service)
self.assertFalse(response["ok"])
self.assertEqual(response["error_code"], "GENERATION_FAILED")
replacement_task_id = response["details"]["replacement_task_id"]
self.assertTrue(replacement_task_id.startswith("task_"))
self.assertEqual(response["details"]["retry_lineage"]["replacement_task_id"], replacement_task_id)
replacement = self.queue_service.get_job(replacement_task_id)
self.assertEqual(replacement.status, "failed")
self.assertEqual(replacement.metadata["retry_lineage"]["retry_of_task_id"], original.task_id)
self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded")
def test_preflight_failure_restores_original_status(self):
original = self._failed_generation_task(status="dead_letter")
service = GenerationJobRetryService(
self.queue_service,
_RejectsBeforeTaskCreation(self.workflow_service),
)
response = retry_job(self.ctx, original.task_id, service=service)
self.assertFalse(response["ok"])
self.assertTrue(response["details"]["original_status_restored"])
self.assertIsNone(response["details"]["replacement_task_id"])
restored = self.queue_service.get_job(original.task_id)
self.assertEqual(restored.status, "dead_letter")
self.assertEqual(restored.metadata["retry_execution"]["status"], "rejected")
def test_cancelled_task_with_matching_active_node_can_retry(self):
original = self._failed_generation_task(status="cancelled", node_status="running")
response = retry_job(self.ctx, original.task_id, service=self.retry_service)
self.assertTrue(response["ok"])
self.assertEqual(response["data"]["status"], "succeeded")
self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded")
def test_preflight_failure_restores_matching_active_node(self):
original = self._failed_generation_task(status="cancelled", node_status="running")
before_node = self.workflow_service.get_node(str(original.node_id))
service = GenerationJobRetryService(
self.queue_service,
_RejectsBeforeTaskCreation(self.workflow_service),
)
response = retry_job(self.ctx, original.task_id, service=service)
self.assertFalse(response["ok"])
self.assertTrue(response["details"]["original_status_restored"])
self.assertTrue(response["details"]["node_status_restored"])
self.assertEqual(self.queue_service.get_job(original.task_id).status, "cancelled")
after_node = self.workflow_service.get_node(str(original.node_id))
self.assertEqual(after_node.status, "running")
self.assertEqual(after_node.metadata, before_node.metadata)
def test_retry_requires_replacement_for_ephemeral_inline_references(self):
original = self._failed_generation_task(status="failed")
self.queue_service.update_task(
original.task_id,
{
"request": {
**original.request,
"reference_persistence": {"ephemeral_data_urls_omitted": 1},
}
},
self.ctx,
)
rejected = retry_job(self.ctx, original.task_id, service=self.retry_service)
accepted = retry_job(
self.ctx,
original.task_id,
{"input_images": ["https://example.com/replacement.png"]},
service=self.retry_service,
)
self.assertFalse(rejected["ok"])
self.assertEqual(rejected["error_code"], "TASK_NOT_RETRYABLE")
self.assertEqual(rejected["details"]["ephemeral_data_urls_omitted"], 1)
self.assertTrue(accepted["ok"])
def test_retry_claim_is_compare_and_set_under_concurrency(self):
task = self.queue_service.create_task({"status": "failed"}, self.ctx)
barrier = threading.Barrier(2)
def claim(index: int) -> str:
barrier.wait(timeout=2)
try:
return self.queue_service.begin_retry(
task.task_id,
self.ctx,
operation_id=f"op_{index}",
).status
except ApiError as exc:
return exc.error_code.value
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(claim, (1, 2)))
self.assertEqual(sorted(results), ["TASK_NOT_RETRYABLE", "retrying"])
self.assertEqual(self.queue_service.get_job(task.task_id).status, "retrying")
def test_dead_letter_retry_is_synchronous_and_capped(self):
tasks = [self._failed_generation_task(status="dead_letter") for _ in range(7)]
service = GenerationJobRetryService(
self.queue_service,
_CompletesReplacement(self.queue_service, self.workflow_service),
)
response = retry_dead_letter_jobs(self.ctx, {"limit": 50}, service)
self.assertTrue(response["ok"])
data = response["data"]
self.assertEqual(data["execution_mode"], "synchronous")
self.assertEqual(data["applied_limit"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
self.assertEqual(data["processed"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
self.assertEqual(data["total"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
self.assertEqual(data["remaining"], len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
statuses = [self.queue_service.get_job(task.task_id).status for task in tasks]
self.assertEqual(statuses.count("superseded"), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
self.assertEqual(statuses.count("dead_letter"), len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
def _failed_generation_task(self, *, status: str, node_status: str = "failed"):
profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx)
workflow = self.workflow_service.create_workflow(
profile.profile_id,
{"workflow_type": "single_image", "display_name": "Retry"},
self.ctx,
)
node = workflow.nodes[0]
task = self.queue_service.create_task(
{
"profile_id": profile.profile_id,
"workflow_id": workflow.workflow.workflow_id,
"node_id": node.node_id,
"status": status,
"model_route": "image_low",
"request": {"prompt_patch": "retry portrait", "model_route": "image_low"},
"error_message": "provider outage",
},
self.ctx,
)
if node_status == "running":
queued = self.workflow_service.update_node(
node.node_id,
{"status": "queued", "metadata": {**node.metadata, "last_task_id": task.task_id}},
self.ctx,
)
self.workflow_service.update_node(
node.node_id,
{"status": "running", "metadata": queued.metadata},
self.ctx,
)
else:
self.workflow_service.update_node(
node.node_id,
{"status": node_status, "metadata": {**node.metadata, "last_task_id": task.task_id}},
self.ctx,
)
return task
class _CompletesReplacement:
def __init__(self, queue_service: QueueService, workflow_service: WorkflowService):
self.queue_service = queue_service
self.workflow_service = workflow_service
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
node = self.workflow_service.get_node(node_id)
workflow = self.workflow_service.get_workflow(node.workflow_id).workflow
task = self.queue_service.create_task(
{
"profile_id": workflow.profile_id,
"workflow_id": workflow.workflow_id,
"node_id": node_id,
"status": "succeeded",
"idempotency_key": request["idempotency_key"],
"request": request,
"progress": 100,
},
ctx,
)
return {"task_id": task.task_id, "task_status": task.status}
class _CreatesFailedReplacement(_CompletesReplacement):
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
node = self.workflow_service.get_node(node_id)
workflow = self.workflow_service.get_workflow(node.workflow_id).workflow
task = self.queue_service.create_task(
{
"profile_id": workflow.profile_id,
"workflow_id": workflow.workflow_id,
"node_id": node_id,
"status": "failed",
"idempotency_key": request["idempotency_key"],
"request": request,
"error_message": "provider still unavailable",
},
ctx,
)
raise ApiError(
ErrorCode.GENERATION_FAILED,
"replacement generation failed",
{"provider_error": "temporary outage", "created_task_id": task.task_id},
)
class _RejectsBeforeTaskCreation:
def __init__(self, workflow_service: WorkflowService):
self.workflow_service = workflow_service
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
raise ApiError(
ErrorCode.MODEL_CAPABILITY_UNCONFIRMED,
"model route is unavailable",
{"node_id": node_id},
)
if __name__ == "__main__":
unittest.main()