from __future__ import annotations import tempfile import threading import unittest from concurrent.futures import ThreadPoolExecutor from pathlib import Path from backend.dto.common import ApiError, ErrorCode from backend.routes.job_routes import retry_dead_letter_jobs, retry_job from backend.services.generation_job_retry_service import ( MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES, GenerationJobRetryService, ) from backend.services.generation_service import GenerationService from backend.services.image_service import ImageService from backend.services.profile_service import ProfileService from backend.services.queue_service import QueueService from backend.services.workflow_service import WorkflowService from core.audit_service import AuditService from core.config_service import ConfigService from core.identity import RequestContext from core.storage import VfsStorage from database.db_helper import Database, DatabaseSettings, SqliteSettings from database.repositories.image_repository import ImageRepository from database.repositories.job_repository import JobRepository from database.repositories.profile_repository import ProfileRepository from database.repositories.workflow_repository import WorkflowRepository from engines.single_flow import SingleFlowEngine class GenerationJobRetryServiceTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.config_path = self.root / "config.yaml" self.config_path.write_text( f""" plugin: name: astrbot_plugin_imagestudio storage: images_path: "{(self.root / 'images').as_posix()}" thumbnails_path: "{(self.root / 'thumbnails').as_posix()}" materials_path: "{(self.root / 'materials').as_posix()}" styles_path: "{(self.root / 'styles').as_posix()}" archives_path: "{(self.root / 'archives').as_posix()}" temp_path: "{(self.root / 'temp').as_posix()}" """, encoding="utf-8", ) for folder in ("images", "thumbnails", "materials", "styles", "archives", "temp"): (self.root / folder).mkdir(parents=True, exist_ok=True) self.database = Database( DatabaseSettings(sqlite=SqliteSettings(path=str(self.root / "imagestudio.db"))) ) self.database.initialize() self.audit_service = AuditService(self.database) self.profile_repository = ProfileRepository(self.database) self.workflow_repository = WorkflowRepository(self.database) self.queue_service = QueueService(JobRepository(self.database), self.audit_service) self.profile_service = ProfileService(self.profile_repository, self.audit_service) self.workflow_service = WorkflowService( self.workflow_repository, self.profile_repository, self.audit_service, ) self.image_service = ImageService( ImageRepository(self.database), self.workflow_repository, self.audit_service, ) self.generation_service = GenerationService( workflow_service=self.workflow_service, queue_service=self.queue_service, image_service=self.image_service, single_flow_engine=SingleFlowEngine( VfsStorage(ConfigService(self.config_path)), allow_mock_fallback=True, ), ) self.retry_service = GenerationJobRetryService(self.queue_service, self.generation_service) self.ctx = RequestContext.from_astrbot( username="admin", user_id="user_001", permissions={"can_write", "can_retry_tasks"}, ) def tearDown(self): self.tmp.cleanup() def test_retry_executes_replacement_and_supersedes_original(self): original = self._failed_generation_task(status="failed") response = retry_job(self.ctx, original.task_id, service=self.retry_service) self.assertTrue(response["ok"]) replacement = response["data"] self.assertNotEqual(replacement["task_id"], original.task_id) self.assertEqual(replacement["status"], "succeeded") self.assertEqual(replacement["metadata"]["retry_lineage"]["retry_of_task_id"], original.task_id) superseded = self.queue_service.get_job(original.task_id) self.assertEqual(superseded.status, "superseded") self.assertEqual(superseded.result["replacement_task_id"], replacement["task_id"]) self.assertEqual(self.queue_service.queue_summary()["pending"], 0) repeated = retry_job(self.ctx, original.task_id, service=self.retry_service) self.assertFalse(repeated["ok"]) self.assertEqual(repeated["error_code"], "TASK_NOT_RETRYABLE") def test_failed_replacement_is_returned_in_error_lineage(self): original = self._failed_generation_task(status="failed") service = GenerationJobRetryService( self.queue_service, _CreatesFailedReplacement(self.queue_service, self.workflow_service), ) response = retry_job(self.ctx, original.task_id, service=service) self.assertFalse(response["ok"]) self.assertEqual(response["error_code"], "GENERATION_FAILED") replacement_task_id = response["details"]["replacement_task_id"] self.assertTrue(replacement_task_id.startswith("task_")) self.assertEqual(response["details"]["retry_lineage"]["replacement_task_id"], replacement_task_id) replacement = self.queue_service.get_job(replacement_task_id) self.assertEqual(replacement.status, "failed") self.assertEqual(replacement.metadata["retry_lineage"]["retry_of_task_id"], original.task_id) self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded") def test_preflight_failure_restores_original_status(self): original = self._failed_generation_task(status="dead_letter") service = GenerationJobRetryService( self.queue_service, _RejectsBeforeTaskCreation(self.workflow_service), ) response = retry_job(self.ctx, original.task_id, service=service) self.assertFalse(response["ok"]) self.assertTrue(response["details"]["original_status_restored"]) self.assertIsNone(response["details"]["replacement_task_id"]) restored = self.queue_service.get_job(original.task_id) self.assertEqual(restored.status, "dead_letter") self.assertEqual(restored.metadata["retry_execution"]["status"], "rejected") def test_cancelled_task_with_matching_active_node_can_retry(self): original = self._failed_generation_task(status="cancelled", node_status="running") response = retry_job(self.ctx, original.task_id, service=self.retry_service) self.assertTrue(response["ok"]) self.assertEqual(response["data"]["status"], "succeeded") self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded") def test_preflight_failure_restores_matching_active_node(self): original = self._failed_generation_task(status="cancelled", node_status="running") before_node = self.workflow_service.get_node(str(original.node_id)) service = GenerationJobRetryService( self.queue_service, _RejectsBeforeTaskCreation(self.workflow_service), ) response = retry_job(self.ctx, original.task_id, service=service) self.assertFalse(response["ok"]) self.assertTrue(response["details"]["original_status_restored"]) self.assertTrue(response["details"]["node_status_restored"]) self.assertEqual(self.queue_service.get_job(original.task_id).status, "cancelled") after_node = self.workflow_service.get_node(str(original.node_id)) self.assertEqual(after_node.status, "running") self.assertEqual(after_node.metadata, before_node.metadata) def test_retry_requires_replacement_for_ephemeral_inline_references(self): original = self._failed_generation_task(status="failed") self.queue_service.update_task( original.task_id, { "request": { **original.request, "reference_persistence": {"ephemeral_data_urls_omitted": 1}, } }, self.ctx, ) rejected = retry_job(self.ctx, original.task_id, service=self.retry_service) accepted = retry_job( self.ctx, original.task_id, {"input_images": ["https://example.com/replacement.png"]}, service=self.retry_service, ) self.assertFalse(rejected["ok"]) self.assertEqual(rejected["error_code"], "TASK_NOT_RETRYABLE") self.assertEqual(rejected["details"]["ephemeral_data_urls_omitted"], 1) self.assertTrue(accepted["ok"]) def test_retry_claim_is_compare_and_set_under_concurrency(self): task = self.queue_service.create_task({"status": "failed"}, self.ctx) barrier = threading.Barrier(2) def claim(index: int) -> str: barrier.wait(timeout=2) try: return self.queue_service.begin_retry( task.task_id, self.ctx, operation_id=f"op_{index}", ).status except ApiError as exc: return exc.error_code.value with ThreadPoolExecutor(max_workers=2) as executor: results = list(executor.map(claim, (1, 2))) self.assertEqual(sorted(results), ["TASK_NOT_RETRYABLE", "retrying"]) self.assertEqual(self.queue_service.get_job(task.task_id).status, "retrying") def test_dead_letter_retry_is_synchronous_and_capped(self): tasks = [self._failed_generation_task(status="dead_letter") for _ in range(7)] service = GenerationJobRetryService( self.queue_service, _CompletesReplacement(self.queue_service, self.workflow_service), ) response = retry_dead_letter_jobs(self.ctx, {"limit": 50}, service) self.assertTrue(response["ok"]) data = response["data"] self.assertEqual(data["execution_mode"], "synchronous") self.assertEqual(data["applied_limit"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) self.assertEqual(data["processed"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) self.assertEqual(data["total"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) self.assertEqual(data["remaining"], len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) statuses = [self.queue_service.get_job(task.task_id).status for task in tasks] self.assertEqual(statuses.count("superseded"), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) self.assertEqual(statuses.count("dead_letter"), len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES) def _failed_generation_task(self, *, status: str, node_status: str = "failed"): profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx) workflow = self.workflow_service.create_workflow( profile.profile_id, {"workflow_type": "single_image", "display_name": "Retry"}, self.ctx, ) node = workflow.nodes[0] task = self.queue_service.create_task( { "profile_id": profile.profile_id, "workflow_id": workflow.workflow.workflow_id, "node_id": node.node_id, "status": status, "model_route": "image_low", "request": {"prompt_patch": "retry portrait", "model_route": "image_low"}, "error_message": "provider outage", }, self.ctx, ) if node_status == "running": queued = self.workflow_service.update_node( node.node_id, {"status": "queued", "metadata": {**node.metadata, "last_task_id": task.task_id}}, self.ctx, ) self.workflow_service.update_node( node.node_id, {"status": "running", "metadata": queued.metadata}, self.ctx, ) else: self.workflow_service.update_node( node.node_id, {"status": node_status, "metadata": {**node.metadata, "last_task_id": task.task_id}}, self.ctx, ) return task class _CompletesReplacement: def __init__(self, queue_service: QueueService, workflow_service: WorkflowService): self.queue_service = queue_service self.workflow_service = workflow_service def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict: node = self.workflow_service.get_node(node_id) workflow = self.workflow_service.get_workflow(node.workflow_id).workflow task = self.queue_service.create_task( { "profile_id": workflow.profile_id, "workflow_id": workflow.workflow_id, "node_id": node_id, "status": "succeeded", "idempotency_key": request["idempotency_key"], "request": request, "progress": 100, }, ctx, ) return {"task_id": task.task_id, "task_status": task.status} class _CreatesFailedReplacement(_CompletesReplacement): def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict: node = self.workflow_service.get_node(node_id) workflow = self.workflow_service.get_workflow(node.workflow_id).workflow task = self.queue_service.create_task( { "profile_id": workflow.profile_id, "workflow_id": workflow.workflow_id, "node_id": node_id, "status": "failed", "idempotency_key": request["idempotency_key"], "request": request, "error_message": "provider still unavailable", }, ctx, ) raise ApiError( ErrorCode.GENERATION_FAILED, "replacement generation failed", {"provider_error": "temporary outage", "created_task_id": task.task_id}, ) class _RejectsBeforeTaskCreation: def __init__(self, workflow_service: WorkflowService): self.workflow_service = workflow_service def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict: raise ApiError( ErrorCode.MODEL_CAPABILITY_UNCONFIRMED, "model route is unavailable", {"node_id": node_id}, ) if __name__ == "__main__": unittest.main()