339 行
14 KiB
Python
339 行
14 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
from backend.routes.job_routes import retry_dead_letter_jobs, retry_job
|
|
from backend.services.generation_job_retry_service import (
|
|
MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES,
|
|
GenerationJobRetryService,
|
|
)
|
|
from backend.services.generation_service import GenerationService
|
|
from backend.services.image_service import ImageService
|
|
from backend.services.profile_service import ProfileService
|
|
from backend.services.queue_service import QueueService
|
|
from backend.services.workflow_service import WorkflowService
|
|
from core.audit_service import AuditService
|
|
from core.config_service import ConfigService
|
|
from core.identity import RequestContext
|
|
from core.storage import VfsStorage
|
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
|
from database.repositories.image_repository import ImageRepository
|
|
from database.repositories.job_repository import JobRepository
|
|
from database.repositories.profile_repository import ProfileRepository
|
|
from database.repositories.workflow_repository import WorkflowRepository
|
|
from engines.single_flow import SingleFlowEngine
|
|
|
|
|
|
class GenerationJobRetryServiceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.config_path = self.root / "config.yaml"
|
|
self.config_path.write_text(
|
|
f"""
|
|
plugin:
|
|
name: astrbot_plugin_imagestudio
|
|
|
|
storage:
|
|
images_path: "{(self.root / 'images').as_posix()}"
|
|
thumbnails_path: "{(self.root / 'thumbnails').as_posix()}"
|
|
materials_path: "{(self.root / 'materials').as_posix()}"
|
|
styles_path: "{(self.root / 'styles').as_posix()}"
|
|
archives_path: "{(self.root / 'archives').as_posix()}"
|
|
temp_path: "{(self.root / 'temp').as_posix()}"
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
for folder in ("images", "thumbnails", "materials", "styles", "archives", "temp"):
|
|
(self.root / folder).mkdir(parents=True, exist_ok=True)
|
|
self.database = Database(
|
|
DatabaseSettings(sqlite=SqliteSettings(path=str(self.root / "imagestudio.db")))
|
|
)
|
|
self.database.initialize()
|
|
self.audit_service = AuditService(self.database)
|
|
self.profile_repository = ProfileRepository(self.database)
|
|
self.workflow_repository = WorkflowRepository(self.database)
|
|
self.queue_service = QueueService(JobRepository(self.database), self.audit_service)
|
|
self.profile_service = ProfileService(self.profile_repository, self.audit_service)
|
|
self.workflow_service = WorkflowService(
|
|
self.workflow_repository,
|
|
self.profile_repository,
|
|
self.audit_service,
|
|
)
|
|
self.image_service = ImageService(
|
|
ImageRepository(self.database),
|
|
self.workflow_repository,
|
|
self.audit_service,
|
|
)
|
|
self.generation_service = GenerationService(
|
|
workflow_service=self.workflow_service,
|
|
queue_service=self.queue_service,
|
|
image_service=self.image_service,
|
|
single_flow_engine=SingleFlowEngine(
|
|
VfsStorage(ConfigService(self.config_path)),
|
|
allow_mock_fallback=True,
|
|
),
|
|
)
|
|
self.retry_service = GenerationJobRetryService(self.queue_service, self.generation_service)
|
|
self.ctx = RequestContext.from_astrbot(
|
|
username="admin",
|
|
user_id="user_001",
|
|
permissions={"can_write", "can_retry_tasks"},
|
|
)
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def test_retry_executes_replacement_and_supersedes_original(self):
|
|
original = self._failed_generation_task(status="failed")
|
|
|
|
response = retry_job(self.ctx, original.task_id, service=self.retry_service)
|
|
|
|
self.assertTrue(response["ok"])
|
|
replacement = response["data"]
|
|
self.assertNotEqual(replacement["task_id"], original.task_id)
|
|
self.assertEqual(replacement["status"], "succeeded")
|
|
self.assertEqual(replacement["metadata"]["retry_lineage"]["retry_of_task_id"], original.task_id)
|
|
superseded = self.queue_service.get_job(original.task_id)
|
|
self.assertEqual(superseded.status, "superseded")
|
|
self.assertEqual(superseded.result["replacement_task_id"], replacement["task_id"])
|
|
self.assertEqual(self.queue_service.queue_summary()["pending"], 0)
|
|
|
|
repeated = retry_job(self.ctx, original.task_id, service=self.retry_service)
|
|
self.assertFalse(repeated["ok"])
|
|
self.assertEqual(repeated["error_code"], "TASK_NOT_RETRYABLE")
|
|
|
|
def test_failed_replacement_is_returned_in_error_lineage(self):
|
|
original = self._failed_generation_task(status="failed")
|
|
service = GenerationJobRetryService(
|
|
self.queue_service,
|
|
_CreatesFailedReplacement(self.queue_service, self.workflow_service),
|
|
)
|
|
|
|
response = retry_job(self.ctx, original.task_id, service=service)
|
|
|
|
self.assertFalse(response["ok"])
|
|
self.assertEqual(response["error_code"], "GENERATION_FAILED")
|
|
replacement_task_id = response["details"]["replacement_task_id"]
|
|
self.assertTrue(replacement_task_id.startswith("task_"))
|
|
self.assertEqual(response["details"]["retry_lineage"]["replacement_task_id"], replacement_task_id)
|
|
replacement = self.queue_service.get_job(replacement_task_id)
|
|
self.assertEqual(replacement.status, "failed")
|
|
self.assertEqual(replacement.metadata["retry_lineage"]["retry_of_task_id"], original.task_id)
|
|
self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded")
|
|
|
|
def test_preflight_failure_restores_original_status(self):
|
|
original = self._failed_generation_task(status="dead_letter")
|
|
service = GenerationJobRetryService(
|
|
self.queue_service,
|
|
_RejectsBeforeTaskCreation(self.workflow_service),
|
|
)
|
|
|
|
response = retry_job(self.ctx, original.task_id, service=service)
|
|
|
|
self.assertFalse(response["ok"])
|
|
self.assertTrue(response["details"]["original_status_restored"])
|
|
self.assertIsNone(response["details"]["replacement_task_id"])
|
|
restored = self.queue_service.get_job(original.task_id)
|
|
self.assertEqual(restored.status, "dead_letter")
|
|
self.assertEqual(restored.metadata["retry_execution"]["status"], "rejected")
|
|
|
|
def test_cancelled_task_with_matching_active_node_can_retry(self):
|
|
original = self._failed_generation_task(status="cancelled", node_status="running")
|
|
|
|
response = retry_job(self.ctx, original.task_id, service=self.retry_service)
|
|
|
|
self.assertTrue(response["ok"])
|
|
self.assertEqual(response["data"]["status"], "succeeded")
|
|
self.assertEqual(self.queue_service.get_job(original.task_id).status, "superseded")
|
|
|
|
def test_preflight_failure_restores_matching_active_node(self):
|
|
original = self._failed_generation_task(status="cancelled", node_status="running")
|
|
before_node = self.workflow_service.get_node(str(original.node_id))
|
|
service = GenerationJobRetryService(
|
|
self.queue_service,
|
|
_RejectsBeforeTaskCreation(self.workflow_service),
|
|
)
|
|
|
|
response = retry_job(self.ctx, original.task_id, service=service)
|
|
|
|
self.assertFalse(response["ok"])
|
|
self.assertTrue(response["details"]["original_status_restored"])
|
|
self.assertTrue(response["details"]["node_status_restored"])
|
|
self.assertEqual(self.queue_service.get_job(original.task_id).status, "cancelled")
|
|
after_node = self.workflow_service.get_node(str(original.node_id))
|
|
self.assertEqual(after_node.status, "running")
|
|
self.assertEqual(after_node.metadata, before_node.metadata)
|
|
|
|
def test_retry_requires_replacement_for_ephemeral_inline_references(self):
|
|
original = self._failed_generation_task(status="failed")
|
|
self.queue_service.update_task(
|
|
original.task_id,
|
|
{
|
|
"request": {
|
|
**original.request,
|
|
"reference_persistence": {"ephemeral_data_urls_omitted": 1},
|
|
}
|
|
},
|
|
self.ctx,
|
|
)
|
|
|
|
rejected = retry_job(self.ctx, original.task_id, service=self.retry_service)
|
|
accepted = retry_job(
|
|
self.ctx,
|
|
original.task_id,
|
|
{"input_images": ["https://example.com/replacement.png"]},
|
|
service=self.retry_service,
|
|
)
|
|
|
|
self.assertFalse(rejected["ok"])
|
|
self.assertEqual(rejected["error_code"], "TASK_NOT_RETRYABLE")
|
|
self.assertEqual(rejected["details"]["ephemeral_data_urls_omitted"], 1)
|
|
self.assertTrue(accepted["ok"])
|
|
|
|
def test_retry_claim_is_compare_and_set_under_concurrency(self):
|
|
task = self.queue_service.create_task({"status": "failed"}, self.ctx)
|
|
barrier = threading.Barrier(2)
|
|
|
|
def claim(index: int) -> str:
|
|
barrier.wait(timeout=2)
|
|
try:
|
|
return self.queue_service.begin_retry(
|
|
task.task_id,
|
|
self.ctx,
|
|
operation_id=f"op_{index}",
|
|
).status
|
|
except ApiError as exc:
|
|
return exc.error_code.value
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(executor.map(claim, (1, 2)))
|
|
|
|
self.assertEqual(sorted(results), ["TASK_NOT_RETRYABLE", "retrying"])
|
|
self.assertEqual(self.queue_service.get_job(task.task_id).status, "retrying")
|
|
|
|
def test_dead_letter_retry_is_synchronous_and_capped(self):
|
|
tasks = [self._failed_generation_task(status="dead_letter") for _ in range(7)]
|
|
service = GenerationJobRetryService(
|
|
self.queue_service,
|
|
_CompletesReplacement(self.queue_service, self.workflow_service),
|
|
)
|
|
|
|
response = retry_dead_letter_jobs(self.ctx, {"limit": 50}, service)
|
|
|
|
self.assertTrue(response["ok"])
|
|
data = response["data"]
|
|
self.assertEqual(data["execution_mode"], "synchronous")
|
|
self.assertEqual(data["applied_limit"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
self.assertEqual(data["processed"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
self.assertEqual(data["total"], MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
self.assertEqual(data["remaining"], len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
statuses = [self.queue_service.get_job(task.task_id).status for task in tasks]
|
|
self.assertEqual(statuses.count("superseded"), MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
self.assertEqual(statuses.count("dead_letter"), len(tasks) - MAX_SYNCHRONOUS_DEAD_LETTER_RETRIES)
|
|
|
|
def _failed_generation_task(self, *, status: str, node_status: str = "failed"):
|
|
profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx)
|
|
workflow = self.workflow_service.create_workflow(
|
|
profile.profile_id,
|
|
{"workflow_type": "single_image", "display_name": "Retry"},
|
|
self.ctx,
|
|
)
|
|
node = workflow.nodes[0]
|
|
task = self.queue_service.create_task(
|
|
{
|
|
"profile_id": profile.profile_id,
|
|
"workflow_id": workflow.workflow.workflow_id,
|
|
"node_id": node.node_id,
|
|
"status": status,
|
|
"model_route": "image_low",
|
|
"request": {"prompt_patch": "retry portrait", "model_route": "image_low"},
|
|
"error_message": "provider outage",
|
|
},
|
|
self.ctx,
|
|
)
|
|
if node_status == "running":
|
|
queued = self.workflow_service.update_node(
|
|
node.node_id,
|
|
{"status": "queued", "metadata": {**node.metadata, "last_task_id": task.task_id}},
|
|
self.ctx,
|
|
)
|
|
self.workflow_service.update_node(
|
|
node.node_id,
|
|
{"status": "running", "metadata": queued.metadata},
|
|
self.ctx,
|
|
)
|
|
else:
|
|
self.workflow_service.update_node(
|
|
node.node_id,
|
|
{"status": node_status, "metadata": {**node.metadata, "last_task_id": task.task_id}},
|
|
self.ctx,
|
|
)
|
|
return task
|
|
|
|
|
|
class _CompletesReplacement:
|
|
def __init__(self, queue_service: QueueService, workflow_service: WorkflowService):
|
|
self.queue_service = queue_service
|
|
self.workflow_service = workflow_service
|
|
|
|
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
|
|
node = self.workflow_service.get_node(node_id)
|
|
workflow = self.workflow_service.get_workflow(node.workflow_id).workflow
|
|
task = self.queue_service.create_task(
|
|
{
|
|
"profile_id": workflow.profile_id,
|
|
"workflow_id": workflow.workflow_id,
|
|
"node_id": node_id,
|
|
"status": "succeeded",
|
|
"idempotency_key": request["idempotency_key"],
|
|
"request": request,
|
|
"progress": 100,
|
|
},
|
|
ctx,
|
|
)
|
|
return {"task_id": task.task_id, "task_status": task.status}
|
|
|
|
|
|
class _CreatesFailedReplacement(_CompletesReplacement):
|
|
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
|
|
node = self.workflow_service.get_node(node_id)
|
|
workflow = self.workflow_service.get_workflow(node.workflow_id).workflow
|
|
task = self.queue_service.create_task(
|
|
{
|
|
"profile_id": workflow.profile_id,
|
|
"workflow_id": workflow.workflow_id,
|
|
"node_id": node_id,
|
|
"status": "failed",
|
|
"idempotency_key": request["idempotency_key"],
|
|
"request": request,
|
|
"error_message": "provider still unavailable",
|
|
},
|
|
ctx,
|
|
)
|
|
raise ApiError(
|
|
ErrorCode.GENERATION_FAILED,
|
|
"replacement generation failed",
|
|
{"provider_error": "temporary outage", "created_task_id": task.task_id},
|
|
)
|
|
|
|
|
|
class _RejectsBeforeTaskCreation:
|
|
def __init__(self, workflow_service: WorkflowService):
|
|
self.workflow_service = workflow_service
|
|
|
|
def generate_node(self, node_id: str, request: dict, ctx: RequestContext) -> dict:
|
|
raise ApiError(
|
|
ErrorCode.MODEL_CAPABILITY_UNCONFIRMED,
|
|
"model route is unavailable",
|
|
{"node_id": node_id},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|