From cfeb91f62cd702d9f7f571c68da1412272263e3d Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Sun, 5 Jul 2026 10:56:13 +0800 Subject: [PATCH] feat: add dead letter queue maintenance --- backend/route_registry.py | 2 ++ backend/routes/job_routes.py | 30 ++++++++++++++++++++++++++++++ backend/services/queue_service.py | 27 +++++++++++++++++++++++++++ tests/test_s2_routes.py | 28 +++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/backend/route_registry.py b/backend/route_registry.py index 51f61da..1245b91 100644 --- a/backend/route_registry.py +++ b/backend/route_registry.py @@ -56,6 +56,8 @@ ROUTES: tuple[RouteSpec, ...] = ( RouteSpec("GET", "jobs/{task_id}", "backend.routes.job_routes.get_job", "read"), RouteSpec("POST", "jobs/{task_id}/cancel", "backend.routes.job_routes.cancel_job", "write"), RouteSpec("POST", "jobs/{task_id}/retry", "backend.routes.job_routes.retry_job", "write"), + RouteSpec("POST", "jobs/dead-letter/retry", "backend.routes.job_routes.retry_dead_letter_jobs", "write"), + RouteSpec("POST", "jobs/dead-letter/move-failed", "backend.routes.job_routes.move_failed_jobs_to_dead_letter", "write"), RouteSpec("GET", "settings/schema", "backend.routes.settings_routes.get_settings_schema", "read"), RouteSpec("GET", "settings/current", "backend.routes.settings_routes.get_current_settings", "read"), RouteSpec("POST", "settings/validate", "backend.routes.settings_routes.validate_settings", "config"), diff --git a/backend/routes/job_routes.py b/backend/routes/job_routes.py index 9b55c0a..2c1ab9b 100644 --- a/backend/routes/job_routes.py +++ b/backend/routes/job_routes.py @@ -76,6 +76,36 @@ def retry_job( return handle_request(ctx, action) +def retry_dead_letter_jobs( + ctx: RequestContext, + payload: dict[str, Any] | None = None, + service: QueueService | None = None, +) -> dict: + queue_service = service or _create_queue_service() + params = payload or {} + + def action() -> dict[str, Any]: + require_write(ctx) + return queue_service.retry_dead_letters(ctx, limit=_int_or_default(params.get("limit"), 50)) + + return handle_request(ctx, action) + + +def move_failed_jobs_to_dead_letter( + ctx: RequestContext, + payload: dict[str, Any] | None = None, + service: QueueService | None = None, +) -> dict: + queue_service = service or _create_queue_service() + params = payload or {} + + def action() -> dict[str, Any]: + require_write(ctx) + return queue_service.move_failed_to_dead_letter(ctx, limit=_int_or_default(params.get("limit"), 50)) + + return handle_request(ctx, action) + + def _create_queue_service() -> QueueService: database = Database(load_database_settings()) return QueueService( diff --git a/backend/services/queue_service.py b/backend/services/queue_service.py index 56138eb..7347e05 100644 --- a/backend/services/queue_service.py +++ b/backend/services/queue_service.py @@ -184,6 +184,33 @@ class QueueService: def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]: return [task.to_dict() for task in self.repository.recent(limit=limit)] + def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: + jobs = self.list_jobs(status="dead_letter", page=1, page_size=limit).items + retried = [] + errors = [] + for job in jobs: + try: + retried.append(self.retry_job(job.task_id, ctx).to_dict()) + except ApiError as exc: + errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message}) + return {"retried": retried, "errors": errors, "total": len(retried)} + + def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]: + jobs = self.list_jobs(status="failed", page=1, page_size=limit).items + moved = [] + for job in jobs: + moved.append( + self.update_task( + job.task_id, + { + "status": "dead_letter", + "metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"}, + }, + ctx, + ).to_dict() + ) + return {"moved": moved, "total": len(moved)} + def new_task_id() -> str: return f"task_{uuid4().hex}" diff --git a/tests/test_s2_routes.py b/tests/test_s2_routes.py index f73a3f1..bbefd93 100644 --- a/tests/test_s2_routes.py +++ b/tests/test_s2_routes.py @@ -13,7 +13,14 @@ from backend.routes.image_routes import ( list_node_images, select_image, ) -from backend.routes.job_routes import cancel_job, get_job, list_jobs, retry_job +from backend.routes.job_routes import ( + cancel_job, + get_job, + list_jobs, + move_failed_jobs_to_dead_letter, + retry_dead_letter_jobs, + retry_job, +) from backend.routes.media_routes import fetch_media, sign_media from backend.routes.node_routes import ( generate_node, @@ -369,6 +376,25 @@ storage: self.assertFalse(response["ok"]) self.assertEqual(response["error_code"], "FORBIDDEN") + def test_dead_letter_batch_routes(self): + failed = self.queue_service.create_task({"status": "failed", "error_message": "bad"}, self.ctx) + self.queue_service.create_task({"status": "running"}, self.ctx) + + moved = move_failed_jobs_to_dead_letter(self.ctx, {"limit": 10}, self.queue_service) + self.assertTrue(moved["ok"]) + self.assertEqual(moved["data"]["total"], 1) + self.assertEqual(self.queue_service.get_job(failed.task_id).status, "dead_letter") + + retried = retry_dead_letter_jobs(self.ctx, {"limit": 10}, self.queue_service) + self.assertTrue(retried["ok"]) + self.assertEqual(retried["data"]["total"], 1) + self.assertEqual(self.queue_service.get_job(failed.task_id).status, "queued") + + read_only = RequestContext.from_astrbot(username="viewer", user_id="viewer_001") + denied = retry_dead_letter_jobs(read_only, {}, self.queue_service) + self.assertFalse(denied["ok"]) + self.assertEqual(denied["error_code"], "FORBIDDEN") + def test_settings_schema_current_validate_and_diff(self): self.config_path.write_text( """