feat: add dead letter queue maintenance

这个提交存在于:
YokiiroBW 2026-07-05 10:56:13 +08:00
父节点 123ccf96f1
当前提交 cfeb91f62c
共有 4 个文件被更改,包括 86 次插入 和 1 次删除

查看文件

@ -56,6 +56,8 @@ ROUTES: tuple[RouteSpec, ...] = (
RouteSpec("GET", "jobs/{task_id}", "backend.routes.job_routes.get_job", "read"),
RouteSpec("POST", "jobs/{task_id}/cancel", "backend.routes.job_routes.cancel_job", "write"),
RouteSpec("POST", "jobs/{task_id}/retry", "backend.routes.job_routes.retry_job", "write"),
RouteSpec("POST", "jobs/dead-letter/retry", "backend.routes.job_routes.retry_dead_letter_jobs", "write"),
RouteSpec("POST", "jobs/dead-letter/move-failed", "backend.routes.job_routes.move_failed_jobs_to_dead_letter", "write"),
RouteSpec("GET", "settings/schema", "backend.routes.settings_routes.get_settings_schema", "read"),
RouteSpec("GET", "settings/current", "backend.routes.settings_routes.get_current_settings", "read"),
RouteSpec("POST", "settings/validate", "backend.routes.settings_routes.validate_settings", "config"),

查看文件

@ -76,6 +76,36 @@ def retry_job(
return handle_request(ctx, action)
def retry_dead_letter_jobs(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = payload or {}
def action() -> dict[str, Any]:
require_write(ctx)
return queue_service.retry_dead_letters(ctx, limit=_int_or_default(params.get("limit"), 50))
return handle_request(ctx, action)
def move_failed_jobs_to_dead_letter(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = payload or {}
def action() -> dict[str, Any]:
require_write(ctx)
return queue_service.move_failed_to_dead_letter(ctx, limit=_int_or_default(params.get("limit"), 50))
return handle_request(ctx, action)
def _create_queue_service() -> QueueService:
database = Database(load_database_settings())
return QueueService(

查看文件

@ -184,6 +184,33 @@ class QueueService:
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
return [task.to_dict() for task in self.repository.recent(limit=limit)]
def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
jobs = self.list_jobs(status="dead_letter", page=1, page_size=limit).items
retried = []
errors = []
for job in jobs:
try:
retried.append(self.retry_job(job.task_id, ctx).to_dict())
except ApiError as exc:
errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message})
return {"retried": retried, "errors": errors, "total": len(retried)}
def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
jobs = self.list_jobs(status="failed", page=1, page_size=limit).items
moved = []
for job in jobs:
moved.append(
self.update_task(
job.task_id,
{
"status": "dead_letter",
"metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"},
},
ctx,
).to_dict()
)
return {"moved": moved, "total": len(moved)}
def new_task_id() -> str:
return f"task_{uuid4().hex}"

查看文件

@ -13,7 +13,14 @@ from backend.routes.image_routes import (
list_node_images,
select_image,
)
from backend.routes.job_routes import cancel_job, get_job, list_jobs, retry_job
from backend.routes.job_routes import (
cancel_job,
get_job,
list_jobs,
move_failed_jobs_to_dead_letter,
retry_dead_letter_jobs,
retry_job,
)
from backend.routes.media_routes import fetch_media, sign_media
from backend.routes.node_routes import (
generate_node,
@ -369,6 +376,25 @@ storage:
self.assertFalse(response["ok"])
self.assertEqual(response["error_code"], "FORBIDDEN")
def test_dead_letter_batch_routes(self):
failed = self.queue_service.create_task({"status": "failed", "error_message": "bad"}, self.ctx)
self.queue_service.create_task({"status": "running"}, self.ctx)
moved = move_failed_jobs_to_dead_letter(self.ctx, {"limit": 10}, self.queue_service)
self.assertTrue(moved["ok"])
self.assertEqual(moved["data"]["total"], 1)
self.assertEqual(self.queue_service.get_job(failed.task_id).status, "dead_letter")
retried = retry_dead_letter_jobs(self.ctx, {"limit": 10}, self.queue_service)
self.assertTrue(retried["ok"])
self.assertEqual(retried["data"]["total"], 1)
self.assertEqual(self.queue_service.get_job(failed.task_id).status, "queued")
read_only = RequestContext.from_astrbot(username="viewer", user_id="viewer_001")
denied = retry_dead_letter_jobs(read_only, {}, self.queue_service)
self.assertFalse(denied["ok"])
self.assertEqual(denied["error_code"], "FORBIDDEN")
def test_settings_schema_current_validate_and_diff(self):
self.config_path.write_text(
"""