feat: add dead letter queue maintenance
这个提交存在于:
父节点
123ccf96f1
当前提交
cfeb91f62c
共有 4 个文件被更改,包括 86 次插入 和 1 次删除
|
|
@ -56,6 +56,8 @@ ROUTES: tuple[RouteSpec, ...] = (
|
|||
RouteSpec("GET", "jobs/{task_id}", "backend.routes.job_routes.get_job", "read"),
|
||||
RouteSpec("POST", "jobs/{task_id}/cancel", "backend.routes.job_routes.cancel_job", "write"),
|
||||
RouteSpec("POST", "jobs/{task_id}/retry", "backend.routes.job_routes.retry_job", "write"),
|
||||
RouteSpec("POST", "jobs/dead-letter/retry", "backend.routes.job_routes.retry_dead_letter_jobs", "write"),
|
||||
RouteSpec("POST", "jobs/dead-letter/move-failed", "backend.routes.job_routes.move_failed_jobs_to_dead_letter", "write"),
|
||||
RouteSpec("GET", "settings/schema", "backend.routes.settings_routes.get_settings_schema", "read"),
|
||||
RouteSpec("GET", "settings/current", "backend.routes.settings_routes.get_current_settings", "read"),
|
||||
RouteSpec("POST", "settings/validate", "backend.routes.settings_routes.validate_settings", "config"),
|
||||
|
|
|
|||
|
|
@ -76,6 +76,36 @@ def retry_job(
|
|||
return handle_request(ctx, action)
|
||||
|
||||
|
||||
def retry_dead_letter_jobs(
|
||||
ctx: RequestContext,
|
||||
payload: dict[str, Any] | None = None,
|
||||
service: QueueService | None = None,
|
||||
) -> dict:
|
||||
queue_service = service or _create_queue_service()
|
||||
params = payload or {}
|
||||
|
||||
def action() -> dict[str, Any]:
|
||||
require_write(ctx)
|
||||
return queue_service.retry_dead_letters(ctx, limit=_int_or_default(params.get("limit"), 50))
|
||||
|
||||
return handle_request(ctx, action)
|
||||
|
||||
|
||||
def move_failed_jobs_to_dead_letter(
|
||||
ctx: RequestContext,
|
||||
payload: dict[str, Any] | None = None,
|
||||
service: QueueService | None = None,
|
||||
) -> dict:
|
||||
queue_service = service or _create_queue_service()
|
||||
params = payload or {}
|
||||
|
||||
def action() -> dict[str, Any]:
|
||||
require_write(ctx)
|
||||
return queue_service.move_failed_to_dead_letter(ctx, limit=_int_or_default(params.get("limit"), 50))
|
||||
|
||||
return handle_request(ctx, action)
|
||||
|
||||
|
||||
def _create_queue_service() -> QueueService:
|
||||
database = Database(load_database_settings())
|
||||
return QueueService(
|
||||
|
|
|
|||
|
|
@ -184,6 +184,33 @@ class QueueService:
|
|||
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
|
||||
return [task.to_dict() for task in self.repository.recent(limit=limit)]
|
||||
|
||||
def retry_dead_letters(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
||||
jobs = self.list_jobs(status="dead_letter", page=1, page_size=limit).items
|
||||
retried = []
|
||||
errors = []
|
||||
for job in jobs:
|
||||
try:
|
||||
retried.append(self.retry_job(job.task_id, ctx).to_dict())
|
||||
except ApiError as exc:
|
||||
errors.append({"task_id": job.task_id, "error_code": exc.error_code.value, "message": exc.message})
|
||||
return {"retried": retried, "errors": errors, "total": len(retried)}
|
||||
|
||||
def move_failed_to_dead_letter(self, ctx: RequestContext, *, limit: int = 50) -> dict[str, Any]:
|
||||
jobs = self.list_jobs(status="failed", page=1, page_size=limit).items
|
||||
moved = []
|
||||
for job in jobs:
|
||||
moved.append(
|
||||
self.update_task(
|
||||
job.task_id,
|
||||
{
|
||||
"status": "dead_letter",
|
||||
"metadata": {**job.metadata, "dead_letter_reason": job.error_message or "manual_move"},
|
||||
},
|
||||
ctx,
|
||||
).to_dict()
|
||||
)
|
||||
return {"moved": moved, "total": len(moved)}
|
||||
|
||||
|
||||
def new_task_id() -> str:
|
||||
return f"task_{uuid4().hex}"
|
||||
|
|
|
|||
|
|
@ -13,7 +13,14 @@ from backend.routes.image_routes import (
|
|||
list_node_images,
|
||||
select_image,
|
||||
)
|
||||
from backend.routes.job_routes import cancel_job, get_job, list_jobs, retry_job
|
||||
from backend.routes.job_routes import (
|
||||
cancel_job,
|
||||
get_job,
|
||||
list_jobs,
|
||||
move_failed_jobs_to_dead_letter,
|
||||
retry_dead_letter_jobs,
|
||||
retry_job,
|
||||
)
|
||||
from backend.routes.media_routes import fetch_media, sign_media
|
||||
from backend.routes.node_routes import (
|
||||
generate_node,
|
||||
|
|
@ -369,6 +376,25 @@ storage:
|
|||
self.assertFalse(response["ok"])
|
||||
self.assertEqual(response["error_code"], "FORBIDDEN")
|
||||
|
||||
def test_dead_letter_batch_routes(self):
|
||||
failed = self.queue_service.create_task({"status": "failed", "error_message": "bad"}, self.ctx)
|
||||
self.queue_service.create_task({"status": "running"}, self.ctx)
|
||||
|
||||
moved = move_failed_jobs_to_dead_letter(self.ctx, {"limit": 10}, self.queue_service)
|
||||
self.assertTrue(moved["ok"])
|
||||
self.assertEqual(moved["data"]["total"], 1)
|
||||
self.assertEqual(self.queue_service.get_job(failed.task_id).status, "dead_letter")
|
||||
|
||||
retried = retry_dead_letter_jobs(self.ctx, {"limit": 10}, self.queue_service)
|
||||
self.assertTrue(retried["ok"])
|
||||
self.assertEqual(retried["data"]["total"], 1)
|
||||
self.assertEqual(self.queue_service.get_job(failed.task_id).status, "queued")
|
||||
|
||||
read_only = RequestContext.from_astrbot(username="viewer", user_id="viewer_001")
|
||||
denied = retry_dead_letter_jobs(read_only, {}, self.queue_service)
|
||||
self.assertFalse(denied["ok"])
|
||||
self.assertEqual(denied["error_code"], "FORBIDDEN")
|
||||
|
||||
def test_settings_schema_current_validate_and_diff(self):
|
||||
self.config_path.write_text(
|
||||
"""
|
||||
|
|
|
|||
正在加载…
添加表格
在新工单中引用