feat: add job queue api

这个提交存在于:
YokiiroBW 2026-07-05 09:07:34 +08:00
父节点 e98f32ebfb
当前提交 1c7a2782d7
共有 7 个文件被更改,包括 722 次插入 和 8 次删除

60
backend/dto/job_dto.py 普通文件
查看文件

@ -0,0 +1,60 @@
"""Generation job DTOs."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class GenerationTask:
task_id: str
profile_id: str | None
workflow_id: str | None
node_id: str | None
task_type: str
status: str
model_route: str | None
progress: int
idempotency_key: str | None
request: dict[str, Any]
result: dict[str, Any]
error_message: str | None
metadata: dict[str, Any]
created_at: str
updated_at: str
def to_dict(self) -> dict[str, Any]:
return {
"task_id": self.task_id,
"profile_id": self.profile_id,
"workflow_id": self.workflow_id,
"node_id": self.node_id,
"task_type": self.task_type,
"status": self.status,
"model_route": self.model_route,
"progress": self.progress,
"idempotency_key": self.idempotency_key,
"request": self.request,
"result": self.result,
"error_message": self.error_message,
"metadata": self.metadata,
"created_at": self.created_at,
"updated_at": self.updated_at,
}
@dataclass(frozen=True)
class JobListResult:
items: list[GenerationTask]
total: int
page: int
page_size: int
def to_dict(self) -> dict[str, Any]:
return {
"items": [item.to_dict() for item in self.items],
"total": self.total,
"page": self.page,
"page_size": self.page_size,
}

查看文件

@ -0,0 +1,98 @@
"""Job and queue route handlers."""
from __future__ import annotations
from typing import Any
from backend.routes.common import handle_request
from backend.services.queue_service import QueueService
from core.audit_service import AuditService
from core.identity import RequestContext
from core.security import require_authenticated, require_write
from database.db_helper import Database, load_database_settings
from database.repositories.job_repository import JobRepository
def list_jobs(
ctx: RequestContext,
query: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = query or {}
def action() -> dict[str, Any]:
require_authenticated(ctx)
return queue_service.list_jobs(
status=_optional_str(params.get("status")),
workflow_id=_optional_str(params.get("workflow_id")),
profile_id=_optional_str(params.get("profile_id")),
page=_int_or_default(params.get("page"), 1),
page_size=_int_or_default(params.get("page_size"), 30),
).to_dict()
return handle_request(ctx, action)
def get_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_authenticated(ctx)
return queue_service.get_job(task_id).to_dict()
return handle_request(ctx, action)
def cancel_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_write(ctx)
return queue_service.cancel_job(task_id, ctx).to_dict()
return handle_request(ctx, action)
def retry_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_write(ctx)
return queue_service.retry_job(task_id, ctx).to_dict()
return handle_request(ctx, action)
def _create_queue_service() -> QueueService:
database = Database(load_database_settings())
return QueueService(
repository=JobRepository(database),
audit_service=AuditService(database),
)
def _optional_str(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _int_or_default(value: Any, default: int) -> int:
try:
return int(value)
except (TypeError, ValueError):
return default

查看文件

@ -7,6 +7,7 @@ from typing import Any
from core.config_service import ConfigService, ImageStudioConfig
from database.db_helper import Database, REPO_ROOT
from database.repositories.job_repository import JobRepository
class DashboardService:
@ -20,32 +21,38 @@ class DashboardService:
def get_summary(self) -> dict[str, Any]:
config = self.config_service.load()
queue_repository = JobRepository(self.database)
return {
"plugin_status": "running",
"today_generation_count": 0,
"active_profiles": self._count_active_profiles(),
"queue": {
"pending": 0,
"running": 0,
"failed": 0,
"dead_letter": 0,
},
"queue": self._queue_summary(queue_repository),
"storage": self._storage_summary(config),
"recent_jobs": [],
"recent_jobs": [task.to_dict() for task in queue_repository.recent(limit=10)],
"recent_images": self._recent_images(),
}
def _count_active_profiles(self) -> int:
placeholder = self.database._placeholder
conn = self.database.connect()
try:
row = conn.execute(
"SELECT COUNT(*) AS total FROM character_profiles WHERE status = ?",
f"SELECT COUNT(*) AS total FROM character_profiles WHERE status = {placeholder}",
("active",),
).fetchone()
finally:
conn.close()
return int(row["total"] if row else 0)
def _queue_summary(self, queue_repository: JobRepository) -> dict[str, int]:
counts = queue_repository.count_by_status()
return {
"pending": counts.get("created", 0) + counts.get("queued", 0),
"running": counts.get("running", 0),
"failed": counts.get("failed", 0),
"dead_letter": counts.get("dead_letter", 0),
}
def _recent_images(self) -> list[dict[str, Any]]:
conn = self.database.connect()
try:

查看文件

@ -0,0 +1,167 @@
"""Queue and job service."""
from __future__ import annotations
from typing import Any
from uuid import uuid4
from backend.dto.common import ApiError, ErrorCode
from backend.dto.job_dto import GenerationTask, JobListResult
from core.audit_service import AuditService
from core.identity import RequestContext
from database.repositories.job_repository import JobRepository
TASK_STATUSES = {
"created",
"queued",
"running",
"completed",
"succeeded",
"failed",
"cancelled",
"dead_letter",
}
class QueueService:
def __init__(self, repository: JobRepository, audit_service: AuditService):
self.repository = repository
self.audit_service = audit_service
def create_task(self, payload: dict[str, Any], ctx: RequestContext) -> GenerationTask:
status = str(payload.get("status", "created"))
if status not in TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"unsupported task status",
{"field": "status", "allowed": sorted(TASK_STATUSES)},
)
task = self.repository.create(
{
"task_id": payload.get("task_id") or new_task_id(),
"profile_id": payload.get("profile_id"),
"workflow_id": payload.get("workflow_id"),
"node_id": payload.get("node_id"),
"task_type": payload.get("task_type", "generation"),
"status": status,
"model_route": payload.get("model_route"),
"progress": payload.get("progress", 0),
"idempotency_key": payload.get("idempotency_key"),
"request": payload.get("request") or {},
"result": payload.get("result") or {},
"error_message": payload.get("error_message"),
"metadata": payload.get("metadata") or {},
}
)
self.audit_service.record(
ctx=ctx,
action="create_task",
target_type="generation_task",
target_id=task.task_id,
after_snapshot=task.to_dict(),
)
return task
def list_jobs(
self,
*,
status: str | None = None,
workflow_id: str | None = None,
profile_id: str | None = None,
page: int = 1,
page_size: int = 30,
) -> JobListResult:
if status and status not in TASK_STATUSES:
raise ApiError(
ErrorCode.VALIDATION_FAILED,
"unsupported task status",
{"field": "status", "allowed": sorted(TASK_STATUSES)},
)
normalized_page = max(page, 1)
normalized_page_size = min(max(page_size, 1), 100)
items, total = self.repository.list(
status=status,
workflow_id=workflow_id,
profile_id=profile_id,
page=normalized_page,
page_size=normalized_page_size,
)
return JobListResult(
items=items,
total=total,
page=normalized_page,
page_size=normalized_page_size,
)
def get_job(self, task_id: str) -> GenerationTask:
task = self.repository.get(task_id)
if task is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
return task
def cancel_job(self, task_id: str, ctx: RequestContext) -> GenerationTask:
before = self.get_job(task_id)
if before.status in {"completed", "succeeded", "cancelled"}:
raise ApiError(
ErrorCode.TASK_NOT_CANCELABLE,
"task cannot be cancelled from current status",
{"task_id": task_id, "status": before.status},
)
after = self.repository.update(task_id, {"status": "cancelled"})
if after is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
self.audit_service.record(
ctx=ctx,
action="cancel_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
return after
def retry_job(self, task_id: str, ctx: RequestContext) -> GenerationTask:
before = self.get_job(task_id)
if before.status not in {"failed", "dead_letter", "cancelled"}:
raise ApiError(
ErrorCode.TASK_NOT_RETRYABLE,
"task cannot be retried from current status",
{"task_id": task_id, "status": before.status},
)
after = self.repository.update(
task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {**before.metadata, "retry_requested": True},
},
)
if after is None:
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "task not found", {"task_id": task_id})
self.audit_service.record(
ctx=ctx,
action="retry_task",
target_type="generation_task",
target_id=task_id,
before_snapshot=before.to_dict(),
after_snapshot=after.to_dict(),
)
return after
def queue_summary(self) -> dict[str, int]:
counts = self.repository.count_by_status()
return {
"pending": counts.get("created", 0) + counts.get("queued", 0),
"running": counts.get("running", 0),
"failed": counts.get("failed", 0),
"dead_letter": counts.get("dead_letter", 0),
}
def recent_jobs(self, limit: int = 10) -> list[dict[str, Any]]:
return [task.to_dict() for task in self.repository.recent(limit=limit)]
def new_task_id() -> str:
return f"task_{uuid4().hex}"

查看文件

@ -0,0 +1,33 @@
CREATE TABLE IF NOT EXISTS generation_tasks (
task_id TEXT PRIMARY KEY,
profile_id TEXT,
workflow_id TEXT,
node_id TEXT,
task_type TEXT NOT NULL DEFAULT 'generation',
status TEXT NOT NULL DEFAULT 'created',
model_route TEXT,
progress INTEGER NOT NULL DEFAULT 0,
idempotency_key TEXT,
request_json TEXT,
result_json TEXT,
error_message TEXT,
metadata_json TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(profile_id) REFERENCES character_profiles(profile_id) ON DELETE SET NULL,
FOREIGN KEY(workflow_id) REFERENCES profile_workflows(workflow_id) ON DELETE SET NULL,
FOREIGN KEY(node_id) REFERENCES workflow_nodes(node_id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_generation_tasks_status
ON generation_tasks(status, created_at);
CREATE INDEX IF NOT EXISTS idx_generation_tasks_workflow
ON generation_tasks(workflow_id, status, created_at);
CREATE INDEX IF NOT EXISTS idx_generation_tasks_profile
ON generation_tasks(profile_id, status, created_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_generation_tasks_idempotency
ON generation_tasks(idempotency_key)
WHERE idempotency_key IS NOT NULL;

查看文件

@ -0,0 +1,256 @@
"""Repository for generation tasks and queue status."""
from __future__ import annotations
import json
from typing import Any
from backend.dto.job_dto import GenerationTask
from database.db_helper import Database
TASK_COLUMNS = """
task_id,
profile_id,
workflow_id,
node_id,
task_type,
status,
model_route,
progress,
idempotency_key,
request_json,
result_json,
error_message,
metadata_json,
created_at,
updated_at
"""
class JobRepository:
def __init__(self, database: Database):
self.database = database
@property
def placeholder(self) -> str:
return self.database._placeholder
def create(self, payload: dict[str, Any]) -> GenerationTask:
placeholder = self.placeholder
conn = self.database.connect()
try:
conn.execute(
f"""
INSERT INTO generation_tasks (
task_id,
profile_id,
workflow_id,
node_id,
task_type,
status,
model_route,
progress,
idempotency_key,
request_json,
result_json,
error_message,
metadata_json
) VALUES (
{placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}, {placeholder}, {placeholder}, {placeholder},
{placeholder}
)
""",
(
payload["task_id"],
payload.get("profile_id"),
payload.get("workflow_id"),
payload.get("node_id"),
payload.get("task_type", "generation"),
payload.get("status", "created"),
payload.get("model_route"),
int(payload.get("progress", 0)),
payload.get("idempotency_key"),
_json_or_none(payload.get("request")),
_json_or_none(payload.get("result")),
payload.get("error_message"),
_json_or_none(payload.get("metadata")),
),
)
conn.commit()
finally:
conn.close()
task = self.get(payload["task_id"])
if task is None:
raise RuntimeError("generation task was inserted but could not be read back")
return task
def get(self, task_id: str) -> GenerationTask | None:
placeholder = self.placeholder
conn = self.database.connect()
try:
row = conn.execute(
f"SELECT {TASK_COLUMNS} FROM generation_tasks WHERE task_id = {placeholder}",
(task_id,),
).fetchone()
finally:
conn.close()
return _row_to_task(row) if row else None
def list(
self,
*,
status: str | None = None,
workflow_id: str | None = None,
profile_id: str | None = None,
page: int = 1,
page_size: int = 30,
) -> tuple[list[GenerationTask], int]:
placeholder = self.placeholder
where: list[str] = []
params: list[Any] = []
if status:
where.append(f"status = {placeholder}")
params.append(status)
if workflow_id:
where.append(f"workflow_id = {placeholder}")
params.append(workflow_id)
if profile_id:
where.append(f"profile_id = {placeholder}")
params.append(profile_id)
where_clause = f"WHERE {' AND '.join(where)}" if where else ""
offset = max(page - 1, 0) * page_size
conn = self.database.connect()
try:
total_row = conn.execute(
f"SELECT COUNT(*) AS total FROM generation_tasks {where_clause}",
tuple(params),
).fetchone()
rows = conn.execute(
f"""
SELECT {TASK_COLUMNS}
FROM generation_tasks
{where_clause}
ORDER BY created_at DESC
LIMIT {placeholder} OFFSET {placeholder}
""",
tuple(params + [page_size, offset]),
).fetchall()
finally:
conn.close()
return [_row_to_task(row) for row in rows], int(total_row["total"] if total_row else 0)
def update(self, task_id: str, payload: dict[str, Any]) -> GenerationTask | None:
allowed = {
"status",
"model_route",
"progress",
"request",
"result",
"error_message",
"metadata",
}
assignments: list[str] = []
params: list[Any] = []
placeholder = self.placeholder
for key in allowed:
if key not in payload:
continue
column = key
value = payload[key]
if key == "request":
column = "request_json"
value = _json_or_none(value)
elif key == "result":
column = "result_json"
value = _json_or_none(value)
elif key == "metadata":
column = "metadata_json"
value = _json_or_none(value)
elif key == "progress":
value = int(value)
assignments.append(f"{column} = {placeholder}")
params.append(value)
if not assignments:
return self.get(task_id)
assignments.append("updated_at = CURRENT_TIMESTAMP")
params.append(task_id)
conn = self.database.connect()
try:
conn.execute(
f"""
UPDATE generation_tasks
SET {", ".join(assignments)}
WHERE task_id = {placeholder}
""",
tuple(params),
)
conn.commit()
finally:
conn.close()
return self.get(task_id)
def count_by_status(self) -> dict[str, int]:
conn = self.database.connect()
try:
rows = conn.execute(
"""
SELECT status, COUNT(*) AS total
FROM generation_tasks
GROUP BY status
"""
).fetchall()
finally:
conn.close()
return {row["status"]: int(row["total"]) for row in rows}
def recent(self, limit: int = 10) -> list[GenerationTask]:
placeholder = self.placeholder
conn = self.database.connect()
try:
rows = conn.execute(
f"""
SELECT {TASK_COLUMNS}
FROM generation_tasks
ORDER BY created_at DESC
LIMIT {placeholder}
""",
(limit,),
).fetchall()
finally:
conn.close()
return [_row_to_task(row) for row in rows]
def _row_to_task(row) -> GenerationTask:
return GenerationTask(
task_id=row["task_id"],
profile_id=row["profile_id"],
workflow_id=row["workflow_id"],
node_id=row["node_id"],
task_type=row["task_type"],
status=row["status"],
model_route=row["model_route"],
progress=int(row["progress"] or 0),
idempotency_key=row["idempotency_key"],
request=_json_loads(row["request_json"]),
result=_json_loads(row["result_json"]),
error_message=row["error_message"],
metadata=_json_loads(row["metadata_json"]),
created_at=str(row["created_at"]),
updated_at=str(row["updated_at"]),
)
def _json_or_none(value: Any | None) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, sort_keys=True)
def _json_loads(value: str | None) -> dict[str, Any]:
if not value:
return {}
return json.loads(value)

查看文件

@ -12,6 +12,7 @@ from backend.routes.image_routes import (
list_node_images,
select_image,
)
from backend.routes.job_routes import cancel_job, get_job, list_jobs, retry_job
from backend.routes.node_routes import (
generate_node,
get_node,
@ -38,6 +39,7 @@ from backend.routes.workflow_routes import (
from backend.services.dashboard_service import DashboardService
from backend.services.image_service import ImageService
from backend.services.profile_service import ProfileService
from backend.services.queue_service import QueueService
from backend.services.ui_service import UiService
from backend.services.workflow_service import WorkflowService
from core.audit_service import AuditService
@ -45,6 +47,7 @@ from core.config_service import ConfigService
from core.identity import RequestContext
from database.db_helper import Database, DatabaseSettings, SqliteSettings
from database.repositories.image_repository import ImageRepository
from database.repositories.job_repository import JobRepository
from database.repositories.profile_repository import ProfileRepository
from database.repositories.workflow_repository import WorkflowRepository
@ -93,6 +96,7 @@ storage:
self.profile_repository = ProfileRepository(self.database)
self.workflow_repository = WorkflowRepository(self.database)
self.image_repository = ImageRepository(self.database)
self.job_repository = JobRepository(self.database)
self.profile_service = ProfileService(
repository=self.profile_repository,
audit_service=self.audit_service,
@ -107,6 +111,10 @@ storage:
workflow_repository=self.workflow_repository,
audit_service=self.audit_service,
)
self.queue_service = QueueService(
repository=self.job_repository,
audit_service=self.audit_service,
)
self.ctx = RequestContext.from_astrbot(
username="admin",
user_id="user_001",
@ -183,6 +191,24 @@ storage:
self.assertEqual(len(data["recent_images"]), 1)
self.assertGreaterEqual(data["storage"]["total_bytes"], 8)
def test_dashboard_summary_uses_real_queue_counts(self):
self.queue_service.create_task({"status": "queued"}, self.ctx)
self.queue_service.create_task({"status": "running"}, self.ctx)
self.queue_service.create_task({"status": "failed"}, self.ctx)
self.queue_service.create_task({"status": "dead_letter"}, self.ctx)
response = get_dashboard_summary(
self.ctx,
DashboardService(self.database, self.config_service),
)
self.assertTrue(response["ok"])
self.assertEqual(
response["data"]["queue"],
{"pending": 1, "running": 1, "failed": 1, "dead_letter": 1},
)
self.assertEqual(len(response["data"]["recent_jobs"]), 4)
def test_profile_crud_routes_return_wrapped_responses(self):
created = create_profile(
self.ctx,
@ -256,6 +282,73 @@ storage:
self.assertIn("魔法学院学生", preview["base_prompt"])
self.assertIn("银白色 hair", preview["appearance_prompt"])
def test_job_routes_list_detail_cancel_and_retry(self):
profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx)
workflow = self.workflow_service.create_workflow(
profile.profile_id,
{"workflow_type": "single_image", "display_name": "Job workflow"},
self.ctx,
)
running_task = self.queue_service.create_task(
{
"profile_id": profile.profile_id,
"workflow_id": workflow.workflow.workflow_id,
"node_id": workflow.nodes[0].node_id,
"status": "running",
"model_route": "image_low",
"request": {"prompt": "portrait"},
},
self.ctx,
)
failed_task = self.queue_service.create_task(
{
"profile_id": profile.profile_id,
"workflow_id": workflow.workflow.workflow_id,
"node_id": workflow.nodes[0].node_id,
"status": "failed",
"error_message": "mock failure",
},
self.ctx,
)
listed = list_jobs(
self.ctx,
{"workflow_id": workflow.workflow.workflow_id, "page": "1", "page_size": "10"},
self.queue_service,
)
self.assertTrue(listed["ok"])
self.assertEqual(listed["data"]["total"], 2)
failed_only = list_jobs(self.ctx, {"status": "failed"}, self.queue_service)
self.assertTrue(failed_only["ok"])
self.assertEqual(failed_only["data"]["items"][0]["task_id"], failed_task.task_id)
detail = get_job(self.ctx, running_task.task_id, self.queue_service)
self.assertTrue(detail["ok"])
self.assertEqual(detail["data"]["status"], "running")
cancelled = cancel_job(self.ctx, running_task.task_id, self.queue_service)
self.assertTrue(cancelled["ok"])
self.assertEqual(cancelled["data"]["status"], "cancelled")
retried = retry_job(self.ctx, failed_task.task_id, self.queue_service)
self.assertTrue(retried["ok"])
self.assertEqual(retried["data"]["status"], "queued")
self.assertIsNone(retried["data"]["error_message"])
not_retryable = retry_job(self.ctx, failed_task.task_id, self.queue_service)
self.assertFalse(not_retryable["ok"])
self.assertEqual(not_retryable["error_code"], "TASK_NOT_RETRYABLE")
def test_job_write_routes_require_write_permission(self):
task = self.queue_service.create_task({"status": "running"}, self.ctx)
read_only = RequestContext.from_astrbot(username="viewer", user_id="viewer_001")
response = cancel_job(read_only, task.task_id, self.queue_service)
self.assertFalse(response["ok"])
self.assertEqual(response["error_code"], "FORBIDDEN")
def test_workflow_node_image_routes_return_wrapped_responses(self):
profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx)