astrbot_plugin_imagestudio/backend/routes/job_routes.py

319 行
11 KiB
Python

"""Job and queue route handlers."""
from __future__ import annotations
from typing import Any
from backend.routes.common import handle_request
from backend.routes.params import int_or_default, optional_str
from backend.services.event_stream_service import get_default_event_bus
from backend.services.generation_job_retry_service import GenerationJobRetryService
from backend.services.queue_service import QueueService
from core.audit_service import AuditService
from core.identity import RequestContext
from core.security import require_authenticated, require_permission, require_task_retry, require_write
from database.db_helper import Database, load_database_settings
from database.repositories.job_repository import JobRepository
def list_jobs(
ctx: RequestContext,
query: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = query or {}
def action() -> dict[str, Any]:
require_authenticated(ctx)
result = queue_service.list_jobs(
status=optional_str(params.get("status")),
workflow_id=optional_str(params.get("workflow_id")),
profile_id=optional_str(params.get("profile_id")),
page=int_or_default(params.get("page"), 1),
page_size=int_or_default(params.get("page_size"), 30),
).to_dict()
if not ctx.has_permission("can_manage_materials"):
visible = [item for item in result["items"] if item.get("metadata", {}).get("source") != "advanced_material"]
result = {**result, "items": visible, "total": max(int(result.get("total") or 0) - (len(result["items"]) - len(visible)), 0)}
return result
return handle_request(ctx, action)
def get_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_authenticated(ctx)
task = queue_service.get_job(task_id)
if task.metadata.get("source") == "advanced_material":
require_permission(ctx, "can_manage_materials")
return task.to_dict()
return handle_request(ctx, action)
def delete_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_task_retry(ctx)
return queue_service.delete_job(task_id, ctx)
return handle_request(ctx, action)
def cancel_job(
ctx: RequestContext,
task_id: str,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
def action() -> dict[str, Any]:
require_write(ctx)
task = queue_service.get_job(task_id)
if task.metadata.get("source") == "advanced_material":
require_permission(ctx, "can_manage_materials")
from backend.services.advanced_material_service import create_advanced_material_service
import_id = str(task.metadata.get("advanced_material_import_id") or "").strip()
return create_advanced_material_service().cancel_import(import_id, ctx)
return queue_service.cancel_job(task_id, ctx).to_dict()
return handle_request(ctx, action)
def retry_job(
ctx: RequestContext,
task_id: str,
payload: dict[str, Any] | None = None,
service: GenerationJobRetryService | None = None,
) -> dict:
retry_service = service
def action() -> dict[str, Any]:
require_task_retry(ctx)
queue_service = _create_queue_service()
task = queue_service.get_job(task_id)
return _retry_task(task, ctx, payload=payload, retry_service=retry_service)
return handle_request(ctx, action)
def retry_failed_jobs(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
) -> dict:
params = payload or {}
def action() -> dict[str, Any]:
require_task_retry(ctx)
queue_service = _create_queue_service()
limit = max(1, min(int_or_default(params.get("limit"), 100), 100))
tasks = []
seen_task_ids: set[str] = set()
for status in ("failed", "dead_letter", "cancelled"):
remaining = limit - len(tasks)
if remaining <= 0:
break
for task in queue_service.list_jobs(status=status, page=1, page_size=remaining).items:
if task.task_id not in seen_task_ids:
seen_task_ids.add(task.task_id)
tasks.append(task)
tasks = _latest_retryable_tasks(tasks)
retried: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
retry_service: GenerationJobRetryService | None = None
for task in tasks:
try:
if task.metadata.get("source") != "advanced_material":
retry_service = retry_service or _create_generation_job_retry_service()
retried.append(_retry_task(task, ctx, retry_service=retry_service))
except Exception as exc:
errors.append(_retry_error_payload(task.task_id, exc))
return {
"retried": retried,
"errors": errors,
"total": len(retried),
"processed": len(retried) + len(errors),
"limit": limit,
"remaining_by_status": {
status: queue_service.repository.count_by_status().get(status, 0)
for status in ("failed", "dead_letter", "cancelled")
},
}
return handle_request(ctx, action)
def _latest_retryable_tasks(tasks: list[Any]) -> list[Any]:
selected: list[Any] = []
lineage_indexes: dict[tuple[str, str], int] = {}
for task in tasks:
lineage_key = _retry_lineage_key(task)
if lineage_key is None:
selected.append(task)
continue
previous_index = lineage_indexes.get(lineage_key)
if previous_index is None:
lineage_indexes[lineage_key] = len(selected)
selected.append(task)
continue
previous_task = selected[previous_index]
if _task_sort_key(task) > _task_sort_key(previous_task):
selected[previous_index] = task
return selected
def _retry_lineage_key(task: Any) -> tuple[str, str] | None:
metadata = task.metadata if isinstance(getattr(task, "metadata", None), dict) else {}
source_id = str(metadata.get("advanced_style_source_id") or "").strip()
case_id = str(metadata.get("advanced_style_validation_case_id") or "").strip()
if task.task_type == "advanced_style_source_recognition" and source_id:
return ("source", source_id)
if task.task_type == "advanced_style_validation_generation" and case_id:
return ("validation", case_id)
return None
def _task_sort_key(task: Any) -> str:
return str(getattr(task, "updated_at", "") or getattr(task, "created_at", "") or "")
def _retry_task(
task: Any,
ctx: RequestContext,
*,
payload: dict[str, Any] | None = None,
retry_service: GenerationJobRetryService | None = None,
) -> dict[str, Any]:
if task.metadata.get("source") == "advanced_material":
require_permission(ctx, "can_manage_materials")
from backend.services.advanced_material_service import create_advanced_material_service
advanced_service = create_advanced_material_service()
import_id = str(task.metadata.get("advanced_material_import_id") or "").strip()
if task.task_type == "advanced_style_source_recognition":
source_id = str(task.metadata.get("advanced_style_source_id") or "").strip()
result = advanced_service.style_sets.retry_source(import_id, source_id, ctx)
elif task.task_type == "advanced_style_validation_generation":
case_id = str(task.metadata.get("advanced_style_validation_case_id") or "").strip()
result = advanced_service.style_sets.retry_validation_case(import_id, case_id, ctx)
else:
result = advanced_service.retry_import(import_id, ctx)
if isinstance(result, dict) and isinstance(result.get("task"), dict):
return result["task"]
return result
resolved_retry_service = retry_service or _create_generation_job_retry_service()
return resolved_retry_service.retry_job(task.task_id, ctx, overrides=payload or {}).to_dict()
def _retry_error_payload(task_id: str, exc: Exception) -> dict[str, Any]:
error_code = getattr(getattr(exc, "error_code", None), "value", None) or str(getattr(exc, "error_code", "retry_failed"))
message = str(getattr(exc, "message", None) or str(exc))
raw_details = getattr(exc, "details", None)
details = dict(raw_details) if isinstance(raw_details, dict) else {}
return {
"task_id": task_id,
"error_code": error_code,
"message": message,
**({"replacement_task_id": details["replacement_task_id"]} if details.get("replacement_task_id") else {}),
}
def retry_dead_letter_jobs(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: GenerationJobRetryService | None = None,
) -> dict:
retry_service = service or _create_generation_job_retry_service()
params = payload or {}
def action() -> dict[str, Any]:
require_task_retry(ctx)
return retry_service.retry_dead_letters(ctx, limit=int_or_default(params.get("limit"), 5))
return handle_request(ctx, action)
def move_failed_jobs_to_dead_letter(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = payload or {}
def action() -> dict[str, Any]:
require_task_retry(ctx)
return queue_service.move_failed_to_dead_letter(ctx, limit=int_or_default(params.get("limit"), 50))
return handle_request(ctx, action)
def clear_failed_jobs(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = payload or {}
def action() -> dict[str, Any]:
require_task_retry(ctx)
return queue_service.clear_failed_jobs(ctx, limit=int_or_default(params.get("limit"), 100))
return handle_request(ctx, action)
def clear_terminal_jobs(
ctx: RequestContext,
payload: dict[str, Any] | None = None,
service: QueueService | None = None,
) -> dict:
queue_service = service or _create_queue_service()
params = payload or {}
def action() -> dict[str, Any]:
require_task_retry(ctx)
return queue_service.clear_terminal_jobs(
ctx,
statuses=params.get("statuses"),
limit=int_or_default(params.get("limit"), 100),
)
return handle_request(ctx, action)
def _create_queue_service() -> QueueService:
database = Database(load_database_settings())
return QueueService(
repository=JobRepository(database),
audit_service=AuditService(database),
event_bus=get_default_event_bus(),
)
def _create_generation_job_retry_service() -> GenerationJobRetryService:
# Import lazily so route discovery does not construct provider runtimes.
from backend.routes.node_routes import _create_generation_service
generation_service = _create_generation_service()
return GenerationJobRetryService(
queue_service=generation_service.queue_service,
generation_service=generation_service,
)