"""Job and queue route handlers.""" from __future__ import annotations from typing import Any from backend.routes.common import handle_request from backend.routes.params import int_or_default, optional_str from backend.services.event_stream_service import get_default_event_bus from backend.services.generation_job_retry_service import GenerationJobRetryService from backend.services.queue_service import QueueService from core.audit_service import AuditService from core.identity import RequestContext from core.security import require_authenticated, require_permission, require_task_retry, require_write from database.db_helper import Database, load_database_settings from database.repositories.job_repository import JobRepository def list_jobs( ctx: RequestContext, query: dict[str, Any] | None = None, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() params = query or {} def action() -> dict[str, Any]: require_authenticated(ctx) result = queue_service.list_jobs( status=optional_str(params.get("status")), workflow_id=optional_str(params.get("workflow_id")), profile_id=optional_str(params.get("profile_id")), page=int_or_default(params.get("page"), 1), page_size=int_or_default(params.get("page_size"), 30), ).to_dict() if not ctx.has_permission("can_manage_materials"): visible = [item for item in result["items"] if item.get("metadata", {}).get("source") != "advanced_material"] result = {**result, "items": visible, "total": max(int(result.get("total") or 0) - (len(result["items"]) - len(visible)), 0)} return result return handle_request(ctx, action) def get_job( ctx: RequestContext, task_id: str, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() def action() -> dict[str, Any]: require_authenticated(ctx) task = queue_service.get_job(task_id) if task.metadata.get("source") == "advanced_material": require_permission(ctx, "can_manage_materials") return task.to_dict() return handle_request(ctx, action) def delete_job( ctx: RequestContext, task_id: str, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() def action() -> dict[str, Any]: require_task_retry(ctx) return queue_service.delete_job(task_id, ctx) return handle_request(ctx, action) def cancel_job( ctx: RequestContext, task_id: str, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() def action() -> dict[str, Any]: require_write(ctx) task = queue_service.get_job(task_id) if task.metadata.get("source") == "advanced_material": require_permission(ctx, "can_manage_materials") from backend.services.advanced_material_service import create_advanced_material_service import_id = str(task.metadata.get("advanced_material_import_id") or "").strip() return create_advanced_material_service().cancel_import(import_id, ctx) return queue_service.cancel_job(task_id, ctx).to_dict() return handle_request(ctx, action) def retry_job( ctx: RequestContext, task_id: str, payload: dict[str, Any] | None = None, service: GenerationJobRetryService | None = None, ) -> dict: retry_service = service def action() -> dict[str, Any]: require_task_retry(ctx) queue_service = _create_queue_service() task = queue_service.get_job(task_id) return _retry_task(task, ctx, payload=payload, retry_service=retry_service) return handle_request(ctx, action) def retry_failed_jobs( ctx: RequestContext, payload: dict[str, Any] | None = None, ) -> dict: params = payload or {} def action() -> dict[str, Any]: require_task_retry(ctx) queue_service = _create_queue_service() limit = max(1, min(int_or_default(params.get("limit"), 100), 100)) tasks = [] seen_task_ids: set[str] = set() for status in ("failed", "dead_letter", "cancelled"): remaining = limit - len(tasks) if remaining <= 0: break for task in queue_service.list_jobs(status=status, page=1, page_size=remaining).items: if task.task_id not in seen_task_ids: seen_task_ids.add(task.task_id) tasks.append(task) tasks = _latest_retryable_tasks(tasks) retried: list[dict[str, Any]] = [] errors: list[dict[str, Any]] = [] retry_service: GenerationJobRetryService | None = None for task in tasks: try: if task.metadata.get("source") != "advanced_material": retry_service = retry_service or _create_generation_job_retry_service() retried.append(_retry_task(task, ctx, retry_service=retry_service)) except Exception as exc: errors.append(_retry_error_payload(task.task_id, exc)) return { "retried": retried, "errors": errors, "total": len(retried), "processed": len(retried) + len(errors), "limit": limit, "remaining_by_status": { status: queue_service.repository.count_by_status().get(status, 0) for status in ("failed", "dead_letter", "cancelled") }, } return handle_request(ctx, action) def _latest_retryable_tasks(tasks: list[Any]) -> list[Any]: selected: list[Any] = [] lineage_indexes: dict[tuple[str, str], int] = {} for task in tasks: lineage_key = _retry_lineage_key(task) if lineage_key is None: selected.append(task) continue previous_index = lineage_indexes.get(lineage_key) if previous_index is None: lineage_indexes[lineage_key] = len(selected) selected.append(task) continue previous_task = selected[previous_index] if _task_sort_key(task) > _task_sort_key(previous_task): selected[previous_index] = task return selected def _retry_lineage_key(task: Any) -> tuple[str, str] | None: metadata = task.metadata if isinstance(getattr(task, "metadata", None), dict) else {} source_id = str(metadata.get("advanced_style_source_id") or "").strip() case_id = str(metadata.get("advanced_style_validation_case_id") or "").strip() if task.task_type == "advanced_style_source_recognition" and source_id: return ("source", source_id) if task.task_type == "advanced_style_validation_generation" and case_id: return ("validation", case_id) return None def _task_sort_key(task: Any) -> str: return str(getattr(task, "updated_at", "") or getattr(task, "created_at", "") or "") def _retry_task( task: Any, ctx: RequestContext, *, payload: dict[str, Any] | None = None, retry_service: GenerationJobRetryService | None = None, ) -> dict[str, Any]: if task.metadata.get("source") == "advanced_material": require_permission(ctx, "can_manage_materials") from backend.services.advanced_material_service import create_advanced_material_service advanced_service = create_advanced_material_service() import_id = str(task.metadata.get("advanced_material_import_id") or "").strip() if task.task_type == "advanced_style_source_recognition": source_id = str(task.metadata.get("advanced_style_source_id") or "").strip() result = advanced_service.style_sets.retry_source(import_id, source_id, ctx) elif task.task_type == "advanced_style_validation_generation": case_id = str(task.metadata.get("advanced_style_validation_case_id") or "").strip() result = advanced_service.style_sets.retry_validation_case(import_id, case_id, ctx) else: result = advanced_service.retry_import(import_id, ctx) if isinstance(result, dict) and isinstance(result.get("task"), dict): return result["task"] return result resolved_retry_service = retry_service or _create_generation_job_retry_service() return resolved_retry_service.retry_job(task.task_id, ctx, overrides=payload or {}).to_dict() def _retry_error_payload(task_id: str, exc: Exception) -> dict[str, Any]: error_code = getattr(getattr(exc, "error_code", None), "value", None) or str(getattr(exc, "error_code", "retry_failed")) message = str(getattr(exc, "message", None) or str(exc)) raw_details = getattr(exc, "details", None) details = dict(raw_details) if isinstance(raw_details, dict) else {} return { "task_id": task_id, "error_code": error_code, "message": message, **({"replacement_task_id": details["replacement_task_id"]} if details.get("replacement_task_id") else {}), } def retry_dead_letter_jobs( ctx: RequestContext, payload: dict[str, Any] | None = None, service: GenerationJobRetryService | None = None, ) -> dict: retry_service = service or _create_generation_job_retry_service() params = payload or {} def action() -> dict[str, Any]: require_task_retry(ctx) return retry_service.retry_dead_letters(ctx, limit=int_or_default(params.get("limit"), 5)) return handle_request(ctx, action) def move_failed_jobs_to_dead_letter( ctx: RequestContext, payload: dict[str, Any] | None = None, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() params = payload or {} def action() -> dict[str, Any]: require_task_retry(ctx) return queue_service.move_failed_to_dead_letter(ctx, limit=int_or_default(params.get("limit"), 50)) return handle_request(ctx, action) def clear_failed_jobs( ctx: RequestContext, payload: dict[str, Any] | None = None, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() params = payload or {} def action() -> dict[str, Any]: require_task_retry(ctx) return queue_service.clear_failed_jobs(ctx, limit=int_or_default(params.get("limit"), 100)) return handle_request(ctx, action) def clear_terminal_jobs( ctx: RequestContext, payload: dict[str, Any] | None = None, service: QueueService | None = None, ) -> dict: queue_service = service or _create_queue_service() params = payload or {} def action() -> dict[str, Any]: require_task_retry(ctx) return queue_service.clear_terminal_jobs( ctx, statuses=params.get("statuses"), limit=int_or_default(params.get("limit"), 100), ) return handle_request(ctx, action) def _create_queue_service() -> QueueService: database = Database(load_database_settings()) return QueueService( repository=JobRepository(database), audit_service=AuditService(database), event_bus=get_default_event_bus(), ) def _create_generation_job_retry_service() -> GenerationJobRetryService: # Import lazily so route discovery does not construct provider runtimes. from backend.routes.node_routes import _create_generation_service generation_service = _create_generation_service() return GenerationJobRetryService( queue_service=generation_service.queue_service, generation_service=generation_service, )