"""Repository for generation tasks and queue status.""" from __future__ import annotations import json from typing import Any from backend.dto.job_dto import GenerationTask from database.db_helper import Database TASK_COLUMNS = """ task_id, profile_id, workflow_id, node_id, task_type, status, model_route, progress, idempotency_key, request_json, result_json, error_message, metadata_json, created_at, updated_at """ class JobRepository: def __init__(self, database: Database): self.database = database @property def placeholder(self) -> str: return self.database._placeholder def create(self, payload: dict[str, Any]) -> GenerationTask: placeholder = self.placeholder conn = self.database.connect() try: conn.execute( f""" INSERT INTO generation_tasks ( task_id, profile_id, workflow_id, node_id, task_type, status, model_route, progress, idempotency_key, request_json, result_json, error_message, metadata_json ) VALUES ( {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder}, {placeholder} ) """, ( payload["task_id"], payload.get("profile_id"), payload.get("workflow_id"), payload.get("node_id"), payload.get("task_type", "generation"), payload.get("status", "created"), payload.get("model_route"), int(payload.get("progress", 0)), payload.get("idempotency_key"), _json_or_none(payload.get("request")), _json_or_none(payload.get("result")), payload.get("error_message"), _json_or_none(payload.get("metadata")), ), ) conn.commit() finally: conn.close() task = self.get(payload["task_id"]) if task is None: raise RuntimeError("generation task was inserted but could not be read back") return task def get(self, task_id: str) -> GenerationTask | None: placeholder = self.placeholder conn = self.database.connect() try: row = conn.execute( f"SELECT {TASK_COLUMNS} FROM generation_tasks WHERE task_id = {placeholder}", (task_id,), ).fetchone() finally: conn.close() return _row_to_task(row) if row else None def get_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None: placeholder = self.placeholder conn = self.database.connect() try: row = conn.execute( f"SELECT {TASK_COLUMNS} FROM generation_tasks WHERE idempotency_key = {placeholder}", (idempotency_key,), ).fetchone() finally: conn.close() return _row_to_task(row) if row else None def list( self, *, status: str | None = None, workflow_id: str | None = None, profile_id: str | None = None, page: int = 1, page_size: int = 30, ) -> tuple[list[GenerationTask], int]: placeholder = self.placeholder where: list[str] = [] params: list[Any] = [] if status: where.append(f"status = {placeholder}") params.append(status) if workflow_id: where.append(f"workflow_id = {placeholder}") params.append(workflow_id) if profile_id: where.append(f"profile_id = {placeholder}") params.append(profile_id) where_clause = f"WHERE {' AND '.join(where)}" if where else "" normalized_page = _page(page) normalized_page_size = _page_size(page_size) offset = (normalized_page - 1) * normalized_page_size conn = self.database.connect() try: total_row = conn.execute( f"SELECT COUNT(*) AS total FROM generation_tasks {where_clause}", tuple(params), ).fetchone() rows = conn.execute( f""" SELECT {TASK_COLUMNS} FROM generation_tasks {where_clause} ORDER BY created_at DESC LIMIT {placeholder} OFFSET {placeholder} """, tuple(params + [normalized_page_size, offset]), ).fetchall() finally: conn.close() return [_row_to_task(row) for row in rows], int(total_row["total"] if total_row else 0) def list_stale( self, *, statuses: set[str] | tuple[str, ...] | list[str], updated_before: str | None, limit: int | None = 100, ) -> list[GenerationTask]: """Return oldest tasks that were left in an executable state. Recovery decides which task sources are safe to requeue. Keeping the JSON-source filter in the service makes this query portable across the SQLite and PostgreSQL backends. Startup recovery may omit both the age boundary and limit because it runs before the new worker pool is started and must inspect every task owned by the previous process. """ normalized_statuses = tuple( sorted({str(status).strip() for status in statuses if str(status).strip()}) ) if not normalized_statuses: return [] placeholder = self.placeholder status_placeholders = ", ".join(placeholder for _ in normalized_statuses) age_clause = f"AND updated_at <= {placeholder}" if updated_before else "" limit_clause = f"LIMIT {placeholder}" if limit is not None else "" params: list[Any] = list(normalized_statuses) if updated_before: params.append(updated_before) if limit is not None: params.append(_limit(limit)) conn = self.database.connect() try: rows = conn.execute( f""" SELECT {TASK_COLUMNS} FROM generation_tasks WHERE status IN ({status_placeholders}) {age_clause} ORDER BY updated_at ASC, created_at ASC, task_id ASC {limit_clause} """, tuple(params), ).fetchall() finally: conn.close() return [_row_to_task(row) for row in rows] def update(self, task_id: str, payload: dict[str, Any]) -> GenerationTask | None: placeholder = self.placeholder assignments, params = _update_assignments(payload, placeholder) if not assignments: return self.get(task_id) assignments.append("updated_at = CURRENT_TIMESTAMP") params.append(task_id) conn = self.database.connect() try: conn.execute( f""" UPDATE generation_tasks SET {", ".join(assignments)} WHERE task_id = {placeholder} """, tuple(params), ) conn.commit() finally: conn.close() return self.get(task_id) def update_if_status( self, task_id: str, expected_statuses: set[str] | tuple[str, ...] | list[str], payload: dict[str, Any], ) -> GenerationTask | None: """Update a task only while it is still in an expected state. The status predicate and mutation run in one SQL statement. This is the compare-and-set primitive used by retry orchestration and queue workers to prevent two callers from claiming the same task. """ normalized_statuses = tuple( sorted({str(status).strip() for status in expected_statuses if str(status).strip()}) ) if not normalized_statuses: return None placeholder = self.placeholder assignments, params = _update_assignments(payload, placeholder) if not assignments: task = self.get(task_id) return task if task and task.status in normalized_statuses else None assignments.append("updated_at = CURRENT_TIMESTAMP") status_placeholders = ", ".join(placeholder for _ in normalized_statuses) params.extend([task_id, *normalized_statuses]) conn = self.database.connect() try: cursor = conn.execute( f""" UPDATE generation_tasks SET {", ".join(assignments)} WHERE task_id = {placeholder} AND status IN ({status_placeholders}) """, tuple(params), ) changed = int(cursor.rowcount or 0) > 0 conn.commit() finally: conn.close() return self.get(task_id) if changed else None def update_if_status_unless_workflow_cancelled( self, task_id: str, expected_statuses: set[str] | tuple[str, ...] | list[str], payload: dict[str, Any], ) -> GenerationTask | None: """CAS-update a task while its owning workflow is not cancelled.""" normalized_statuses = tuple( sorted({str(status).strip() for status in expected_statuses if str(status).strip()}) ) if not normalized_statuses: return None placeholder = self.placeholder assignments, params = _update_assignments(payload, placeholder) if not assignments: task = self.get(task_id) if task is None or task.status not in normalized_statuses: return None if task.workflow_id: conn = self.database.connect() try: row = conn.execute( f"SELECT status FROM profile_workflows WHERE workflow_id = {placeholder}", (task.workflow_id,), ).fetchone() finally: conn.close() if row is None or str(row["status"]) == "cancelled": return None return task assignments.append("updated_at = CURRENT_TIMESTAMP") status_placeholders = ", ".join(placeholder for _ in normalized_statuses) params.extend([task_id, *normalized_statuses, "cancelled"]) conn = self.database.connect() try: cursor = conn.execute( f""" UPDATE generation_tasks SET {", ".join(assignments)} WHERE task_id = {placeholder} AND status IN ({status_placeholders}) AND ( workflow_id IS NULL OR EXISTS ( SELECT 1 FROM profile_workflows WHERE profile_workflows.workflow_id = generation_tasks.workflow_id AND profile_workflows.status <> {placeholder} ) ) """, tuple(params), ) changed = int(cursor.rowcount or 0) > 0 conn.commit() finally: conn.close() return self.get(task_id) if changed else None def count_by_status(self) -> dict[str, int]: conn = self.database.connect() try: rows = conn.execute( """ SELECT status, COUNT(*) AS total FROM generation_tasks GROUP BY status """ ).fetchall() finally: conn.close() return {row["status"]: int(row["total"]) for row in rows} def recent(self, limit: int = 10) -> list[GenerationTask]: placeholder = self.placeholder conn = self.database.connect() try: rows = conn.execute( f""" SELECT {TASK_COLUMNS} FROM generation_tasks ORDER BY created_at DESC LIMIT {placeholder} """, (_limit(limit),), ).fetchall() finally: conn.close() return [_row_to_task(row) for row in rows] def delete_if_status( self, task_id: str, expected_statuses: set[str] | tuple[str, ...] | list[str], ) -> dict[str, Any] | None: """Delete a terminal task and release a node reference owned by it. Queue rows are not the source of generated image metadata, so terminal task records can be removed without touching gallery images. The node's JSON reference is repaired in the same transaction to avoid a dangling ``last_task_id`` blocking a later submission. """ normalized_statuses = tuple( sorted({str(status).strip() for status in expected_statuses if str(status).strip()}) ) if not normalized_statuses: return None placeholder = self.placeholder status_placeholders = ", ".join(placeholder for _ in normalized_statuses) conn = self.database.connect() try: row = conn.execute( f""" SELECT {TASK_COLUMNS} FROM generation_tasks WHERE task_id = {placeholder} AND status IN ({status_placeholders}) """, (task_id, *normalized_statuses), ).fetchone() if row is None: return None task = _row_to_task(row) cursor = conn.execute( f""" DELETE FROM generation_tasks WHERE task_id = {placeholder} AND status IN ({status_placeholders}) """, (task_id, *normalized_statuses), ) if int(cursor.rowcount or 0) <= 0: return None # Delete/claim the terminal row before touching its node. This # prevents a concurrent retry claim from succeeding after we have # already repaired the node metadata. The node read is locked on # PostgreSQL; SQLite already holds the database write lock after # the DELETE above. node_cleanup = _release_task_node_reference( conn, placeholder, task, lock_row=self.database.backend == "postgresql", ) workflow_cleanup = _recover_failed_character_workflow( conn, placeholder, task, node_cleanup=node_cleanup, lock_row=self.database.backend == "postgresql", ) conn.commit() return { "task": task, "node_cleanup": node_cleanup, "workflow_cleanup": workflow_cleanup, } finally: conn.close() def _row_to_task(row) -> GenerationTask: return GenerationTask( task_id=row["task_id"], profile_id=row["profile_id"], workflow_id=row["workflow_id"], node_id=row["node_id"], task_type=row["task_type"], status=row["status"], model_route=row["model_route"], progress=int(row["progress"] or 0), idempotency_key=row["idempotency_key"], request=_json_loads(row["request_json"]), result=_json_loads(row["result_json"]), error_message=row["error_message"], metadata=_json_loads(row["metadata_json"]), created_at=str(row["created_at"]), updated_at=str(row["updated_at"]), ) def _release_task_node_reference( conn, placeholder: str, task: GenerationTask, *, lock_row: bool = False, ) -> dict[str, Any]: if not task.node_id: return {"released": False, "reason": "task_has_no_node"} lock_clause = " FOR UPDATE" if lock_row else "" node = conn.execute( f"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = {placeholder}{lock_clause}", (task.node_id,), ).fetchone() if node is None: return {"released": False, "reason": "node_not_found", "node_id": task.node_id} metadata = _json_loads(node["metadata_json"]) if str(metadata.get("last_task_id") or "").strip() != task.task_id: return { "released": False, "reason": "node_owned_by_other_task", "node_id": task.node_id, "active_task_id": metadata.get("last_task_id"), } metadata.pop("last_task_id", None) metadata.pop("last_generate_request", None) metadata.pop("generation_failure", None) preparation = metadata.get("retry_preparation") if isinstance(preparation, dict) and str(preparation.get("task_id") or "").strip() == task.task_id: metadata.pop("retry_preparation", None) previous_status = str(node["status"] or "") next_status = "draft" if previous_status in {"failed", "cancelled"} else previous_status conn.execute( f""" UPDATE workflow_nodes SET status = {placeholder}, metadata_json = {placeholder}, updated_at = CURRENT_TIMESTAMP WHERE node_id = {placeholder} """, (next_status, _json_or_none(metadata), task.node_id), ) return { "released": True, "node_id": task.node_id, "previous_status": previous_status, "status": next_status, } def _recover_failed_character_workflow( conn, placeholder: str, task: GenerationTask, *, node_cleanup: dict[str, Any], lock_row: bool = False, ) -> dict[str, Any]: if task.status != "failed" or not task.workflow_id or node_cleanup.get("released") is not True: return {"recovered": False, "reason": "task_did_not_release_failed_node"} lock_clause = " FOR UPDATE" if lock_row else "" workflow = conn.execute( ( "SELECT workflow_type, status, metadata_json " f"FROM profile_workflows WHERE workflow_id = {placeholder}{lock_clause}" ), (task.workflow_id,), ).fetchone() if workflow is None: return {"recovered": False, "reason": "workflow_not_found", "workflow_id": task.workflow_id} workflow_type = str(workflow["workflow_type"] or "") status = str(workflow["status"] or "") if workflow_type not in {"character_fast", "character_gradual"}: return { "recovered": False, "reason": "workflow_type_not_character_generation", "workflow_id": task.workflow_id, "workflow_type": workflow_type, } if status != "failed": return { "recovered": False, "reason": "workflow_status_not_failed", "workflow_id": task.workflow_id, "status": status, } metadata = _json_loads(workflow["metadata_json"]) metadata.pop("quick_character_failure", None) preparation = metadata.get("retry_preparation") if isinstance(preparation, dict) and str(preparation.get("task_id") or "").strip() == task.task_id: metadata.pop("retry_preparation", None) conn.execute( f""" UPDATE profile_workflows SET status = {placeholder}, metadata_json = {placeholder}, updated_at = CURRENT_TIMESTAMP WHERE workflow_id = {placeholder} AND status = {placeholder} """, ("running", _json_or_none(metadata), task.workflow_id, "failed"), ) return { "recovered": True, "workflow_id": task.workflow_id, "workflow_type": workflow_type, "previous_status": "failed", "status": "running", } def _json_or_none(value: Any | None) -> str | None: if value is None: return None return json.dumps(value, ensure_ascii=False, sort_keys=True) def _update_assignments(payload: dict[str, Any], placeholder: str) -> tuple[list[str], list[Any]]: allowed = { "status", "model_route", "progress", "request", "result", "error_message", "metadata", } assignments: list[str] = [] params: list[Any] = [] for key in allowed: if key not in payload: continue column = key value = payload[key] if key == "request": column = "request_json" value = _json_or_none(value) elif key == "result": column = "result_json" value = _json_or_none(value) elif key == "metadata": column = "metadata_json" value = _json_or_none(value) elif key == "progress": value = int(value) assignments.append(f"{column} = {placeholder}") params.append(value) return assignments, params def _limit(value: int) -> int: try: limit = int(value) except (TypeError, ValueError): limit = 10 return min(max(limit, 1), 100) def _page(value: Any) -> int: try: page = int(value) except (TypeError, ValueError): page = 1 return max(page, 1) def _page_size(value: Any) -> int: try: page_size = int(value) except (TypeError, ValueError): page_size = 30 return min(max(page_size, 1), 100) def _json_loads(value: str | None) -> dict[str, Any]: if not value: return {} return json.loads(value)