628 行
22 KiB
Python
628 行
22 KiB
Python
"""Repository for generation tasks and queue status."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from backend.dto.job_dto import GenerationTask
|
|
from database.db_helper import Database
|
|
|
|
|
|
TASK_COLUMNS = """
|
|
task_id,
|
|
profile_id,
|
|
workflow_id,
|
|
node_id,
|
|
task_type,
|
|
status,
|
|
model_route,
|
|
progress,
|
|
idempotency_key,
|
|
request_json,
|
|
result_json,
|
|
error_message,
|
|
metadata_json,
|
|
created_at,
|
|
updated_at
|
|
"""
|
|
|
|
|
|
class JobRepository:
|
|
def __init__(self, database: Database):
|
|
self.database = database
|
|
|
|
@property
|
|
def placeholder(self) -> str:
|
|
return self.database._placeholder
|
|
|
|
def create(self, payload: dict[str, Any]) -> GenerationTask:
|
|
placeholder = self.placeholder
|
|
conn = self.database.connect()
|
|
try:
|
|
conn.execute(
|
|
f"""
|
|
INSERT INTO generation_tasks (
|
|
task_id,
|
|
profile_id,
|
|
workflow_id,
|
|
node_id,
|
|
task_type,
|
|
status,
|
|
model_route,
|
|
progress,
|
|
idempotency_key,
|
|
request_json,
|
|
result_json,
|
|
error_message,
|
|
metadata_json
|
|
) VALUES (
|
|
{placeholder}, {placeholder}, {placeholder}, {placeholder},
|
|
{placeholder}, {placeholder}, {placeholder}, {placeholder},
|
|
{placeholder}, {placeholder}, {placeholder}, {placeholder},
|
|
{placeholder}
|
|
)
|
|
""",
|
|
(
|
|
payload["task_id"],
|
|
payload.get("profile_id"),
|
|
payload.get("workflow_id"),
|
|
payload.get("node_id"),
|
|
payload.get("task_type", "generation"),
|
|
payload.get("status", "created"),
|
|
payload.get("model_route"),
|
|
int(payload.get("progress", 0)),
|
|
payload.get("idempotency_key"),
|
|
_json_or_none(payload.get("request")),
|
|
_json_or_none(payload.get("result")),
|
|
payload.get("error_message"),
|
|
_json_or_none(payload.get("metadata")),
|
|
),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
task = self.get(payload["task_id"])
|
|
if task is None:
|
|
raise RuntimeError("generation task was inserted but could not be read back")
|
|
return task
|
|
|
|
def get(self, task_id: str) -> GenerationTask | None:
|
|
placeholder = self.placeholder
|
|
conn = self.database.connect()
|
|
try:
|
|
row = conn.execute(
|
|
f"SELECT {TASK_COLUMNS} FROM generation_tasks WHERE task_id = {placeholder}",
|
|
(task_id,),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
return _row_to_task(row) if row else None
|
|
|
|
def get_by_idempotency_key(self, idempotency_key: str) -> GenerationTask | None:
|
|
placeholder = self.placeholder
|
|
conn = self.database.connect()
|
|
try:
|
|
row = conn.execute(
|
|
f"SELECT {TASK_COLUMNS} FROM generation_tasks WHERE idempotency_key = {placeholder}",
|
|
(idempotency_key,),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
return _row_to_task(row) if row else None
|
|
|
|
def list(
|
|
self,
|
|
*,
|
|
status: str | None = None,
|
|
workflow_id: str | None = None,
|
|
profile_id: str | None = None,
|
|
page: int = 1,
|
|
page_size: int = 30,
|
|
) -> tuple[list[GenerationTask], int]:
|
|
placeholder = self.placeholder
|
|
where: list[str] = []
|
|
params: list[Any] = []
|
|
if status:
|
|
where.append(f"status = {placeholder}")
|
|
params.append(status)
|
|
if workflow_id:
|
|
where.append(f"workflow_id = {placeholder}")
|
|
params.append(workflow_id)
|
|
if profile_id:
|
|
where.append(f"profile_id = {placeholder}")
|
|
params.append(profile_id)
|
|
where_clause = f"WHERE {' AND '.join(where)}" if where else ""
|
|
normalized_page = _page(page)
|
|
normalized_page_size = _page_size(page_size)
|
|
offset = (normalized_page - 1) * normalized_page_size
|
|
conn = self.database.connect()
|
|
try:
|
|
total_row = conn.execute(
|
|
f"SELECT COUNT(*) AS total FROM generation_tasks {where_clause}",
|
|
tuple(params),
|
|
).fetchone()
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT {TASK_COLUMNS}
|
|
FROM generation_tasks
|
|
{where_clause}
|
|
ORDER BY created_at DESC
|
|
LIMIT {placeholder} OFFSET {placeholder}
|
|
""",
|
|
tuple(params + [normalized_page_size, offset]),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [_row_to_task(row) for row in rows], int(total_row["total"] if total_row else 0)
|
|
|
|
def list_stale(
|
|
self,
|
|
*,
|
|
statuses: set[str] | tuple[str, ...] | list[str],
|
|
updated_before: str | None,
|
|
limit: int | None = 100,
|
|
) -> list[GenerationTask]:
|
|
"""Return oldest tasks that were left in an executable state.
|
|
|
|
Recovery decides which task sources are safe to requeue. Keeping the
|
|
JSON-source filter in the service makes this query portable across the
|
|
SQLite and PostgreSQL backends. Startup recovery may omit both the
|
|
age boundary and limit because it runs before the new worker pool is
|
|
started and must inspect every task owned by the previous process.
|
|
"""
|
|
|
|
normalized_statuses = tuple(
|
|
sorted({str(status).strip() for status in statuses if str(status).strip()})
|
|
)
|
|
if not normalized_statuses:
|
|
return []
|
|
placeholder = self.placeholder
|
|
status_placeholders = ", ".join(placeholder for _ in normalized_statuses)
|
|
age_clause = f"AND updated_at <= {placeholder}" if updated_before else ""
|
|
limit_clause = f"LIMIT {placeholder}" if limit is not None else ""
|
|
params: list[Any] = list(normalized_statuses)
|
|
if updated_before:
|
|
params.append(updated_before)
|
|
if limit is not None:
|
|
params.append(_limit(limit))
|
|
conn = self.database.connect()
|
|
try:
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT {TASK_COLUMNS}
|
|
FROM generation_tasks
|
|
WHERE status IN ({status_placeholders})
|
|
{age_clause}
|
|
ORDER BY updated_at ASC, created_at ASC, task_id ASC
|
|
{limit_clause}
|
|
""",
|
|
tuple(params),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [_row_to_task(row) for row in rows]
|
|
|
|
def update(self, task_id: str, payload: dict[str, Any]) -> GenerationTask | None:
|
|
placeholder = self.placeholder
|
|
assignments, params = _update_assignments(payload, placeholder)
|
|
if not assignments:
|
|
return self.get(task_id)
|
|
assignments.append("updated_at = CURRENT_TIMESTAMP")
|
|
params.append(task_id)
|
|
conn = self.database.connect()
|
|
try:
|
|
conn.execute(
|
|
f"""
|
|
UPDATE generation_tasks
|
|
SET {", ".join(assignments)}
|
|
WHERE task_id = {placeholder}
|
|
""",
|
|
tuple(params),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return self.get(task_id)
|
|
|
|
def update_if_status(
|
|
self,
|
|
task_id: str,
|
|
expected_statuses: set[str] | tuple[str, ...] | list[str],
|
|
payload: dict[str, Any],
|
|
) -> GenerationTask | None:
|
|
"""Update a task only while it is still in an expected state.
|
|
|
|
The status predicate and mutation run in one SQL statement. This is the
|
|
compare-and-set primitive used by retry orchestration and queue workers
|
|
to prevent two callers from claiming the same task.
|
|
"""
|
|
|
|
normalized_statuses = tuple(
|
|
sorted({str(status).strip() for status in expected_statuses if str(status).strip()})
|
|
)
|
|
if not normalized_statuses:
|
|
return None
|
|
placeholder = self.placeholder
|
|
assignments, params = _update_assignments(payload, placeholder)
|
|
if not assignments:
|
|
task = self.get(task_id)
|
|
return task if task and task.status in normalized_statuses else None
|
|
assignments.append("updated_at = CURRENT_TIMESTAMP")
|
|
status_placeholders = ", ".join(placeholder for _ in normalized_statuses)
|
|
params.extend([task_id, *normalized_statuses])
|
|
conn = self.database.connect()
|
|
try:
|
|
cursor = conn.execute(
|
|
f"""
|
|
UPDATE generation_tasks
|
|
SET {", ".join(assignments)}
|
|
WHERE task_id = {placeholder}
|
|
AND status IN ({status_placeholders})
|
|
""",
|
|
tuple(params),
|
|
)
|
|
changed = int(cursor.rowcount or 0) > 0
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return self.get(task_id) if changed else None
|
|
|
|
def update_if_status_unless_workflow_cancelled(
|
|
self,
|
|
task_id: str,
|
|
expected_statuses: set[str] | tuple[str, ...] | list[str],
|
|
payload: dict[str, Any],
|
|
) -> GenerationTask | None:
|
|
"""CAS-update a task while its owning workflow is not cancelled."""
|
|
|
|
normalized_statuses = tuple(
|
|
sorted({str(status).strip() for status in expected_statuses if str(status).strip()})
|
|
)
|
|
if not normalized_statuses:
|
|
return None
|
|
placeholder = self.placeholder
|
|
assignments, params = _update_assignments(payload, placeholder)
|
|
if not assignments:
|
|
task = self.get(task_id)
|
|
if task is None or task.status not in normalized_statuses:
|
|
return None
|
|
if task.workflow_id:
|
|
conn = self.database.connect()
|
|
try:
|
|
row = conn.execute(
|
|
f"SELECT status FROM profile_workflows WHERE workflow_id = {placeholder}",
|
|
(task.workflow_id,),
|
|
).fetchone()
|
|
finally:
|
|
conn.close()
|
|
if row is None or str(row["status"]) == "cancelled":
|
|
return None
|
|
return task
|
|
assignments.append("updated_at = CURRENT_TIMESTAMP")
|
|
status_placeholders = ", ".join(placeholder for _ in normalized_statuses)
|
|
params.extend([task_id, *normalized_statuses, "cancelled"])
|
|
conn = self.database.connect()
|
|
try:
|
|
cursor = conn.execute(
|
|
f"""
|
|
UPDATE generation_tasks
|
|
SET {", ".join(assignments)}
|
|
WHERE task_id = {placeholder}
|
|
AND status IN ({status_placeholders})
|
|
AND (
|
|
workflow_id IS NULL
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM profile_workflows
|
|
WHERE profile_workflows.workflow_id = generation_tasks.workflow_id
|
|
AND profile_workflows.status <> {placeholder}
|
|
)
|
|
)
|
|
""",
|
|
tuple(params),
|
|
)
|
|
changed = int(cursor.rowcount or 0) > 0
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return self.get(task_id) if changed else None
|
|
|
|
def count_by_status(self) -> dict[str, int]:
|
|
conn = self.database.connect()
|
|
try:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT status, COUNT(*) AS total
|
|
FROM generation_tasks
|
|
GROUP BY status
|
|
"""
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return {row["status"]: int(row["total"]) for row in rows}
|
|
|
|
def recent(self, limit: int = 10) -> list[GenerationTask]:
|
|
placeholder = self.placeholder
|
|
conn = self.database.connect()
|
|
try:
|
|
rows = conn.execute(
|
|
f"""
|
|
SELECT {TASK_COLUMNS}
|
|
FROM generation_tasks
|
|
ORDER BY created_at DESC
|
|
LIMIT {placeholder}
|
|
""",
|
|
(_limit(limit),),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [_row_to_task(row) for row in rows]
|
|
|
|
def delete_if_status(
|
|
self,
|
|
task_id: str,
|
|
expected_statuses: set[str] | tuple[str, ...] | list[str],
|
|
) -> dict[str, Any] | None:
|
|
"""Delete a terminal task and release a node reference owned by it.
|
|
|
|
Queue rows are not the source of generated image metadata, so terminal
|
|
task records can be removed without touching gallery images. The
|
|
node's JSON reference is repaired in the same transaction to avoid a
|
|
dangling ``last_task_id`` blocking a later submission.
|
|
"""
|
|
|
|
normalized_statuses = tuple(
|
|
sorted({str(status).strip() for status in expected_statuses if str(status).strip()})
|
|
)
|
|
if not normalized_statuses:
|
|
return None
|
|
placeholder = self.placeholder
|
|
status_placeholders = ", ".join(placeholder for _ in normalized_statuses)
|
|
conn = self.database.connect()
|
|
try:
|
|
row = conn.execute(
|
|
f"""
|
|
SELECT {TASK_COLUMNS}
|
|
FROM generation_tasks
|
|
WHERE task_id = {placeholder}
|
|
AND status IN ({status_placeholders})
|
|
""",
|
|
(task_id, *normalized_statuses),
|
|
).fetchone()
|
|
if row is None:
|
|
return None
|
|
task = _row_to_task(row)
|
|
cursor = conn.execute(
|
|
f"""
|
|
DELETE FROM generation_tasks
|
|
WHERE task_id = {placeholder}
|
|
AND status IN ({status_placeholders})
|
|
""",
|
|
(task_id, *normalized_statuses),
|
|
)
|
|
if int(cursor.rowcount or 0) <= 0:
|
|
return None
|
|
# Delete/claim the terminal row before touching its node. This
|
|
# prevents a concurrent retry claim from succeeding after we have
|
|
# already repaired the node metadata. The node read is locked on
|
|
# PostgreSQL; SQLite already holds the database write lock after
|
|
# the DELETE above.
|
|
node_cleanup = _release_task_node_reference(
|
|
conn,
|
|
placeholder,
|
|
task,
|
|
lock_row=self.database.backend == "postgresql",
|
|
)
|
|
workflow_cleanup = _recover_failed_character_workflow(
|
|
conn,
|
|
placeholder,
|
|
task,
|
|
node_cleanup=node_cleanup,
|
|
lock_row=self.database.backend == "postgresql",
|
|
)
|
|
conn.commit()
|
|
return {
|
|
"task": task,
|
|
"node_cleanup": node_cleanup,
|
|
"workflow_cleanup": workflow_cleanup,
|
|
}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _row_to_task(row) -> GenerationTask:
|
|
return GenerationTask(
|
|
task_id=row["task_id"],
|
|
profile_id=row["profile_id"],
|
|
workflow_id=row["workflow_id"],
|
|
node_id=row["node_id"],
|
|
task_type=row["task_type"],
|
|
status=row["status"],
|
|
model_route=row["model_route"],
|
|
progress=int(row["progress"] or 0),
|
|
idempotency_key=row["idempotency_key"],
|
|
request=_json_loads(row["request_json"]),
|
|
result=_json_loads(row["result_json"]),
|
|
error_message=row["error_message"],
|
|
metadata=_json_loads(row["metadata_json"]),
|
|
created_at=str(row["created_at"]),
|
|
updated_at=str(row["updated_at"]),
|
|
)
|
|
|
|
|
|
def _release_task_node_reference(
|
|
conn,
|
|
placeholder: str,
|
|
task: GenerationTask,
|
|
*,
|
|
lock_row: bool = False,
|
|
) -> dict[str, Any]:
|
|
if not task.node_id:
|
|
return {"released": False, "reason": "task_has_no_node"}
|
|
lock_clause = " FOR UPDATE" if lock_row else ""
|
|
node = conn.execute(
|
|
f"SELECT status, metadata_json FROM workflow_nodes WHERE node_id = {placeholder}{lock_clause}",
|
|
(task.node_id,),
|
|
).fetchone()
|
|
if node is None:
|
|
return {"released": False, "reason": "node_not_found", "node_id": task.node_id}
|
|
metadata = _json_loads(node["metadata_json"])
|
|
if str(metadata.get("last_task_id") or "").strip() != task.task_id:
|
|
return {
|
|
"released": False,
|
|
"reason": "node_owned_by_other_task",
|
|
"node_id": task.node_id,
|
|
"active_task_id": metadata.get("last_task_id"),
|
|
}
|
|
|
|
metadata.pop("last_task_id", None)
|
|
metadata.pop("last_generate_request", None)
|
|
metadata.pop("generation_failure", None)
|
|
preparation = metadata.get("retry_preparation")
|
|
if isinstance(preparation, dict) and str(preparation.get("task_id") or "").strip() == task.task_id:
|
|
metadata.pop("retry_preparation", None)
|
|
previous_status = str(node["status"] or "")
|
|
next_status = "draft" if previous_status in {"failed", "cancelled"} else previous_status
|
|
conn.execute(
|
|
f"""
|
|
UPDATE workflow_nodes
|
|
SET status = {placeholder}, metadata_json = {placeholder}, updated_at = CURRENT_TIMESTAMP
|
|
WHERE node_id = {placeholder}
|
|
""",
|
|
(next_status, _json_or_none(metadata), task.node_id),
|
|
)
|
|
return {
|
|
"released": True,
|
|
"node_id": task.node_id,
|
|
"previous_status": previous_status,
|
|
"status": next_status,
|
|
}
|
|
|
|
|
|
def _recover_failed_character_workflow(
|
|
conn,
|
|
placeholder: str,
|
|
task: GenerationTask,
|
|
*,
|
|
node_cleanup: dict[str, Any],
|
|
lock_row: bool = False,
|
|
) -> dict[str, Any]:
|
|
if task.status != "failed" or not task.workflow_id or node_cleanup.get("released") is not True:
|
|
return {"recovered": False, "reason": "task_did_not_release_failed_node"}
|
|
lock_clause = " FOR UPDATE" if lock_row else ""
|
|
workflow = conn.execute(
|
|
(
|
|
"SELECT workflow_type, status, metadata_json "
|
|
f"FROM profile_workflows WHERE workflow_id = {placeholder}{lock_clause}"
|
|
),
|
|
(task.workflow_id,),
|
|
).fetchone()
|
|
if workflow is None:
|
|
return {"recovered": False, "reason": "workflow_not_found", "workflow_id": task.workflow_id}
|
|
workflow_type = str(workflow["workflow_type"] or "")
|
|
status = str(workflow["status"] or "")
|
|
if workflow_type not in {"character_fast", "character_gradual"}:
|
|
return {
|
|
"recovered": False,
|
|
"reason": "workflow_type_not_character_generation",
|
|
"workflow_id": task.workflow_id,
|
|
"workflow_type": workflow_type,
|
|
}
|
|
if status != "failed":
|
|
return {
|
|
"recovered": False,
|
|
"reason": "workflow_status_not_failed",
|
|
"workflow_id": task.workflow_id,
|
|
"status": status,
|
|
}
|
|
metadata = _json_loads(workflow["metadata_json"])
|
|
metadata.pop("quick_character_failure", None)
|
|
preparation = metadata.get("retry_preparation")
|
|
if isinstance(preparation, dict) and str(preparation.get("task_id") or "").strip() == task.task_id:
|
|
metadata.pop("retry_preparation", None)
|
|
conn.execute(
|
|
f"""
|
|
UPDATE profile_workflows
|
|
SET status = {placeholder}, metadata_json = {placeholder}, updated_at = CURRENT_TIMESTAMP
|
|
WHERE workflow_id = {placeholder}
|
|
AND status = {placeholder}
|
|
""",
|
|
("running", _json_or_none(metadata), task.workflow_id, "failed"),
|
|
)
|
|
return {
|
|
"recovered": True,
|
|
"workflow_id": task.workflow_id,
|
|
"workflow_type": workflow_type,
|
|
"previous_status": "failed",
|
|
"status": "running",
|
|
}
|
|
|
|
|
|
def _json_or_none(value: Any | None) -> str | None:
|
|
if value is None:
|
|
return None
|
|
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
|
|
|
|
def _update_assignments(payload: dict[str, Any], placeholder: str) -> tuple[list[str], list[Any]]:
|
|
allowed = {
|
|
"status",
|
|
"model_route",
|
|
"progress",
|
|
"request",
|
|
"result",
|
|
"error_message",
|
|
"metadata",
|
|
}
|
|
assignments: list[str] = []
|
|
params: list[Any] = []
|
|
for key in allowed:
|
|
if key not in payload:
|
|
continue
|
|
column = key
|
|
value = payload[key]
|
|
if key == "request":
|
|
column = "request_json"
|
|
value = _json_or_none(value)
|
|
elif key == "result":
|
|
column = "result_json"
|
|
value = _json_or_none(value)
|
|
elif key == "metadata":
|
|
column = "metadata_json"
|
|
value = _json_or_none(value)
|
|
elif key == "progress":
|
|
value = int(value)
|
|
assignments.append(f"{column} = {placeholder}")
|
|
params.append(value)
|
|
return assignments, params
|
|
|
|
|
|
def _limit(value: int) -> int:
|
|
try:
|
|
limit = int(value)
|
|
except (TypeError, ValueError):
|
|
limit = 10
|
|
return min(max(limit, 1), 100)
|
|
|
|
|
|
def _page(value: Any) -> int:
|
|
try:
|
|
page = int(value)
|
|
except (TypeError, ValueError):
|
|
page = 1
|
|
return max(page, 1)
|
|
|
|
|
|
def _page_size(value: Any) -> int:
|
|
try:
|
|
page_size = int(value)
|
|
except (TypeError, ValueError):
|
|
page_size = 30
|
|
return min(max(page_size, 1), 100)
|
|
|
|
|
|
def _json_loads(value: str | None) -> dict[str, Any]:
|
|
if not value:
|
|
return {}
|
|
return json.loads(value)
|