2026-07-06 02:19:07 +08:00
|
|
|
"""Persistent queue worker primitives."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
2026-07-20 17:57:02 +08:00
|
|
|
import logging
|
2026-07-06 02:19:07 +08:00
|
|
|
import threading
|
|
|
|
|
from collections.abc import Callable
|
|
|
|
|
from dataclasses import dataclass, field
|
2026-07-06 02:27:07 +08:00
|
|
|
from datetime import datetime, timedelta, timezone
|
2026-07-06 02:19:07 +08:00
|
|
|
from typing import Any
|
|
|
|
|
|
2026-07-20 17:57:02 +08:00
|
|
|
from backend.dto.common import ApiError
|
2026-07-06 02:19:07 +08:00
|
|
|
from backend.dto.job_dto import GenerationTask
|
|
|
|
|
from backend.services.queue_service import QueueService
|
|
|
|
|
from core.identity import RequestContext
|
|
|
|
|
|
2026-07-20 17:57:02 +08:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
|
|
|
|
|
WorkerHandler = Callable[[GenerationTask], "WorkerTaskUpdate | dict[str, Any] | None"]
|
2026-07-15 23:01:07 +08:00
|
|
|
WorkerTaskFilter = Callable[[GenerationTask], bool]
|
|
|
|
|
WorkerTerminalCallback = Callable[[GenerationTask, GenerationTask, Exception | None], None]
|
2026-07-06 02:19:07 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class WorkerTaskUpdate:
|
|
|
|
|
status: str = "succeeded"
|
|
|
|
|
progress: int = 100
|
|
|
|
|
result: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
error_message: str | None = None
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:27:07 +08:00
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class RetryPolicy:
|
|
|
|
|
max_attempts: int = 1
|
|
|
|
|
base_delay_seconds: float = 0.0
|
|
|
|
|
backoff_multiplier: float = 2.0
|
|
|
|
|
retryable_error_types: tuple[str, ...] = ()
|
|
|
|
|
|
|
|
|
|
def should_retry(self, attempt: int, exc: Exception) -> bool:
|
|
|
|
|
if attempt >= max(int(self.max_attempts), 1):
|
|
|
|
|
return False
|
|
|
|
|
if not self.retryable_error_types:
|
|
|
|
|
return True
|
|
|
|
|
return type(exc).__name__ in self.retryable_error_types
|
|
|
|
|
|
|
|
|
|
def next_retry_at(self, attempt: int) -> str:
|
|
|
|
|
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
|
|
|
|
|
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
@dataclass(frozen=True)
|
|
|
|
|
class WorkerRunResult:
|
|
|
|
|
worker_id: str
|
|
|
|
|
status: str
|
|
|
|
|
task_id: str | None = None
|
|
|
|
|
processed: int = 0
|
|
|
|
|
succeeded: int = 0
|
|
|
|
|
failed: int = 0
|
|
|
|
|
error_message: str | None = None
|
|
|
|
|
|
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
|
|
|
return {
|
|
|
|
|
"worker_id": self.worker_id,
|
|
|
|
|
"status": self.status,
|
|
|
|
|
"task_id": self.task_id,
|
|
|
|
|
"processed": self.processed,
|
|
|
|
|
"succeeded": self.succeeded,
|
|
|
|
|
"failed": self.failed,
|
|
|
|
|
"error_message": self.error_message,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class QueueWorker:
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
queue_service: QueueService,
|
|
|
|
|
handler: WorkerHandler,
|
|
|
|
|
*,
|
|
|
|
|
worker_id: str = "imagestudio-worker",
|
|
|
|
|
interval_seconds: float = 1.0,
|
|
|
|
|
claim_batch_size: int = 5,
|
2026-07-06 02:27:07 +08:00
|
|
|
retry_policy: RetryPolicy | None = None,
|
2026-07-15 23:01:07 +08:00
|
|
|
task_filter: WorkerTaskFilter | None = None,
|
|
|
|
|
terminal_callback: WorkerTerminalCallback | None = None,
|
2026-07-06 02:19:07 +08:00
|
|
|
):
|
|
|
|
|
self.queue_service = queue_service
|
|
|
|
|
self.handler = handler
|
|
|
|
|
self.worker_id = worker_id
|
|
|
|
|
self.interval_seconds = max(float(interval_seconds), 0.01)
|
|
|
|
|
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
|
2026-07-06 02:27:07 +08:00
|
|
|
self.retry_policy = retry_policy or RetryPolicy()
|
2026-07-15 23:01:07 +08:00
|
|
|
self.task_filter = task_filter
|
|
|
|
|
self.terminal_callback = terminal_callback
|
2026-07-06 02:19:07 +08:00
|
|
|
self._stop_event = threading.Event()
|
|
|
|
|
self._lock = threading.Lock()
|
|
|
|
|
self._thread: threading.Thread | None = None
|
|
|
|
|
|
|
|
|
|
def run_once(self, ctx: RequestContext) -> WorkerRunResult:
|
|
|
|
|
task = self._claim_next(ctx)
|
|
|
|
|
if task is None:
|
|
|
|
|
return WorkerRunResult(worker_id=self.worker_id, status="idle")
|
|
|
|
|
try:
|
|
|
|
|
update = _normalize_update(self.handler(task))
|
2026-07-06 02:23:05 +08:00
|
|
|
latest = self.queue_service.get_job(task.task_id)
|
|
|
|
|
if latest.status == "cancelled":
|
2026-07-15 23:01:07 +08:00
|
|
|
self._notify_terminal(task, latest, None)
|
2026-07-06 02:23:05 +08:00
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status="cancelled",
|
|
|
|
|
task_id=latest.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
error_message=latest.error_message,
|
|
|
|
|
)
|
2026-07-06 02:19:07 +08:00
|
|
|
completed = self.queue_service.update_task(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": update.status,
|
|
|
|
|
"progress": _progress(update.progress),
|
|
|
|
|
"result": {**task.result, **update.result},
|
|
|
|
|
"error_message": update.error_message,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**task.metadata,
|
|
|
|
|
**update.metadata,
|
|
|
|
|
"worker": {
|
|
|
|
|
"worker_id": self.worker_id,
|
|
|
|
|
"status": update.status,
|
|
|
|
|
"finished_at": _now(),
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
2026-07-15 23:01:07 +08:00
|
|
|
self._notify_terminal(task, completed, None)
|
2026-07-06 02:19:07 +08:00
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status=completed.status,
|
|
|
|
|
task_id=completed.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
succeeded=1 if completed.status in {"completed", "succeeded"} else 0,
|
|
|
|
|
failed=1 if completed.status == "failed" else 0,
|
|
|
|
|
error_message=completed.error_message,
|
|
|
|
|
)
|
|
|
|
|
except Exception as exc:
|
2026-07-06 02:27:07 +08:00
|
|
|
return self._handle_failure(task, exc, ctx)
|
2026-07-06 02:19:07 +08:00
|
|
|
|
|
|
|
|
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
|
|
|
|
|
limit = min(max(int(max_tasks), 1), 100)
|
|
|
|
|
results: list[dict[str, Any]] = []
|
|
|
|
|
for _ in range(limit):
|
|
|
|
|
result = self.run_once(ctx)
|
|
|
|
|
results.append(result.to_dict())
|
|
|
|
|
if result.status == "idle":
|
|
|
|
|
break
|
|
|
|
|
return {
|
|
|
|
|
"worker_id": self.worker_id,
|
|
|
|
|
"processed": sum(item["processed"] for item in results),
|
|
|
|
|
"succeeded": sum(item["succeeded"] for item in results),
|
|
|
|
|
"failed": sum(item["failed"] for item in results),
|
|
|
|
|
"results": results,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def start(self, ctx: RequestContext) -> dict[str, Any]:
|
|
|
|
|
with self._lock:
|
|
|
|
|
if self.is_running():
|
|
|
|
|
return {"worker_id": self.worker_id, "running": True, "already_running": True}
|
2026-07-20 17:57:02 +08:00
|
|
|
# A previous thread may have died without clearing state. Always
|
|
|
|
|
# rebuild a fresh stop event + thread so callers can revive workers.
|
|
|
|
|
self._stop_event = threading.Event()
|
2026-07-06 02:19:07 +08:00
|
|
|
self._thread = threading.Thread(
|
|
|
|
|
target=self._run_loop,
|
|
|
|
|
args=(ctx,),
|
|
|
|
|
daemon=True,
|
|
|
|
|
name=f"ImageStudioQueueWorker-{self.worker_id}",
|
|
|
|
|
)
|
|
|
|
|
self._thread.start()
|
|
|
|
|
return {"worker_id": self.worker_id, "running": True, "already_running": False}
|
|
|
|
|
|
|
|
|
|
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
|
|
|
|
|
self._stop_event.set()
|
|
|
|
|
thread = self._thread
|
|
|
|
|
if thread is not None:
|
|
|
|
|
thread.join(timeout=max(float(timeout_seconds), 0.1))
|
|
|
|
|
return {"worker_id": self.worker_id, "running": self.is_running()}
|
|
|
|
|
|
|
|
|
|
def is_running(self) -> bool:
|
|
|
|
|
thread = self._thread
|
|
|
|
|
return bool(thread and thread.is_alive())
|
|
|
|
|
|
|
|
|
|
def _run_loop(self, ctx: RequestContext) -> None:
|
|
|
|
|
while not self._stop_event.is_set():
|
2026-07-20 17:57:02 +08:00
|
|
|
try:
|
|
|
|
|
result = self.run_once(ctx)
|
|
|
|
|
except Exception:
|
|
|
|
|
# Provider/business failures must never kill the worker thread.
|
|
|
|
|
# A dead thread permanently stalls claim of later queued tasks.
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s crashed while processing; continuing",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
)
|
|
|
|
|
self._stop_event.wait(self.interval_seconds)
|
|
|
|
|
continue
|
2026-07-06 02:19:07 +08:00
|
|
|
if result.status == "idle":
|
|
|
|
|
self._stop_event.wait(self.interval_seconds)
|
|
|
|
|
|
|
|
|
|
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
|
|
|
|
|
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
|
|
|
|
|
for candidate in queued:
|
2026-07-06 02:27:07 +08:00
|
|
|
if _retry_deferred(candidate):
|
|
|
|
|
continue
|
2026-07-15 23:01:07 +08:00
|
|
|
if self.task_filter is not None and not self.task_filter(candidate):
|
|
|
|
|
continue
|
2026-07-11 21:50:34 +08:00
|
|
|
claimed = self.queue_service.claim_queued_task(
|
|
|
|
|
candidate.task_id,
|
|
|
|
|
ctx,
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
)
|
|
|
|
|
if claimed is not None:
|
|
|
|
|
return claimed
|
2026-07-06 02:19:07 +08:00
|
|
|
return None
|
|
|
|
|
|
2026-07-06 02:27:07 +08:00
|
|
|
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
|
2026-07-20 17:57:02 +08:00
|
|
|
"""Mark the claimed task failed without ever re-raising.
|
|
|
|
|
|
|
|
|
|
Queue progress is more important than perfect failure bookkeeping. Any
|
|
|
|
|
secondary error while recording failure must leave the worker thread
|
|
|
|
|
alive so later queued work can still be claimed.
|
|
|
|
|
"""
|
2026-07-06 02:19:07 +08:00
|
|
|
message = _safe_error_message(exc)
|
2026-07-20 17:57:02 +08:00
|
|
|
latest = task
|
|
|
|
|
try:
|
|
|
|
|
latest = self.queue_service.get_job(task.task_id)
|
|
|
|
|
except Exception:
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s could not reload task %s during failure handling",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
task.task_id,
|
|
|
|
|
)
|
|
|
|
|
latest = task
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
if latest.status == "cancelled":
|
|
|
|
|
self._notify_terminal(task, latest, exc)
|
|
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status="cancelled",
|
|
|
|
|
task_id=latest.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
error_message=latest.error_message,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s failed while handling cancelled task %s",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
task.task_id,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
attempt = _int(
|
|
|
|
|
(latest.metadata or {}).get("worker_attempt"),
|
|
|
|
|
_int((task.metadata or {}).get("worker_attempt"), 1),
|
2026-07-06 02:19:07 +08:00
|
|
|
)
|
2026-07-20 17:57:02 +08:00
|
|
|
base_metadata = dict(task.metadata or {})
|
|
|
|
|
try:
|
|
|
|
|
if isinstance(latest.metadata, dict):
|
|
|
|
|
base_metadata = {**base_metadata, **latest.metadata}
|
|
|
|
|
except Exception:
|
|
|
|
|
base_metadata = dict(task.metadata or {})
|
|
|
|
|
|
|
|
|
|
failed = latest
|
|
|
|
|
try:
|
|
|
|
|
failed = self.queue_service.update_task(
|
|
|
|
|
task.task_id,
|
2026-07-06 02:27:07 +08:00
|
|
|
{
|
2026-07-20 17:57:02 +08:00
|
|
|
"status": "failed",
|
|
|
|
|
"error_message": message,
|
2026-07-06 02:27:07 +08:00
|
|
|
"metadata": {
|
2026-07-20 17:57:02 +08:00
|
|
|
**base_metadata,
|
|
|
|
|
"worker": {
|
|
|
|
|
"worker_id": self.worker_id,
|
|
|
|
|
"status": "failed",
|
2026-07-06 02:27:07 +08:00
|
|
|
"error_type": type(exc).__name__,
|
2026-07-20 17:57:02 +08:00
|
|
|
"failed_at": _now(),
|
2026-07-06 02:27:07 +08:00
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
2026-07-20 17:57:02 +08:00
|
|
|
except Exception:
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s could not mark task %s as failed",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
task.task_id,
|
|
|
|
|
)
|
|
|
|
|
self._notify_terminal(task, latest, exc)
|
2026-07-06 02:27:07 +08:00
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
2026-07-20 17:57:02 +08:00
|
|
|
status="failed",
|
|
|
|
|
task_id=task.task_id,
|
2026-07-06 02:27:07 +08:00
|
|
|
processed=1,
|
2026-07-20 17:57:02 +08:00
|
|
|
failed=1,
|
|
|
|
|
error_message=message,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
if self.retry_policy.should_retry(attempt, exc):
|
|
|
|
|
try:
|
|
|
|
|
queued = self.queue_service.update_task(
|
|
|
|
|
failed.task_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"progress": 0,
|
|
|
|
|
"error_message": None,
|
|
|
|
|
"metadata": {
|
|
|
|
|
**(failed.metadata or {}),
|
|
|
|
|
"retry": {
|
|
|
|
|
"scheduled": True,
|
|
|
|
|
"attempt": attempt,
|
|
|
|
|
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
|
|
|
|
|
"next_retry_at": self.retry_policy.next_retry_at(attempt),
|
|
|
|
|
"last_error": message,
|
|
|
|
|
"error_type": type(exc).__name__,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
self._notify_terminal(task, queued, exc)
|
|
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status="retry_scheduled",
|
|
|
|
|
task_id=queued.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
error_message=message,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s could not requeue task %s after failure",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
task.task_id,
|
|
|
|
|
)
|
|
|
|
|
self._notify_terminal(task, failed, exc)
|
|
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status="failed",
|
|
|
|
|
task_id=failed.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
failed=1,
|
|
|
|
|
error_message=failed.error_message or message,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
logger.exception(
|
|
|
|
|
"ImageStudio queue worker %s hit a secondary error after failing task %s",
|
|
|
|
|
self.worker_id,
|
|
|
|
|
task.task_id,
|
|
|
|
|
)
|
|
|
|
|
return WorkerRunResult(
|
|
|
|
|
worker_id=self.worker_id,
|
|
|
|
|
status="failed",
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
processed=1,
|
|
|
|
|
failed=1,
|
2026-07-06 02:27:07 +08:00
|
|
|
error_message=message,
|
|
|
|
|
)
|
2026-07-06 02:19:07 +08:00
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
def _notify_terminal(
|
|
|
|
|
self,
|
|
|
|
|
claimed_task: GenerationTask,
|
|
|
|
|
current_task: GenerationTask,
|
|
|
|
|
exc: Exception | None,
|
|
|
|
|
) -> None:
|
|
|
|
|
if self.terminal_callback is None:
|
|
|
|
|
return
|
|
|
|
|
try:
|
|
|
|
|
self.terminal_callback(claimed_task, current_task, exc)
|
|
|
|
|
except Exception:
|
|
|
|
|
# Queue state is authoritative. A coordinator callback may be
|
|
|
|
|
# replayed from persisted task state after restart, so it must not
|
|
|
|
|
# turn a successfully completed provider request into a failure.
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class QueueWorkerPool:
|
|
|
|
|
"""Bounded parallel workers sharing one persistent queue."""
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
queue_service: QueueService,
|
|
|
|
|
handler: WorkerHandler,
|
|
|
|
|
*,
|
|
|
|
|
worker_count: int = 5,
|
|
|
|
|
worker_id_prefix: str = "imagestudio-round",
|
|
|
|
|
interval_seconds: float = 0.25,
|
|
|
|
|
claim_batch_size: int = 25,
|
|
|
|
|
retry_policy: RetryPolicy | None = None,
|
|
|
|
|
task_filter: WorkerTaskFilter | None = None,
|
|
|
|
|
terminal_callback: WorkerTerminalCallback | None = None,
|
|
|
|
|
per_route_limit: int | None = None,
|
|
|
|
|
):
|
|
|
|
|
count = min(max(int(worker_count), 1), 16)
|
|
|
|
|
bounded_handler = _BoundedWorkerHandler(
|
|
|
|
|
handler,
|
|
|
|
|
global_limit=count,
|
|
|
|
|
per_route_limit=min(max(int(per_route_limit or count), 1), count),
|
|
|
|
|
)
|
|
|
|
|
self.workers = [
|
|
|
|
|
QueueWorker(
|
|
|
|
|
queue_service,
|
|
|
|
|
bounded_handler,
|
|
|
|
|
worker_id=f"{worker_id_prefix}-{index + 1}",
|
|
|
|
|
interval_seconds=interval_seconds,
|
|
|
|
|
claim_batch_size=claim_batch_size,
|
|
|
|
|
retry_policy=retry_policy,
|
|
|
|
|
task_filter=task_filter,
|
|
|
|
|
terminal_callback=terminal_callback,
|
|
|
|
|
)
|
|
|
|
|
for index in range(count)
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
def start(self, ctx: RequestContext) -> dict[str, Any]:
|
|
|
|
|
results = [worker.start(ctx) for worker in self.workers]
|
|
|
|
|
return {
|
|
|
|
|
"running": any(item["running"] for item in results),
|
|
|
|
|
"worker_count": len(self.workers),
|
|
|
|
|
"workers": results,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def stop(self, *, timeout_seconds: float = 5.0) -> dict[str, Any]:
|
|
|
|
|
results = [
|
|
|
|
|
worker.stop(timeout_seconds=timeout_seconds)
|
|
|
|
|
for worker in reversed(self.workers)
|
|
|
|
|
]
|
|
|
|
|
return {
|
|
|
|
|
"running": any(item["running"] for item in results),
|
|
|
|
|
"worker_count": len(self.workers),
|
|
|
|
|
"workers": list(reversed(results)),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def is_running(self) -> bool:
|
|
|
|
|
return any(worker.is_running() for worker in self.workers)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _BoundedWorkerHandler:
|
|
|
|
|
def __init__(self, handler: WorkerHandler, *, global_limit: int, per_route_limit: int):
|
|
|
|
|
self.handler = handler
|
|
|
|
|
self.global_semaphore = threading.BoundedSemaphore(global_limit)
|
|
|
|
|
self.per_route_limit = per_route_limit
|
|
|
|
|
self._route_guard = threading.Lock()
|
|
|
|
|
self._route_semaphores: dict[str, threading.BoundedSemaphore] = {}
|
|
|
|
|
|
|
|
|
|
def __call__(self, task: GenerationTask) -> WorkerTaskUpdate | dict[str, Any] | None:
|
|
|
|
|
route_key = _task_route_key(task)
|
|
|
|
|
with self._route_guard:
|
|
|
|
|
route_semaphore = self._route_semaphores.setdefault(
|
|
|
|
|
route_key,
|
|
|
|
|
threading.BoundedSemaphore(self.per_route_limit),
|
|
|
|
|
)
|
|
|
|
|
with self.global_semaphore, route_semaphore:
|
|
|
|
|
return self.handler(task)
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
|
|
|
|
|
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
|
|
|
|
|
if value is None:
|
|
|
|
|
return WorkerTaskUpdate()
|
|
|
|
|
if isinstance(value, WorkerTaskUpdate):
|
|
|
|
|
return value
|
|
|
|
|
return WorkerTaskUpdate(
|
|
|
|
|
status=str(value.get("status") or "succeeded"),
|
|
|
|
|
progress=_progress(value.get("progress", 100)),
|
|
|
|
|
result=dict(value.get("result") or {}),
|
|
|
|
|
metadata=dict(value.get("metadata") or {}),
|
|
|
|
|
error_message=_optional_text(value.get("error_message")),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _now() -> str:
|
|
|
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:27:07 +08:00
|
|
|
def _retry_deferred(task: GenerationTask) -> bool:
|
|
|
|
|
retry = task.metadata.get("retry")
|
|
|
|
|
if not isinstance(retry, dict):
|
|
|
|
|
return False
|
|
|
|
|
next_retry_at = _optional_text(retry.get("next_retry_at"))
|
|
|
|
|
if not next_retry_at:
|
|
|
|
|
return False
|
|
|
|
|
try:
|
|
|
|
|
parsed = datetime.fromisoformat(next_retry_at)
|
|
|
|
|
except ValueError:
|
|
|
|
|
return False
|
|
|
|
|
if parsed.tzinfo is None:
|
|
|
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
|
|
|
return parsed > datetime.now(timezone.utc)
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:19:07 +08:00
|
|
|
def _progress(value: Any) -> int:
|
|
|
|
|
return min(max(_int(value, 100), 0), 100)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _int(value: Any, default: int) -> int:
|
|
|
|
|
try:
|
|
|
|
|
return int(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return default
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
if value is None:
|
|
|
|
|
return None
|
|
|
|
|
text = str(value).strip()
|
|
|
|
|
return text or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _safe_error_message(exc: Exception) -> str:
|
2026-07-20 17:57:02 +08:00
|
|
|
if isinstance(exc, ApiError):
|
|
|
|
|
code = exc.error_code.value if hasattr(exc.error_code, "value") else str(exc.error_code)
|
|
|
|
|
parts = [str(exc.message or "").strip(), f"[{code}]"]
|
|
|
|
|
if isinstance(exc.details, dict) and exc.details:
|
|
|
|
|
# Keep a short, single-line hint without dumping full provider payloads.
|
|
|
|
|
hint_keys = ("reason", "finish_reason", "provider", "model")
|
|
|
|
|
hints = []
|
|
|
|
|
for key in hint_keys:
|
|
|
|
|
value = exc.details.get(key)
|
|
|
|
|
if value is None:
|
|
|
|
|
continue
|
|
|
|
|
text_value = " ".join(str(value).split())
|
|
|
|
|
if text_value:
|
|
|
|
|
hints.append(f"{key}={text_value[:80]}")
|
|
|
|
|
if hints:
|
|
|
|
|
parts.append("(" + ", ".join(hints) + ")")
|
|
|
|
|
text = " ".join(part for part in parts if part)
|
|
|
|
|
return text[:500] if text else f"ApiError [{code}]"
|
2026-07-06 02:19:07 +08:00
|
|
|
text = " ".join(str(exc or type(exc).__name__).split())
|
|
|
|
|
return text[:500] if text else type(exc).__name__
|
2026-07-15 23:01:07 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _task_route_key(task: GenerationTask) -> str:
|
|
|
|
|
resolution = task.metadata.get("model_route_resolution")
|
|
|
|
|
if not isinstance(resolution, dict):
|
|
|
|
|
resolution = task.request.get("resolved_model_route")
|
|
|
|
|
if not isinstance(resolution, dict):
|
|
|
|
|
resolution = {}
|
|
|
|
|
provider_id = _optional_text(resolution.get("provider_id")) or "unresolved"
|
|
|
|
|
model_id = _optional_text(resolution.get("model_id")) or _optional_text(task.model_route) or "default"
|
|
|
|
|
return f"{provider_id}:{model_id}"
|