S85 后端重试策略

这个提交存在于:
YokiiroBW 2026-07-06 02:27:07 +08:00
父节点 3ac3764340
当前提交 4327e74868
共有 2 个文件被更改,包括 135 次插入 和 14 次删除

查看文件

@ -5,7 +5,7 @@ from __future__ import annotations
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Any
from backend.dto.common import ApiError
@ -26,6 +26,25 @@ class WorkerTaskUpdate:
error_message: str | None = None
@dataclass(frozen=True)
class RetryPolicy:
max_attempts: int = 1
base_delay_seconds: float = 0.0
backoff_multiplier: float = 2.0
retryable_error_types: tuple[str, ...] = ()
def should_retry(self, attempt: int, exc: Exception) -> bool:
if attempt >= max(int(self.max_attempts), 1):
return False
if not self.retryable_error_types:
return True
return type(exc).__name__ in self.retryable_error_types
def next_retry_at(self, attempt: int) -> str:
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
@dataclass(frozen=True)
class WorkerRunResult:
worker_id: str
@ -57,12 +76,14 @@ class QueueWorker:
worker_id: str = "imagestudio-worker",
interval_seconds: float = 1.0,
claim_batch_size: int = 5,
retry_policy: RetryPolicy | None = None,
):
self.queue_service = queue_service
self.handler = handler
self.worker_id = worker_id
self.interval_seconds = max(float(interval_seconds), 0.01)
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
self.retry_policy = retry_policy or RetryPolicy()
self._stop_event = threading.Event()
self._lock = threading.Lock()
self._thread: threading.Thread | None = None
@ -111,15 +132,7 @@ class QueueWorker:
error_message=completed.error_message,
)
except Exception as exc:
failed = self._mark_failed(task, exc, ctx)
return WorkerRunResult(
worker_id=self.worker_id,
status=failed.status,
task_id=failed.task_id,
processed=1,
failed=1 if failed.status == "failed" else 0,
error_message=failed.error_message,
)
return self._handle_failure(task, exc, ctx)
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
limit = min(max(int(max_tasks), 1), 100)
@ -171,6 +184,8 @@ class QueueWorker:
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
for candidate in queued:
if _retry_deferred(candidate):
continue
try:
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
return self.queue_service.update_task(
@ -194,12 +209,19 @@ class QueueWorker:
continue
return None
def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask:
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled":
return latest
return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
message = _safe_error_message(exc)
return self.queue_service.update_task(
attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1))
failed = self.queue_service.update_task(
task.task_id,
{
"status": "failed",
@ -216,6 +238,42 @@ class QueueWorker:
},
ctx,
)
if self.retry_policy.should_retry(attempt, exc):
queued = self.queue_service.update_task(
failed.task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**failed.metadata,
"retry": {
"scheduled": True,
"attempt": attempt,
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
"next_retry_at": self.retry_policy.next_retry_at(attempt),
"last_error": message,
"error_type": type(exc).__name__,
},
},
},
ctx,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="retry_scheduled",
task_id=queued.task_id,
processed=1,
error_message=message,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=failed.task_id,
processed=1,
failed=1,
error_message=failed.error_message,
)
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
@ -236,6 +294,22 @@ def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _retry_deferred(task: GenerationTask) -> bool:
retry = task.metadata.get("retry")
if not isinstance(retry, dict):
return False
next_retry_at = _optional_text(retry.get("next_retry_at"))
if not next_retry_at:
return False
try:
parsed = datetime.fromisoformat(next_retry_at)
except ValueError:
return False
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed > datetime.now(timezone.utc)
def _progress(value: Any) -> int:
return min(max(_int(value, 100), 0), 100)

查看文件

@ -6,7 +6,7 @@ import unittest
from pathlib import Path
from backend.services.queue_service import QueueService
from backend.services.worker_service import QueueWorker, WorkerTaskUpdate
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate
from core.audit_service import AuditService
from core.event_bus import EventBus
from core.identity import RequestContext
@ -74,6 +74,53 @@ class QueueWorkerTests(unittest.TestCase):
self.assertEqual(failed.metadata["worker"]["status"], "failed")
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
def test_retry_policy_reschedules_retryable_failure(self):
calls = []
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
def fail_handler(claimed):
calls.append(claimed.task_id)
raise RuntimeError("temporary provider outage")
worker = QueueWorker(
self.queue_service,
fail_handler,
worker_id="retry-worker",
retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60),
)
result = worker.run_once(self.ctx)
deferred = worker.run_once(self.ctx)
self.assertEqual(result.status, "retry_scheduled")
self.assertEqual(deferred.status, "idle")
self.assertEqual(calls, [task.task_id])
queued = self.queue_service.get_job(task.task_id)
self.assertEqual(queued.status, "queued")
self.assertIsNone(queued.error_message)
self.assertEqual(queued.metadata["retry"]["attempt"], 1)
self.assertEqual(queued.metadata["retry"]["max_attempts"], 2)
self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"])
def test_retry_policy_exhausts_to_failed(self):
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
worker = QueueWorker(
self.queue_service,
lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")),
worker_id="retry-exhaust-worker",
retry_policy=RetryPolicy(max_attempts=2),
)
first = worker.run_once(self.ctx)
second = worker.run_once(self.ctx)
self.assertEqual(first.status, "retry_scheduled")
self.assertEqual(second.status, "failed")
failed = self.queue_service.get_job(task.task_id)
self.assertEqual(failed.status, "failed")
self.assertEqual(failed.metadata["worker_attempt"], 2)
self.assertIn("still down", failed.error_message)
def test_background_worker_consumes_task_asynchronously(self):
handled = threading.Event()
task = self.queue_service.create_task({"status": "queued"}, self.ctx)