S85 后端重试策略

这个提交存在于:
YokiiroBW 2026-07-06 02:27:07 +08:00
父节点 3ac3764340
当前提交 4327e74868
共有 2 个文件被更改,包括 135 次插入 和 14 次删除

查看文件

@ -5,7 +5,7 @@ from __future__ import annotations
import threading import threading
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from backend.dto.common import ApiError from backend.dto.common import ApiError
@ -26,6 +26,25 @@ class WorkerTaskUpdate:
error_message: str | None = None error_message: str | None = None
@dataclass(frozen=True)
class RetryPolicy:
max_attempts: int = 1
base_delay_seconds: float = 0.0
backoff_multiplier: float = 2.0
retryable_error_types: tuple[str, ...] = ()
def should_retry(self, attempt: int, exc: Exception) -> bool:
if attempt >= max(int(self.max_attempts), 1):
return False
if not self.retryable_error_types:
return True
return type(exc).__name__ in self.retryable_error_types
def next_retry_at(self, attempt: int) -> str:
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
@dataclass(frozen=True) @dataclass(frozen=True)
class WorkerRunResult: class WorkerRunResult:
worker_id: str worker_id: str
@ -57,12 +76,14 @@ class QueueWorker:
worker_id: str = "imagestudio-worker", worker_id: str = "imagestudio-worker",
interval_seconds: float = 1.0, interval_seconds: float = 1.0,
claim_batch_size: int = 5, claim_batch_size: int = 5,
retry_policy: RetryPolicy | None = None,
): ):
self.queue_service = queue_service self.queue_service = queue_service
self.handler = handler self.handler = handler
self.worker_id = worker_id self.worker_id = worker_id
self.interval_seconds = max(float(interval_seconds), 0.01) self.interval_seconds = max(float(interval_seconds), 0.01)
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
self.retry_policy = retry_policy or RetryPolicy()
self._stop_event = threading.Event() self._stop_event = threading.Event()
self._lock = threading.Lock() self._lock = threading.Lock()
self._thread: threading.Thread | None = None self._thread: threading.Thread | None = None
@ -111,15 +132,7 @@ class QueueWorker:
error_message=completed.error_message, error_message=completed.error_message,
) )
except Exception as exc: except Exception as exc:
failed = self._mark_failed(task, exc, ctx) return self._handle_failure(task, exc, ctx)
return WorkerRunResult(
worker_id=self.worker_id,
status=failed.status,
task_id=failed.task_id,
processed=1,
failed=1 if failed.status == "failed" else 0,
error_message=failed.error_message,
)
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
limit = min(max(int(max_tasks), 1), 100) limit = min(max(int(max_tasks), 1), 100)
@ -171,6 +184,8 @@ class QueueWorker:
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
for candidate in queued: for candidate in queued:
if _retry_deferred(candidate):
continue
try: try:
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1 attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
return self.queue_service.update_task( return self.queue_service.update_task(
@ -194,12 +209,19 @@ class QueueWorker:
continue continue
return None return None
def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask: def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
latest = self.queue_service.get_job(task.task_id) latest = self.queue_service.get_job(task.task_id)
if latest.status == "cancelled": if latest.status == "cancelled":
return latest return WorkerRunResult(
worker_id=self.worker_id,
status="cancelled",
task_id=latest.task_id,
processed=1,
error_message=latest.error_message,
)
message = _safe_error_message(exc) message = _safe_error_message(exc)
return self.queue_service.update_task( attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1))
failed = self.queue_service.update_task(
task.task_id, task.task_id,
{ {
"status": "failed", "status": "failed",
@ -216,6 +238,42 @@ class QueueWorker:
}, },
ctx, ctx,
) )
if self.retry_policy.should_retry(attempt, exc):
queued = self.queue_service.update_task(
failed.task_id,
{
"status": "queued",
"progress": 0,
"error_message": None,
"metadata": {
**failed.metadata,
"retry": {
"scheduled": True,
"attempt": attempt,
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
"next_retry_at": self.retry_policy.next_retry_at(attempt),
"last_error": message,
"error_type": type(exc).__name__,
},
},
},
ctx,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="retry_scheduled",
task_id=queued.task_id,
processed=1,
error_message=message,
)
return WorkerRunResult(
worker_id=self.worker_id,
status="failed",
task_id=failed.task_id,
processed=1,
failed=1,
error_message=failed.error_message,
)
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
@ -236,6 +294,22 @@ def _now() -> str:
return datetime.now(timezone.utc).isoformat() return datetime.now(timezone.utc).isoformat()
def _retry_deferred(task: GenerationTask) -> bool:
retry = task.metadata.get("retry")
if not isinstance(retry, dict):
return False
next_retry_at = _optional_text(retry.get("next_retry_at"))
if not next_retry_at:
return False
try:
parsed = datetime.fromisoformat(next_retry_at)
except ValueError:
return False
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed > datetime.now(timezone.utc)
def _progress(value: Any) -> int: def _progress(value: Any) -> int:
return min(max(_int(value, 100), 0), 100) return min(max(_int(value, 100), 0), 100)

查看文件

@ -6,7 +6,7 @@ import unittest
from pathlib import Path from pathlib import Path
from backend.services.queue_service import QueueService from backend.services.queue_service import QueueService
from backend.services.worker_service import QueueWorker, WorkerTaskUpdate from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate
from core.audit_service import AuditService from core.audit_service import AuditService
from core.event_bus import EventBus from core.event_bus import EventBus
from core.identity import RequestContext from core.identity import RequestContext
@ -74,6 +74,53 @@ class QueueWorkerTests(unittest.TestCase):
self.assertEqual(failed.metadata["worker"]["status"], "failed") self.assertEqual(failed.metadata["worker"]["status"], "failed")
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError") self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
def test_retry_policy_reschedules_retryable_failure(self):
calls = []
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
def fail_handler(claimed):
calls.append(claimed.task_id)
raise RuntimeError("temporary provider outage")
worker = QueueWorker(
self.queue_service,
fail_handler,
worker_id="retry-worker",
retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60),
)
result = worker.run_once(self.ctx)
deferred = worker.run_once(self.ctx)
self.assertEqual(result.status, "retry_scheduled")
self.assertEqual(deferred.status, "idle")
self.assertEqual(calls, [task.task_id])
queued = self.queue_service.get_job(task.task_id)
self.assertEqual(queued.status, "queued")
self.assertIsNone(queued.error_message)
self.assertEqual(queued.metadata["retry"]["attempt"], 1)
self.assertEqual(queued.metadata["retry"]["max_attempts"], 2)
self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"])
def test_retry_policy_exhausts_to_failed(self):
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
worker = QueueWorker(
self.queue_service,
lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")),
worker_id="retry-exhaust-worker",
retry_policy=RetryPolicy(max_attempts=2),
)
first = worker.run_once(self.ctx)
second = worker.run_once(self.ctx)
self.assertEqual(first.status, "retry_scheduled")
self.assertEqual(second.status, "failed")
failed = self.queue_service.get_job(task.task_id)
self.assertEqual(failed.status, "failed")
self.assertEqual(failed.metadata["worker_attempt"], 2)
self.assertIn("still down", failed.error_message)
def test_background_worker_consumes_task_asynchronously(self): def test_background_worker_consumes_task_asynchronously(self):
handled = threading.Event() handled = threading.Event()
task = self.queue_service.create_task({"status": "queued"}, self.ctx) task = self.queue_service.create_task({"status": "queued"}, self.ctx)