S85 后端重试策略
这个提交存在于:
父节点
3ac3764340
当前提交
4327e74868
共有 2 个文件被更改,包括 135 次插入 和 14 次删除
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from backend.dto.common import ApiError
|
||||
|
|
@ -26,6 +26,25 @@ class WorkerTaskUpdate:
|
|||
error_message: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RetryPolicy:
|
||||
max_attempts: int = 1
|
||||
base_delay_seconds: float = 0.0
|
||||
backoff_multiplier: float = 2.0
|
||||
retryable_error_types: tuple[str, ...] = ()
|
||||
|
||||
def should_retry(self, attempt: int, exc: Exception) -> bool:
|
||||
if attempt >= max(int(self.max_attempts), 1):
|
||||
return False
|
||||
if not self.retryable_error_types:
|
||||
return True
|
||||
return type(exc).__name__ in self.retryable_error_types
|
||||
|
||||
def next_retry_at(self, attempt: int) -> str:
|
||||
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
|
||||
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerRunResult:
|
||||
worker_id: str
|
||||
|
|
@ -57,12 +76,14 @@ class QueueWorker:
|
|||
worker_id: str = "imagestudio-worker",
|
||||
interval_seconds: float = 1.0,
|
||||
claim_batch_size: int = 5,
|
||||
retry_policy: RetryPolicy | None = None,
|
||||
):
|
||||
self.queue_service = queue_service
|
||||
self.handler = handler
|
||||
self.worker_id = worker_id
|
||||
self.interval_seconds = max(float(interval_seconds), 0.01)
|
||||
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
|
||||
self.retry_policy = retry_policy or RetryPolicy()
|
||||
self._stop_event = threading.Event()
|
||||
self._lock = threading.Lock()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
|
@ -111,15 +132,7 @@ class QueueWorker:
|
|||
error_message=completed.error_message,
|
||||
)
|
||||
except Exception as exc:
|
||||
failed = self._mark_failed(task, exc, ctx)
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status=failed.status,
|
||||
task_id=failed.task_id,
|
||||
processed=1,
|
||||
failed=1 if failed.status == "failed" else 0,
|
||||
error_message=failed.error_message,
|
||||
)
|
||||
return self._handle_failure(task, exc, ctx)
|
||||
|
||||
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
|
||||
limit = min(max(int(max_tasks), 1), 100)
|
||||
|
|
@ -171,6 +184,8 @@ class QueueWorker:
|
|||
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
|
||||
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
|
||||
for candidate in queued:
|
||||
if _retry_deferred(candidate):
|
||||
continue
|
||||
try:
|
||||
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
|
||||
return self.queue_service.update_task(
|
||||
|
|
@ -194,12 +209,19 @@ class QueueWorker:
|
|||
continue
|
||||
return None
|
||||
|
||||
def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask:
|
||||
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
|
||||
latest = self.queue_service.get_job(task.task_id)
|
||||
if latest.status == "cancelled":
|
||||
return latest
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status="cancelled",
|
||||
task_id=latest.task_id,
|
||||
processed=1,
|
||||
error_message=latest.error_message,
|
||||
)
|
||||
message = _safe_error_message(exc)
|
||||
return self.queue_service.update_task(
|
||||
attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1))
|
||||
failed = self.queue_service.update_task(
|
||||
task.task_id,
|
||||
{
|
||||
"status": "failed",
|
||||
|
|
@ -216,6 +238,42 @@ class QueueWorker:
|
|||
},
|
||||
ctx,
|
||||
)
|
||||
if self.retry_policy.should_retry(attempt, exc):
|
||||
queued = self.queue_service.update_task(
|
||||
failed.task_id,
|
||||
{
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"error_message": None,
|
||||
"metadata": {
|
||||
**failed.metadata,
|
||||
"retry": {
|
||||
"scheduled": True,
|
||||
"attempt": attempt,
|
||||
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
|
||||
"next_retry_at": self.retry_policy.next_retry_at(attempt),
|
||||
"last_error": message,
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
},
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status="retry_scheduled",
|
||||
task_id=queued.task_id,
|
||||
processed=1,
|
||||
error_message=message,
|
||||
)
|
||||
return WorkerRunResult(
|
||||
worker_id=self.worker_id,
|
||||
status="failed",
|
||||
task_id=failed.task_id,
|
||||
processed=1,
|
||||
failed=1,
|
||||
error_message=failed.error_message,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
|
||||
|
|
@ -236,6 +294,22 @@ def _now() -> str:
|
|||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _retry_deferred(task: GenerationTask) -> bool:
|
||||
retry = task.metadata.get("retry")
|
||||
if not isinstance(retry, dict):
|
||||
return False
|
||||
next_retry_at = _optional_text(retry.get("next_retry_at"))
|
||||
if not next_retry_at:
|
||||
return False
|
||||
try:
|
||||
parsed = datetime.fromisoformat(next_retry_at)
|
||||
except ValueError:
|
||||
return False
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed > datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _progress(value: Any) -> int:
|
||||
return min(max(_int(value, 100), 0), 100)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import unittest
|
|||
from pathlib import Path
|
||||
|
||||
from backend.services.queue_service import QueueService
|
||||
from backend.services.worker_service import QueueWorker, WorkerTaskUpdate
|
||||
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate
|
||||
from core.audit_service import AuditService
|
||||
from core.event_bus import EventBus
|
||||
from core.identity import RequestContext
|
||||
|
|
@ -74,6 +74,53 @@ class QueueWorkerTests(unittest.TestCase):
|
|||
self.assertEqual(failed.metadata["worker"]["status"], "failed")
|
||||
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
|
||||
|
||||
def test_retry_policy_reschedules_retryable_failure(self):
|
||||
calls = []
|
||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||
|
||||
def fail_handler(claimed):
|
||||
calls.append(claimed.task_id)
|
||||
raise RuntimeError("temporary provider outage")
|
||||
|
||||
worker = QueueWorker(
|
||||
self.queue_service,
|
||||
fail_handler,
|
||||
worker_id="retry-worker",
|
||||
retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60),
|
||||
)
|
||||
|
||||
result = worker.run_once(self.ctx)
|
||||
deferred = worker.run_once(self.ctx)
|
||||
|
||||
self.assertEqual(result.status, "retry_scheduled")
|
||||
self.assertEqual(deferred.status, "idle")
|
||||
self.assertEqual(calls, [task.task_id])
|
||||
queued = self.queue_service.get_job(task.task_id)
|
||||
self.assertEqual(queued.status, "queued")
|
||||
self.assertIsNone(queued.error_message)
|
||||
self.assertEqual(queued.metadata["retry"]["attempt"], 1)
|
||||
self.assertEqual(queued.metadata["retry"]["max_attempts"], 2)
|
||||
self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"])
|
||||
|
||||
def test_retry_policy_exhausts_to_failed(self):
|
||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||
worker = QueueWorker(
|
||||
self.queue_service,
|
||||
lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")),
|
||||
worker_id="retry-exhaust-worker",
|
||||
retry_policy=RetryPolicy(max_attempts=2),
|
||||
)
|
||||
|
||||
first = worker.run_once(self.ctx)
|
||||
second = worker.run_once(self.ctx)
|
||||
|
||||
self.assertEqual(first.status, "retry_scheduled")
|
||||
self.assertEqual(second.status, "failed")
|
||||
failed = self.queue_service.get_job(task.task_id)
|
||||
self.assertEqual(failed.status, "failed")
|
||||
self.assertEqual(failed.metadata["worker_attempt"], 2)
|
||||
self.assertIn("still down", failed.error_message)
|
||||
|
||||
def test_background_worker_consumes_task_asynchronously(self):
|
||||
handled = threading.Event()
|
||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||
|
|
|
|||
正在加载…
添加表格
在新工单中引用