S85 后端重试策略
这个提交存在于:
父节点
3ac3764340
当前提交
4327e74868
共有 2 个文件被更改,包括 135 次插入 和 14 次删除
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
||||||
import threading
|
import threading
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from backend.dto.common import ApiError
|
from backend.dto.common import ApiError
|
||||||
|
|
@ -26,6 +26,25 @@ class WorkerTaskUpdate:
|
||||||
error_message: str | None = None
|
error_message: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RetryPolicy:
|
||||||
|
max_attempts: int = 1
|
||||||
|
base_delay_seconds: float = 0.0
|
||||||
|
backoff_multiplier: float = 2.0
|
||||||
|
retryable_error_types: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
def should_retry(self, attempt: int, exc: Exception) -> bool:
|
||||||
|
if attempt >= max(int(self.max_attempts), 1):
|
||||||
|
return False
|
||||||
|
if not self.retryable_error_types:
|
||||||
|
return True
|
||||||
|
return type(exc).__name__ in self.retryable_error_types
|
||||||
|
|
||||||
|
def next_retry_at(self, attempt: int) -> str:
|
||||||
|
delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0))
|
||||||
|
return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat()
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class WorkerRunResult:
|
class WorkerRunResult:
|
||||||
worker_id: str
|
worker_id: str
|
||||||
|
|
@ -57,12 +76,14 @@ class QueueWorker:
|
||||||
worker_id: str = "imagestudio-worker",
|
worker_id: str = "imagestudio-worker",
|
||||||
interval_seconds: float = 1.0,
|
interval_seconds: float = 1.0,
|
||||||
claim_batch_size: int = 5,
|
claim_batch_size: int = 5,
|
||||||
|
retry_policy: RetryPolicy | None = None,
|
||||||
):
|
):
|
||||||
self.queue_service = queue_service
|
self.queue_service = queue_service
|
||||||
self.handler = handler
|
self.handler = handler
|
||||||
self.worker_id = worker_id
|
self.worker_id = worker_id
|
||||||
self.interval_seconds = max(float(interval_seconds), 0.01)
|
self.interval_seconds = max(float(interval_seconds), 0.01)
|
||||||
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
|
self.claim_batch_size = min(max(int(claim_batch_size), 1), 50)
|
||||||
|
self.retry_policy = retry_policy or RetryPolicy()
|
||||||
self._stop_event = threading.Event()
|
self._stop_event = threading.Event()
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
|
|
@ -111,15 +132,7 @@ class QueueWorker:
|
||||||
error_message=completed.error_message,
|
error_message=completed.error_message,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
failed = self._mark_failed(task, exc, ctx)
|
return self._handle_failure(task, exc, ctx)
|
||||||
return WorkerRunResult(
|
|
||||||
worker_id=self.worker_id,
|
|
||||||
status=failed.status,
|
|
||||||
task_id=failed.task_id,
|
|
||||||
processed=1,
|
|
||||||
failed=1 if failed.status == "failed" else 0,
|
|
||||||
error_message=failed.error_message,
|
|
||||||
)
|
|
||||||
|
|
||||||
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
|
def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]:
|
||||||
limit = min(max(int(max_tasks), 1), 100)
|
limit = min(max(int(max_tasks), 1), 100)
|
||||||
|
|
@ -171,6 +184,8 @@ class QueueWorker:
|
||||||
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
|
def _claim_next(self, ctx: RequestContext) -> GenerationTask | None:
|
||||||
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
|
queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items
|
||||||
for candidate in queued:
|
for candidate in queued:
|
||||||
|
if _retry_deferred(candidate):
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
|
attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1
|
||||||
return self.queue_service.update_task(
|
return self.queue_service.update_task(
|
||||||
|
|
@ -194,12 +209,19 @@ class QueueWorker:
|
||||||
continue
|
continue
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask:
|
def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult:
|
||||||
latest = self.queue_service.get_job(task.task_id)
|
latest = self.queue_service.get_job(task.task_id)
|
||||||
if latest.status == "cancelled":
|
if latest.status == "cancelled":
|
||||||
return latest
|
return WorkerRunResult(
|
||||||
|
worker_id=self.worker_id,
|
||||||
|
status="cancelled",
|
||||||
|
task_id=latest.task_id,
|
||||||
|
processed=1,
|
||||||
|
error_message=latest.error_message,
|
||||||
|
)
|
||||||
message = _safe_error_message(exc)
|
message = _safe_error_message(exc)
|
||||||
return self.queue_service.update_task(
|
attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1))
|
||||||
|
failed = self.queue_service.update_task(
|
||||||
task.task_id,
|
task.task_id,
|
||||||
{
|
{
|
||||||
"status": "failed",
|
"status": "failed",
|
||||||
|
|
@ -216,6 +238,42 @@ class QueueWorker:
|
||||||
},
|
},
|
||||||
ctx,
|
ctx,
|
||||||
)
|
)
|
||||||
|
if self.retry_policy.should_retry(attempt, exc):
|
||||||
|
queued = self.queue_service.update_task(
|
||||||
|
failed.task_id,
|
||||||
|
{
|
||||||
|
"status": "queued",
|
||||||
|
"progress": 0,
|
||||||
|
"error_message": None,
|
||||||
|
"metadata": {
|
||||||
|
**failed.metadata,
|
||||||
|
"retry": {
|
||||||
|
"scheduled": True,
|
||||||
|
"attempt": attempt,
|
||||||
|
"max_attempts": max(int(self.retry_policy.max_attempts), 1),
|
||||||
|
"next_retry_at": self.retry_policy.next_retry_at(attempt),
|
||||||
|
"last_error": message,
|
||||||
|
"error_type": type(exc).__name__,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ctx,
|
||||||
|
)
|
||||||
|
return WorkerRunResult(
|
||||||
|
worker_id=self.worker_id,
|
||||||
|
status="retry_scheduled",
|
||||||
|
task_id=queued.task_id,
|
||||||
|
processed=1,
|
||||||
|
error_message=message,
|
||||||
|
)
|
||||||
|
return WorkerRunResult(
|
||||||
|
worker_id=self.worker_id,
|
||||||
|
status="failed",
|
||||||
|
task_id=failed.task_id,
|
||||||
|
processed=1,
|
||||||
|
failed=1,
|
||||||
|
error_message=failed.error_message,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
|
def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate:
|
||||||
|
|
@ -236,6 +294,22 @@ def _now() -> str:
|
||||||
return datetime.now(timezone.utc).isoformat()
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_deferred(task: GenerationTask) -> bool:
|
||||||
|
retry = task.metadata.get("retry")
|
||||||
|
if not isinstance(retry, dict):
|
||||||
|
return False
|
||||||
|
next_retry_at = _optional_text(retry.get("next_retry_at"))
|
||||||
|
if not next_retry_at:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(next_retry_at)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||||
|
return parsed > datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def _progress(value: Any) -> int:
|
def _progress(value: Any) -> int:
|
||||||
return min(max(_int(value, 100), 0), 100)
|
return min(max(_int(value, 100), 0), 100)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from backend.services.queue_service import QueueService
|
from backend.services.queue_service import QueueService
|
||||||
from backend.services.worker_service import QueueWorker, WorkerTaskUpdate
|
from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate
|
||||||
from core.audit_service import AuditService
|
from core.audit_service import AuditService
|
||||||
from core.event_bus import EventBus
|
from core.event_bus import EventBus
|
||||||
from core.identity import RequestContext
|
from core.identity import RequestContext
|
||||||
|
|
@ -74,6 +74,53 @@ class QueueWorkerTests(unittest.TestCase):
|
||||||
self.assertEqual(failed.metadata["worker"]["status"], "failed")
|
self.assertEqual(failed.metadata["worker"]["status"], "failed")
|
||||||
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
|
self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError")
|
||||||
|
|
||||||
|
def test_retry_policy_reschedules_retryable_failure(self):
|
||||||
|
calls = []
|
||||||
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||||
|
|
||||||
|
def fail_handler(claimed):
|
||||||
|
calls.append(claimed.task_id)
|
||||||
|
raise RuntimeError("temporary provider outage")
|
||||||
|
|
||||||
|
worker = QueueWorker(
|
||||||
|
self.queue_service,
|
||||||
|
fail_handler,
|
||||||
|
worker_id="retry-worker",
|
||||||
|
retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = worker.run_once(self.ctx)
|
||||||
|
deferred = worker.run_once(self.ctx)
|
||||||
|
|
||||||
|
self.assertEqual(result.status, "retry_scheduled")
|
||||||
|
self.assertEqual(deferred.status, "idle")
|
||||||
|
self.assertEqual(calls, [task.task_id])
|
||||||
|
queued = self.queue_service.get_job(task.task_id)
|
||||||
|
self.assertEqual(queued.status, "queued")
|
||||||
|
self.assertIsNone(queued.error_message)
|
||||||
|
self.assertEqual(queued.metadata["retry"]["attempt"], 1)
|
||||||
|
self.assertEqual(queued.metadata["retry"]["max_attempts"], 2)
|
||||||
|
self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"])
|
||||||
|
|
||||||
|
def test_retry_policy_exhausts_to_failed(self):
|
||||||
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||||
|
worker = QueueWorker(
|
||||||
|
self.queue_service,
|
||||||
|
lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")),
|
||||||
|
worker_id="retry-exhaust-worker",
|
||||||
|
retry_policy=RetryPolicy(max_attempts=2),
|
||||||
|
)
|
||||||
|
|
||||||
|
first = worker.run_once(self.ctx)
|
||||||
|
second = worker.run_once(self.ctx)
|
||||||
|
|
||||||
|
self.assertEqual(first.status, "retry_scheduled")
|
||||||
|
self.assertEqual(second.status, "failed")
|
||||||
|
failed = self.queue_service.get_job(task.task_id)
|
||||||
|
self.assertEqual(failed.status, "failed")
|
||||||
|
self.assertEqual(failed.metadata["worker_attempt"], 2)
|
||||||
|
self.assertIn("still down", failed.error_message)
|
||||||
|
|
||||||
def test_background_worker_consumes_task_asynchronously(self):
|
def test_background_worker_consumes_task_asynchronously(self):
|
||||||
handled = threading.Event()
|
handled = threading.Event()
|
||||||
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
task = self.queue_service.create_task({"status": "queued"}, self.ctx)
|
||||||
|
|
|
||||||
正在加载…
添加表格
在新工单中引用