From 4327e748683c33ad0bcb368c365b60d2f63ece12 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Mon, 6 Jul 2026 02:27:07 +0800 Subject: [PATCH] =?UTF-8?q?S85=20=E5=90=8E=E7=AB=AF=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/worker_service.py | 100 +++++++++++++++++++++++++---- tests/test_worker_service.py | 49 +++++++++++++- 2 files changed, 135 insertions(+), 14 deletions(-) diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 15888f8..ad77943 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -5,7 +5,7 @@ from __future__ import annotations import threading from collections.abc import Callable from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any from backend.dto.common import ApiError @@ -26,6 +26,25 @@ class WorkerTaskUpdate: error_message: str | None = None +@dataclass(frozen=True) +class RetryPolicy: + max_attempts: int = 1 + base_delay_seconds: float = 0.0 + backoff_multiplier: float = 2.0 + retryable_error_types: tuple[str, ...] = () + + def should_retry(self, attempt: int, exc: Exception) -> bool: + if attempt >= max(int(self.max_attempts), 1): + return False + if not self.retryable_error_types: + return True + return type(exc).__name__ in self.retryable_error_types + + def next_retry_at(self, attempt: int) -> str: + delay = max(float(self.base_delay_seconds), 0.0) * (max(float(self.backoff_multiplier), 1.0) ** max(attempt - 1, 0)) + return (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat() + + @dataclass(frozen=True) class WorkerRunResult: worker_id: str @@ -57,12 +76,14 @@ class QueueWorker: worker_id: str = "imagestudio-worker", interval_seconds: float = 1.0, claim_batch_size: int = 5, + retry_policy: RetryPolicy | None = None, ): self.queue_service = queue_service self.handler = handler self.worker_id = worker_id self.interval_seconds = max(float(interval_seconds), 0.01) self.claim_batch_size = min(max(int(claim_batch_size), 1), 50) + self.retry_policy = retry_policy or RetryPolicy() self._stop_event = threading.Event() self._lock = threading.Lock() self._thread: threading.Thread | None = None @@ -111,15 +132,7 @@ class QueueWorker: error_message=completed.error_message, ) except Exception as exc: - failed = self._mark_failed(task, exc, ctx) - return WorkerRunResult( - worker_id=self.worker_id, - status=failed.status, - task_id=failed.task_id, - processed=1, - failed=1 if failed.status == "failed" else 0, - error_message=failed.error_message, - ) + return self._handle_failure(task, exc, ctx) def run_batch(self, ctx: RequestContext, *, max_tasks: int = 10) -> dict[str, Any]: limit = min(max(int(max_tasks), 1), 100) @@ -171,6 +184,8 @@ class QueueWorker: def _claim_next(self, ctx: RequestContext) -> GenerationTask | None: queued = self.queue_service.list_jobs(status="queued", page=1, page_size=self.claim_batch_size).items for candidate in queued: + if _retry_deferred(candidate): + continue try: attempt = _int(candidate.metadata.get("worker_attempt"), 0) + 1 return self.queue_service.update_task( @@ -194,12 +209,19 @@ class QueueWorker: continue return None - def _mark_failed(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> GenerationTask: + def _handle_failure(self, task: GenerationTask, exc: Exception, ctx: RequestContext) -> WorkerRunResult: latest = self.queue_service.get_job(task.task_id) if latest.status == "cancelled": - return latest + return WorkerRunResult( + worker_id=self.worker_id, + status="cancelled", + task_id=latest.task_id, + processed=1, + error_message=latest.error_message, + ) message = _safe_error_message(exc) - return self.queue_service.update_task( + attempt = _int(latest.metadata.get("worker_attempt"), _int(task.metadata.get("worker_attempt"), 1)) + failed = self.queue_service.update_task( task.task_id, { "status": "failed", @@ -216,6 +238,42 @@ class QueueWorker: }, ctx, ) + if self.retry_policy.should_retry(attempt, exc): + queued = self.queue_service.update_task( + failed.task_id, + { + "status": "queued", + "progress": 0, + "error_message": None, + "metadata": { + **failed.metadata, + "retry": { + "scheduled": True, + "attempt": attempt, + "max_attempts": max(int(self.retry_policy.max_attempts), 1), + "next_retry_at": self.retry_policy.next_retry_at(attempt), + "last_error": message, + "error_type": type(exc).__name__, + }, + }, + }, + ctx, + ) + return WorkerRunResult( + worker_id=self.worker_id, + status="retry_scheduled", + task_id=queued.task_id, + processed=1, + error_message=message, + ) + return WorkerRunResult( + worker_id=self.worker_id, + status="failed", + task_id=failed.task_id, + processed=1, + failed=1, + error_message=failed.error_message, + ) def _normalize_update(value: WorkerTaskUpdate | dict[str, Any] | None) -> WorkerTaskUpdate: @@ -236,6 +294,22 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat() +def _retry_deferred(task: GenerationTask) -> bool: + retry = task.metadata.get("retry") + if not isinstance(retry, dict): + return False + next_retry_at = _optional_text(retry.get("next_retry_at")) + if not next_retry_at: + return False + try: + parsed = datetime.fromisoformat(next_retry_at) + except ValueError: + return False + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed > datetime.now(timezone.utc) + + def _progress(value: Any) -> int: return min(max(_int(value, 100), 0), 100) diff --git a/tests/test_worker_service.py b/tests/test_worker_service.py index cab1c66..d171a9c 100644 --- a/tests/test_worker_service.py +++ b/tests/test_worker_service.py @@ -6,7 +6,7 @@ import unittest from pathlib import Path from backend.services.queue_service import QueueService -from backend.services.worker_service import QueueWorker, WorkerTaskUpdate +from backend.services.worker_service import QueueWorker, RetryPolicy, WorkerTaskUpdate from core.audit_service import AuditService from core.event_bus import EventBus from core.identity import RequestContext @@ -74,6 +74,53 @@ class QueueWorkerTests(unittest.TestCase): self.assertEqual(failed.metadata["worker"]["status"], "failed") self.assertEqual(failed.metadata["worker"]["error_type"], "RuntimeError") + def test_retry_policy_reschedules_retryable_failure(self): + calls = [] + task = self.queue_service.create_task({"status": "queued"}, self.ctx) + + def fail_handler(claimed): + calls.append(claimed.task_id) + raise RuntimeError("temporary provider outage") + + worker = QueueWorker( + self.queue_service, + fail_handler, + worker_id="retry-worker", + retry_policy=RetryPolicy(max_attempts=2, base_delay_seconds=60), + ) + + result = worker.run_once(self.ctx) + deferred = worker.run_once(self.ctx) + + self.assertEqual(result.status, "retry_scheduled") + self.assertEqual(deferred.status, "idle") + self.assertEqual(calls, [task.task_id]) + queued = self.queue_service.get_job(task.task_id) + self.assertEqual(queued.status, "queued") + self.assertIsNone(queued.error_message) + self.assertEqual(queued.metadata["retry"]["attempt"], 1) + self.assertEqual(queued.metadata["retry"]["max_attempts"], 2) + self.assertIn("temporary provider outage", queued.metadata["retry"]["last_error"]) + + def test_retry_policy_exhausts_to_failed(self): + task = self.queue_service.create_task({"status": "queued"}, self.ctx) + worker = QueueWorker( + self.queue_service, + lambda claimed: (_ for _ in ()).throw(RuntimeError("still down")), + worker_id="retry-exhaust-worker", + retry_policy=RetryPolicy(max_attempts=2), + ) + + first = worker.run_once(self.ctx) + second = worker.run_once(self.ctx) + + self.assertEqual(first.status, "retry_scheduled") + self.assertEqual(second.status, "failed") + failed = self.queue_service.get_job(task.task_id) + self.assertEqual(failed.status, "failed") + self.assertEqual(failed.metadata["worker_attempt"], 2) + self.assertIn("still down", failed.error_message) + def test_background_worker_consumes_task_asynchronously(self): handled = threading.Event() task = self.queue_service.create_task({"status": "queued"}, self.ctx)