diff --git a/backend/route_registry.py b/backend/route_registry.py index 76c9fbb..99c9d1a 100644 --- a/backend/route_registry.py +++ b/backend/route_registry.py @@ -72,6 +72,7 @@ ROUTES: tuple[RouteSpec, ...] = ( RouteSpec("POST", "jobs/dead-letter/retry", "backend.routes.job_routes.retry_dead_letter_jobs", "write"), RouteSpec("POST", "jobs/dead-letter/move-failed", "backend.routes.job_routes.move_failed_jobs_to_dead_letter", "write"), RouteSpec("GET", "costs/usage", "backend.routes.cost_routes.get_cost_usage", "read"), + RouteSpec("POST", "costs/guard", "backend.routes.cost_routes.check_cost_guard", "read"), RouteSpec("GET", "settings/schema", "backend.routes.settings_routes.get_settings_schema", "read"), RouteSpec("GET", "settings/current", "backend.routes.settings_routes.get_current_settings", "read"), RouteSpec("POST", "settings/validate", "backend.routes.settings_routes.validate_settings", "config"), diff --git a/backend/routes/cost_routes.py b/backend/routes/cost_routes.py index 9fc9589..542e5d3 100644 --- a/backend/routes/cost_routes.py +++ b/backend/routes/cost_routes.py @@ -6,6 +6,7 @@ import json from typing import Any from backend.routes.common import handle_request +from backend.services.cost_guard_service import CostGuardService from backend.services.cost_service import CostService from core.identity import RequestContext from core.security import require_authenticated @@ -34,6 +35,33 @@ def get_cost_usage( return handle_request(ctx, action) +def check_cost_guard( + ctx: RequestContext, + payload: dict[str, Any] | None = None, + service: CostGuardService | None = None, +) -> dict: + params = payload or {} + guard_service = service or CostGuardService(CostService(Database(load_database_settings()))) + + def action() -> dict[str, Any]: + require_authenticated(ctx) + return guard_service.check_generation_budget( + provider_id=_optional_str(params.get("provider_id")), + provider_model=_optional_str(params.get("provider_model")), + model_route=_optional_str(params.get("model_route")), + planned_task_count=_int_or_default(params.get("planned_task_count"), 1), + planned_image_count=_int_or_default(params.get("planned_image_count"), 1), + planned_total_tokens=_int_or_default(params.get("planned_total_tokens"), 0), + max_task_count=_optional_int(params.get("max_task_count")), + max_image_count=_optional_int(params.get("max_image_count")), + max_estimated_cost=_optional_float(params.get("max_estimated_cost")), + days=_optional_int(params.get("days")), + unit_prices=_unit_prices(params.get("unit_prices")), + ) + + return handle_request(ctx, action) + + def _unit_prices(value: Any) -> dict[str, Any]: if value is None: return {} @@ -65,3 +93,19 @@ def _optional_int(value: Any) -> int | None: return int(value) except (TypeError, ValueError): return None + + +def _optional_float(value: Any) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _int_or_default(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default diff --git a/backend/services/cost_guard_service.py b/backend/services/cost_guard_service.py new file mode 100644 index 0000000..6373b82 --- /dev/null +++ b/backend/services/cost_guard_service.py @@ -0,0 +1,96 @@ +"""Budget guard checks for paid model calls.""" + +from __future__ import annotations + +from typing import Any + +from backend.services.cost_service import CostService, PriceSpec + + +class CostGuardService: + def __init__(self, cost_service: CostService): + self.cost_service = cost_service + + def check_generation_budget( + self, + *, + provider_id: str | None, + provider_model: str | None, + model_route: str | None = None, + planned_task_count: int = 1, + planned_image_count: int = 1, + planned_total_tokens: int = 0, + max_task_count: int | None = None, + max_image_count: int | None = None, + max_estimated_cost: float | None = None, + days: int | None = None, + unit_prices: dict[str, PriceSpec] | None = None, + ) -> dict[str, Any]: + provider = str(provider_id or "unknown") + model = str(provider_model or "unknown") + current = self.cost_service.usage_summary(provider_id=provider, days=days, unit_prices=unit_prices) + planned_cost = self.cost_service.estimate_cost( + provider_id=provider, + provider_model=model, + model_route=str(model_route or "planned"), + task_count=planned_task_count, + image_count=planned_image_count, + total_tokens=planned_total_tokens, + unit_prices=unit_prices, + ) + projected = { + "task_count": current["totals"]["task_count"] + max(int(planned_task_count), 0), + "image_count": current["totals"]["image_count"] + max(int(planned_image_count), 0), + "estimated_cost": round(current["totals"]["estimated_cost"] + planned_cost, 6), + } + limits = { + "max_task_count": _optional_int(max_task_count), + "max_image_count": _optional_int(max_image_count), + "max_estimated_cost": _optional_float(max_estimated_cost), + } + violations = _violations(projected, limits) + return { + "allowed": not violations, + "provider_id": provider, + "provider_model": model, + "model_route": model_route, + "current": current["totals"], + "planned": { + "task_count": max(int(planned_task_count), 0), + "image_count": max(int(planned_image_count), 0), + "total_tokens": max(int(planned_total_tokens), 0), + "estimated_cost": planned_cost, + }, + "projected": projected, + "limits": limits, + "violations": violations, + } + + +def _violations(projected: dict[str, Any], limits: dict[str, Any]) -> list[str]: + violations: list[str] = [] + if limits["max_task_count"] is not None and projected["task_count"] > limits["max_task_count"]: + violations.append("task_budget_exceeded") + if limits["max_image_count"] is not None and projected["image_count"] > limits["max_image_count"]: + violations.append("image_budget_exceeded") + if limits["max_estimated_cost"] is not None and projected["estimated_cost"] > limits["max_estimated_cost"]: + violations.append("cost_budget_exceeded") + return violations + + +def _optional_int(value: Any) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_float(value: Any) -> float | None: + if value is None or value == "": + return None + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/backend/services/cost_service.py b/backend/services/cost_service.py index 1845c36..e3249b4 100644 --- a/backend/services/cost_service.py +++ b/backend/services/cost_service.py @@ -80,6 +80,32 @@ class CostService: ), } + def estimate_cost( + self, + *, + provider_id: str, + provider_model: str, + model_route: str = "planned", + task_count: int = 1, + image_count: int = 1, + total_tokens: int = 0, + input_tokens: int = 0, + output_tokens: int = 0, + unit_prices: dict[str, PriceSpec] | None = None, + ) -> float: + prices = {**self.unit_prices, **(unit_prices or {})} + group = { + "provider_id": provider_id, + "provider_model": provider_model, + "model_route": model_route, + "task_count": max(_int(task_count), 0), + "image_count": max(_int(image_count), 0), + "total_tokens": max(_int(total_tokens), 0), + "input_tokens": max(_int(input_tokens), 0), + "output_tokens": max(_int(output_tokens), 0), + } + return round(_estimate_cost(group, prices), 6) + def _image_rows( self, *, diff --git a/backend/services/generation_service.py b/backend/services/generation_service.py index 22ea648..661d703 100644 --- a/backend/services/generation_service.py +++ b/backend/services/generation_service.py @@ -6,6 +6,7 @@ from typing import Any from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask +from backend.services.cost_guard_service import CostGuardService from backend.services.image_service import ImageService from backend.services.model_provider_service import ModelProviderService from backend.services.queue_service import QueueService @@ -23,6 +24,7 @@ class GenerationService: image_service: ImageService, single_flow_engine: SingleFlowEngine | None = None, model_provider_service: ModelProviderService | None = None, + cost_guard_service: CostGuardService | None = None, event_bus: EventBus | None = None, ): self.workflow_service = workflow_service @@ -30,6 +32,7 @@ class GenerationService: self.image_service = image_service self.single_flow_engine = single_flow_engine or SingleFlowEngine() self.model_provider_service = model_provider_service + self.cost_guard_service = cost_guard_service self.event_bus = event_bus def generate_node(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]: @@ -37,10 +40,12 @@ class GenerationService: workflow = self.workflow_service.get_workflow(node.workflow_id).workflow model_route = str(payload.get("model_route") or node.model_route or "image_low") route_resolution = self._resolve_model_route(model_route) + cost_guard = self._check_cost_budget(payload, model_route, route_resolution) generation_request = { **payload, "model_route": model_route, "resolved_model_route": route_resolution, + **({"cost_guard": cost_guard} if cost_guard else {}), } idempotency_key = _optional_text(payload.get("idempotency_key")) if idempotency_key: @@ -185,6 +190,32 @@ class GenerationService: "priority": resolved["priority"], } + def _check_cost_budget( + self, + payload: dict[str, Any], + model_route: str, + route_resolution: dict[str, Any], + ) -> dict[str, Any] | None: + budget = payload.get("cost_budget") if isinstance(payload.get("cost_budget"), dict) else None + if not budget or not self.cost_guard_service: + return None + guard = self.cost_guard_service.check_generation_budget( + provider_id=route_resolution.get("provider_id"), + provider_model=route_resolution.get("model_id") or model_route, + model_route=model_route, + planned_task_count=1, + planned_image_count=_planned_image_count(payload.get("count")), + planned_total_tokens=int(budget.get("planned_total_tokens") or 0), + max_task_count=budget.get("max_task_count"), + max_image_count=budget.get("max_image_count"), + max_estimated_cost=budget.get("max_estimated_cost"), + days=budget.get("days"), + unit_prices=budget.get("unit_prices") if isinstance(budget.get("unit_prices"), dict) else None, + ) + if not guard["allowed"]: + raise ApiError(ErrorCode.VALIDATION_FAILED, "generation budget exceeded", guard) + return guard + def _publish(self, event_type: str, payload: dict[str, Any]) -> None: if self.event_bus: self.event_bus.publish(event_type, payload) @@ -203,3 +234,11 @@ def _provider_failure_from_assets(metadata_items: list[dict[str, Any]]) -> dict[ if isinstance(failure, dict): return failure return None + + +def _planned_image_count(value: Any) -> int: + try: + count = int(value) + except (TypeError, ValueError): + count = 1 + return min(max(count, 1), 8) diff --git a/tests/test_cost_service.py b/tests/test_cost_service.py index 65fce10..8fb8bf9 100644 --- a/tests/test_cost_service.py +++ b/tests/test_cost_service.py @@ -5,6 +5,8 @@ import unittest from pathlib import Path from backend.routes.cost_routes import get_cost_usage +from backend.routes.cost_routes import check_cost_guard +from backend.services.cost_guard_service import CostGuardService from backend.services.cost_service import CostService from core.identity import RequestContext from database.db_helper import Database, DatabaseSettings, SqliteSettings @@ -99,6 +101,45 @@ class CostServiceTests(unittest.TestCase): self.assertFalse(anonymous["ok"]) self.assertEqual(anonymous["error_code"], "UNAUTHORIZED") + def test_cost_guard_blocks_projected_budget_overrun(self): + self.image_repository.create( + { + "image_id": "img_b_1", + "task_id": "task_b_1", + "image_uri": "storage://images/b_1.png", + "model_route": "image_high", + "metadata": {"provider_id": "site_b", "provider_model": "image2", "usage": {"total_tokens": 100}}, + } + ) + guard = CostGuardService(CostService(self.database)) + + result = guard.check_generation_budget( + provider_id="site_b", + provider_model="image2", + model_route="image_high", + planned_task_count=1, + planned_image_count=1, + max_task_count=1, + max_estimated_cost=0.05, + unit_prices={"site_b": 0.045}, + ) + route_result = check_cost_guard( + self.ctx, + { + "provider_id": "site_b", + "provider_model": "image2", + "max_estimated_cost": 0.05, + "unit_prices": {"site_b": 0.045}, + }, + guard, + ) + + self.assertFalse(result["allowed"]) + self.assertIn("task_budget_exceeded", result["violations"]) + self.assertIn("cost_budget_exceeded", result["violations"]) + self.assertTrue(route_result["ok"]) + self.assertFalse(route_result["data"]["allowed"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_route_registry.py b/tests/test_route_registry.py index 6097e31..39040d0 100644 --- a/tests/test_route_registry.py +++ b/tests/test_route_registry.py @@ -54,6 +54,7 @@ class RouteRegistryTests(unittest.TestCase): self.assertIn(("POST", "storage/migrate"), keys) self.assertIn(("GET", "events/stream"), keys) self.assertIn(("GET", "costs/usage"), keys) + self.assertIn(("POST", "costs/guard"), keys) self.assertEqual(len(routes), len(ROUTES)) def test_webui_adapter_manifest_includes_api_routes(self): diff --git a/tests/test_single_flow_generation.py b/tests/test_single_flow_generation.py index 616b4cd..ddc91a7 100644 --- a/tests/test_single_flow_generation.py +++ b/tests/test_single_flow_generation.py @@ -8,6 +8,8 @@ from backend.routes.node_routes import generate_node from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask from backend.dto.workflow_dto import WorkflowNode +from backend.services.cost_guard_service import CostGuardService +from backend.services.cost_service import CostService from backend.services.generation_service import GenerationService from backend.services.image_service import ImageService from backend.services.model_provider_service import ModelProviderService @@ -392,6 +394,73 @@ storage: self.assertTrue(image.metadata["mock"]) self.assertEqual(image.metadata["provider_error"]["category"], "auth_error") + def test_generate_node_rejects_request_when_cost_budget_would_be_exceeded(self): + self.model_provider_service.upsert_provider( + { + "provider_id": "site_b", + "base_url": "https://botcf.com", + "image_endpoint": "/v1/images/generations", + } + ) + self.model_provider_service.upsert_route( + { + "route_name": "image_high", + "provider_id": "site_b", + "model_id": "image2", + "priority": 5, + } + ) + profile = self.profile_service.create_profile({"name": "Nova"}, self.ctx) + workflow = self.workflow_service.create_workflow( + profile.profile_id, + {"workflow_type": "single_image", "display_name": "Single"}, + self.ctx, + ) + node_id = workflow.nodes[0].node_id + self.image_service.create_image( + { + "profile_id": profile.profile_id, + "workflow_id": workflow.workflow.workflow_id, + "node_id": node_id, + "task_id": "task_existing_paid", + "image_uri": "storage://images/existing_paid.png", + "model_route": "image_high", + "metadata": { + "provider_id": "site_b", + "provider_model": "image2", + "usage": {"total_tokens": 100}, + }, + }, + self.ctx, + ) + generation_service = GenerationService( + workflow_service=self.workflow_service, + queue_service=self.queue_service, + image_service=self.image_service, + single_flow_engine=SingleFlowEngine(VfsStorage(self.config_service)), + model_provider_service=self.model_provider_service, + cost_guard_service=CostGuardService(CostService(self.database)), + event_bus=self.event_bus, + ) + + response = generate_node( + self.ctx, + node_id, + { + "model_route": "image_high", + "cost_budget": { + "max_estimated_cost": 0.05, + "unit_prices": {"site_b": 0.045}, + }, + }, + generation_service=generation_service, + ) + + self.assertFalse(response["ok"]) + self.assertEqual(response["error_code"], "VALIDATION_FAILED") + self.assertIn("cost_budget_exceeded", response["details"]["violations"]) + self.assertEqual(self.queue_service.list_jobs(page=1, page_size=10).total, 0) + def test_queue_service_rejects_invalid_task_transition(self): task = self.queue_service.create_task({"status": "succeeded"}, self.ctx)