"""Generation orchestration service.""" from __future__ import annotations import threading from typing import Any from backend.dto.common import ApiError, ErrorCode from backend.dto.job_dto import GenerationTask from backend.services.cost_guard_service import CostGuardService from backend.services.generation_reference_service import GenerationReferenceService, MaterialResolver from backend.services.image_service import ImageService from backend.services.model_provider_service import ModelProviderService from backend.services.node_dependency_context_service import NodeDependencyContextService from backend.services.prompt_expansion_service import PromptExpansionService from backend.services.queue_service import QueueService from backend.services.workflow_service import ( LOCKED_NODE_STATUSES, WorkflowService, is_repeatable_single_image_node, locked_node_edit_error, ) from core.event_bus import EventBus from core.generation_errors import provider_failure_attribution from core.identity import RequestContext from engines.image_provider import ImageProviderError from engines.single_flow import SingleFlowEngine _NODE_GENERATION_LOCKS_GUARD = threading.Lock() _NODE_GENERATION_LOCKS: dict[str, "_NodeGenerationLock"] = {} class _NodeGenerationLock: def __init__(self): self.lock = threading.RLock() self.users = 0 class GenerationService: def __init__( self, workflow_service: WorkflowService, queue_service: QueueService, image_service: ImageService, single_flow_engine: SingleFlowEngine | None = None, model_provider_service: ModelProviderService | None = None, cost_guard_service: CostGuardService | None = None, prompt_expansion_service: PromptExpansionService | None = None, event_bus: EventBus | None = None, default_model_route: str | None = None, require_resolved_model_route: bool = False, material_resolver: MaterialResolver | None = None, ): self.workflow_service = workflow_service self.queue_service = queue_service self.image_service = image_service self.single_flow_engine = single_flow_engine or SingleFlowEngine() self.reference_service = GenerationReferenceService( image_service, self.single_flow_engine.storage, material_resolver, ) self.model_provider_service = model_provider_service self.cost_guard_service = cost_guard_service self.prompt_expansion_service = prompt_expansion_service self.event_bus = event_bus self.default_model_route = _optional_text(default_model_route) self.require_resolved_model_route = require_resolved_model_route def generate_node(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]: lock_entry = _retain_node_generation_lock(node_id) lock_entry.lock.acquire() try: return self._generate_node_locked(node_id, payload, ctx) finally: lock_entry.lock.release() _release_node_generation_lock(node_id, lock_entry) def _generate_node_locked(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]: node = self.workflow_service.get_node(node_id) idempotency_key = _optional_text(payload.get("idempotency_key")) if idempotency_key: existing_task = self.queue_service.get_job_by_idempotency_key(idempotency_key) if existing_task is not None: if existing_task.node_id != node_id: raise ApiError( ErrorCode.VALIDATION_FAILED, "idempotency_key belongs to another node", { "idempotency_key": idempotency_key, "node_id": node_id, "existing_node_id": existing_task.node_id, }, ) return self._existing_generation_response(existing_task, node_id) active_task = self._active_generation_task(node) if node.status in {"queued", "running"} and active_task is not None: raise ApiError( ErrorCode.VALIDATION_FAILED, "generation is already in progress for this node", { "node_id": node_id, "status": node.status, "last_task_id": active_task.task_id, "task_status": active_task.status, }, ) if node.status in LOCKED_NODE_STATUSES and not is_repeatable_single_image_node(node): raise locked_node_edit_error(node) workflow = self.workflow_service.get_workflow(node.workflow_id).workflow if workflow.status == "cancelled": raise ApiError( ErrorCode.VALIDATION_FAILED, "generation cannot start for a cancelled workflow", {"workflow_id": workflow.workflow_id, "status": workflow.status}, ) profile = self.workflow_service.profile_repository.get(workflow.profile_id) profile_model_route = _optional_text(getattr(profile, "default_model_route", None)) model_route = str( payload.get("model_route") or node.model_route or profile_model_route or self.default_model_route or "image_low" ) route_resolution = self._resolve_model_route(model_route) if self.require_resolved_model_route and route_resolution.get("status") != "resolved": raise ApiError( ErrorCode.MODEL_CAPABILITY_UNCONFIRMED, "未找到可用的生图模型路由,请先在模型配置中完成提供商与路由设置", {"model_route": model_route, "resolution": route_resolution}, ) cost_guard = self._check_cost_budget(payload, model_route, route_resolution) enriched_payload = self._enrich_generation_payload(node.node_id, payload) prompt_expansion = self._expand_prompt(workflow.profile_id, node, enriched_payload) reference_images = self.reference_service.resolve(workflow.workflow_id, enriched_payload) runtime_generation_request = { **payload, "model_route": model_route, "resolved_model_route": route_resolution, **({"cost_guard": cost_guard} if cost_guard else {}), **( { "prompt_expansion": prompt_expansion, "expanded_prompt": prompt_expansion["prompt"], "expanded_negative_prompt": prompt_expansion["negative_prompt"], } if prompt_expansion else {} ), } generation_request, ephemeral_reference_count = _strip_ephemeral_image_data_urls(runtime_generation_request) if ephemeral_reference_count: generation_request["reference_persistence"] = { "ephemeral_data_urls_omitted": ephemeral_reference_count, } task = self.queue_service.create_task( { "profile_id": workflow.profile_id, "workflow_id": workflow.workflow_id, "node_id": node_id, "status": "queued", "model_route": model_route, "idempotency_key": idempotency_key, "request": generation_request, "metadata": { "source": "nodes.generate", "model_route_resolution": route_resolution, }, }, ctx, ) self._publish( "generation.node_queued", {"workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": task.task_id}, ) queued_metadata = { **node.metadata, "last_task_id": task.task_id, "last_generate_request": generation_request, } execution_metadata = queued_metadata.get("execution") if isinstance(execution_metadata, dict) and execution_metadata.get("prepared_only") is True: queued_metadata["execution"] = { **execution_metadata, "prepared_only": False, "task_id": task.task_id, } queued_node_payload = { "status": "queued", "model_route": model_route, "metadata": queued_metadata, } try: self._queue_generation_node(node, queued_node_payload, ctx) except Exception: if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=[], images=[], ctx=ctx, ) try: self.queue_service.update_task_if_status( task.task_id, {"queued"}, { "status": "failed", "progress": 100, "error_message": "node could not enter the generation queue", }, ctx, ) except Exception: pass raise running_task = self.queue_service.update_task_if_status( task.task_id, {"queued"}, {"status": "running", "progress": 20}, ctx, require_workflow_not_cancelled=True, ) if running_task is None: current_task = self.queue_service.get_job(task.task_id) if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=[], images=[], ctx=ctx, ) raise ApiError( ErrorCode.GENERATION_FAILED, "generation task changed before provider execution", {"task_id": task.task_id, "status": current_task.status}, ) task = running_task running_node = self.workflow_service.update_node(node_id, {"status": "running"}, ctx) self._publish( "generation.node_running", {"workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": task.task_id}, ) if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=[], images=[], ctx=ctx, ) assets: list[Any] = [] images: list[Any] = [] try: runtime_generation_request = { **runtime_generation_request, "input_images": reference_images, } runtime_generation_request.pop("image", None) assets = self.single_flow_engine.generate(running_node, task, runtime_generation_request) if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=assets, images=images, ctx=ctx, ) for asset in assets: images.append( self.image_service.create_image( { "profile_id": workflow.profile_id, "workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": task.task_id, "image_uri": asset.image_uri, "thumbnail_uri": asset.thumbnail_uri, "positive_prompt": asset.positive_prompt, "negative_prompt": asset.negative_prompt, "model_route": asset.model_route, "seed": asset.seed, "width": asset.width, "height": asset.height, # Keep records unselected until the task completion CAS # wins. A concurrent cancellation can then roll them # back without disturbing an older selected image. "is_selected": False, "metadata": asset.metadata, }, ctx, ) ) except Exception as exc: if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=assets, images=images, ctx=ctx, ) self._cleanup_generation_outputs(assets=assets, images=images) failure = self._record_generation_failure( task=task, node_id=node_id, workflow_id=workflow.workflow_id, route_resolution=route_resolution, exc=exc, ctx=ctx, ) if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=assets, images=images, ctx=ctx, ) if isinstance(exc, ApiError): raise raise ApiError( ErrorCode.GENERATION_FAILED, "图像生成失败", failure, ) from exc provider_failure = _provider_failure_from_assets([asset.metadata for asset in assets]) result: dict[str, Any] = {"image_ids": [image.image_id for image in images]} update_payload: dict[str, Any] = { "status": "succeeded", "progress": 100, "result": result, } if provider_failure: result["provider_failure"] = provider_failure update_payload["metadata"] = {**task.metadata, "provider_failure": provider_failure} completed_task = self.queue_service.update_task_if_status( task.task_id, {"running"}, update_payload, ctx, require_workflow_not_cancelled=True, ) if completed_task is None: current_task = self.queue_service.get_job(task.task_id) if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx): return self._finalize_cancelled_generation( task_id=task.task_id, node_id=node_id, workflow_id=workflow.workflow_id, assets=assets, images=images, ctx=ctx, ) self._cleanup_generation_outputs(assets=assets, images=images) raise ApiError( ErrorCode.GENERATION_FAILED, "generation task changed before completion", {"task_id": task.task_id, "status": current_task.status}, ) if images: images[0] = self.image_service.select_image(images[0].image_id, ctx) completed_node = self.workflow_service.update_node(node_id, {"status": "completed"}, ctx) self._publish( "generation.node_completed", { "workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": completed_task.task_id, "image_ids": [image.image_id for image in images], }, ) return { "task_id": completed_task.task_id, "task_status": completed_task.status, "node_status": completed_node.status, "images": [image.to_dict() for image in images], } def _existing_generation_response(self, task: GenerationTask, node_id: str) -> dict[str, Any]: node = self.workflow_service.get_node(node_id) images = [] for image_id in task.result.get("image_ids", []): try: images.append(self.image_service.get_image(str(image_id)).to_dict()) except ApiError as exc: if exc.error_code != ErrorCode.RESOURCE_NOT_FOUND: raise return { "task_id": task.task_id, "task_status": task.status, "node_status": node.status, "images": images, "idempotent_replay": True, } def _active_generation_task(self, node: Any) -> GenerationTask | None: metadata = node.metadata if isinstance(node.metadata, dict) else {} task_id = _optional_text(metadata.get("last_task_id")) if not task_id: return None try: task = self.queue_service.get_job(task_id) except ApiError as exc: if exc.error_code == ErrorCode.RESOURCE_NOT_FOUND: return None raise if task.node_id == node.node_id and task.status in {"queued", "running"}: return task return None def _resolve_model_route(self, route_name: str) -> dict[str, Any]: if not self.model_provider_service: return {"route_name": route_name, "status": "unresolved"} resolved = self.model_provider_service.resolve_route(route_name, modality="image_generation") if not resolved: return {"route_name": route_name, "status": "unresolved"} provider = resolved.get("provider") or {} return { "route_name": resolved["route_name"], "route_id": resolved["route_id"], "status": "resolved", "provider_id": resolved["provider_id"], "provider_type": provider.get("provider_type"), "model_id": resolved["model_id"], "modality": resolved["modality"], "priority": resolved["priority"], } def _check_cost_budget( self, payload: dict[str, Any], model_route: str, route_resolution: dict[str, Any], ) -> dict[str, Any] | None: budget = payload.get("cost_budget") if isinstance(payload.get("cost_budget"), dict) else None if not budget or not self.cost_guard_service: return None guard = self.cost_guard_service.check_generation_budget( provider_id=route_resolution.get("provider_id"), provider_model=route_resolution.get("model_id") or model_route, model_route=model_route, planned_task_count=1, planned_image_count=_planned_image_count(payload.get("count")), planned_total_tokens=int(budget.get("planned_total_tokens") or 0), max_task_count=budget.get("max_task_count"), max_image_count=budget.get("max_image_count"), max_estimated_cost=budget.get("max_estimated_cost"), days=budget.get("days"), unit_prices=budget.get("unit_prices") if isinstance(budget.get("unit_prices"), dict) else None, ) if not guard["allowed"]: raise ApiError(ErrorCode.VALIDATION_FAILED, "generation budget exceeded", guard) return guard def _queue_generation_node( self, original_node: Any, payload: dict[str, Any], ctx: RequestContext, ) -> Any: current = self.workflow_service.get_node(original_node.node_id) if current.status not in LOCKED_NODE_STATUSES: return self.workflow_service.update_node(current.node_id, payload, ctx) if not is_repeatable_single_image_node(current): raise locked_node_edit_error(current) repeat_payload = dict(payload) repeat_metadata = repeat_payload.get("metadata") if isinstance(repeat_payload.get("metadata"), dict) else {} repeat_payload["metadata"] = { **repeat_metadata, "repeat_generation": { "previous_status": current.status, "previous_selected_image_id": current.selected_image_id, "task_id": repeat_metadata.get("last_task_id"), }, } after = self.workflow_service.workflow_repository.update_node(current.node_id, repeat_payload) if after is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "node not found", {"node_id": current.node_id}) self.workflow_service.audit_service.record( ctx=ctx, action="update_node", target_type="workflow_node", target_id=current.node_id, before_snapshot=current.to_dict(), after_snapshot=after.to_dict(), metadata={"source": "generation.repeatable_single_image"}, ) if self.workflow_service.event_bus: self.workflow_service.event_bus.publish( "workflow.node_status_changed", { "workflow_id": after.workflow_id, "node_id": after.node_id, "before_status": current.status, "after_status": after.status, }, ) self.workflow_service.event_bus.publish( "workflow.node_updated", {"workflow_id": after.workflow_id, "node_id": after.node_id}, ) return after def _expand_prompt( self, profile_id: str, node, payload: dict[str, Any], ) -> dict[str, Any] | None: if not self.prompt_expansion_service: return None profile = self.workflow_service.profile_repository.get(profile_id) if profile is None: return None return self.prompt_expansion_service.expand_for_node(profile, node, payload) def _enrich_generation_payload(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]: enriched_payload = dict(payload) if not isinstance(enriched_payload.get("dependency_context"), dict): enriched_payload["dependency_context"] = NodeDependencyContextService( self.workflow_service, self.image_service, ).get_context(node_id) return enriched_payload def _publish(self, event_type: str, payload: dict[str, Any]) -> None: if self.event_bus: self.event_bus.publish(event_type, payload) def _cancel_generation_if_requested( self, task_id: str, workflow_id: str, ctx: RequestContext, ) -> bool: task = self.queue_service.get_job(task_id) if task.status == "cancelled": return True workflow = self.workflow_service.get_workflow(workflow_id).workflow if workflow.status != "cancelled" or task.status not in {"queued", "running"}: return False try: task = self.queue_service.cancel_job( task_id, ctx, reason="owning workflow was cancelled", ) except ApiError: task = self.queue_service.get_job(task_id) return task.status == "cancelled" def _finalize_cancelled_generation( self, *, task_id: str, node_id: str, workflow_id: str, assets: list[Any], images: list[Any], ctx: RequestContext, ) -> dict[str, Any]: cleanup = self._cleanup_generation_outputs(assets=assets, images=images) task = self.queue_service.get_job(task_id) node = self.workflow_service.get_node(node_id) node_metadata = node.metadata if isinstance(node.metadata, dict) else {} node_task_id = _optional_text(node_metadata.get("last_task_id")) if node_task_id == task_id and node.status in {"queued", "running", "cancelled"}: cancellation = task.metadata.get("cancellation") if isinstance(task.metadata, dict) else None cancellation_reason = ( _optional_text(cancellation.get("reason")) if isinstance(cancellation, dict) else None ) node_update: dict[str, Any] = { "metadata": { **node_metadata, "generation_cancellation": { "task_id": task_id, "reason": cancellation_reason or "cancelled", "cleanup": cleanup, }, }, } if node.status in {"queued", "running"}: node_update["status"] = "cancelled" node = self.workflow_service.update_node( node_id, node_update, ctx, ) self._publish( "generation.node_cancelled", { "workflow_id": workflow_id, "node_id": node_id, "task_id": task_id, "node_status": node.status, "cleanup": cleanup, }, ) return { "task_id": task.task_id, "task_status": task.status, "node_status": node.status, "images": [], "cancelled": True, } def _cleanup_generation_outputs(self, *, assets: list[Any], images: list[Any]) -> dict[str, Any]: removed_image_ids: list[str] = [] removed_uris: list[str] = [] errors: list[dict[str, str]] = [] for image in reversed(images): image_id = _optional_text(getattr(image, "image_id", None)) if not image_id: continue try: removed = self.image_service.image_repository.delete_metadata(image_id) if removed is not None: removed_image_ids.append(image_id) except Exception as exc: errors.append( { "target": image_id, "error_type": exc.__class__.__name__, "message": str(exc), } ) seen_uris: set[str] = set() for asset in assets: for field in ("image_uri", "thumbnail_uri"): uri = _optional_text(getattr(asset, field, None)) if not uri or uri in seen_uris: continue seen_uris.add(uri) try: path = self.single_flow_engine.storage.resolve_uri(uri) if path.exists() and path.is_file(): path.unlink() removed_uris.append(uri) except Exception as exc: errors.append( { "target": uri, "error_type": exc.__class__.__name__, "message": str(exc), } ) return { "image_records_removed": removed_image_ids, "files_removed": removed_uris, "errors": errors, } def _record_generation_failure( self, *, task: GenerationTask, node_id: str, workflow_id: str, route_resolution: dict[str, Any], exc: Exception, ctx: RequestContext, ) -> dict[str, Any]: if isinstance(exc, ImageProviderError): failure = provider_failure_attribution(exc, route_context=route_resolution, fallback_used=False) else: failure = { "source": "generation_pipeline", "category": "internal_error", "error_type": exc.__class__.__name__, "message": "generation pipeline failed", "retryable": False, "fallback_used": False, } failed_task = None try: failed_task = self.queue_service.update_task_if_status( task.task_id, {"queued", "running"}, { "status": "failed", "progress": 100, "error_message": str(failure.get("message") or "generation failed"), "metadata": {**task.metadata, "generation_failure": failure}, }, ctx, ) except Exception: pass if failed_task is not None: try: node = self.workflow_service.get_node(node_id) node_metadata = node.metadata if isinstance(node.metadata, dict) else {} if ( _optional_text(node_metadata.get("last_task_id")) == task.task_id and node.status in {"queued", "running"} ): self.workflow_service.update_node( node_id, { "status": "failed", "metadata": { **node_metadata, "generation_failure": failure, "last_task_id": task.task_id, }, }, ctx, ) except Exception: pass self._publish( "generation.node_failed", {"workflow_id": workflow_id, "node_id": node_id, "task_id": task.task_id, "failure": failure}, ) return failure def _retain_node_generation_lock(node_id: str) -> _NodeGenerationLock: key = str(node_id) with _NODE_GENERATION_LOCKS_GUARD: entry = _NODE_GENERATION_LOCKS.get(key) if entry is None: entry = _NodeGenerationLock() _NODE_GENERATION_LOCKS[key] = entry entry.users += 1 return entry def _release_node_generation_lock(node_id: str, entry: _NodeGenerationLock) -> None: key = str(node_id) with _NODE_GENERATION_LOCKS_GUARD: entry.users -= 1 if entry.users == 0 and _NODE_GENERATION_LOCKS.get(key) is entry: _NODE_GENERATION_LOCKS.pop(key, None) def _optional_text(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None _OMITTED_EPHEMERAL_REFERENCE = object() def _strip_ephemeral_image_data_urls(value: Any) -> tuple[Any, int]: """Remove inline image payloads before writing a generation request to storage.""" if isinstance(value, str): if value.lstrip().lower().startswith("data:image/"): return _OMITTED_EPHEMERAL_REFERENCE, 1 return value, 0 if isinstance(value, dict): cleaned: dict[Any, Any] = {} omitted = 0 for key, item in value.items(): prepared, item_omitted = _strip_ephemeral_image_data_urls(item) omitted += item_omitted if prepared is not _OMITTED_EPHEMERAL_REFERENCE: cleaned[key] = prepared return cleaned, omitted if isinstance(value, (list, tuple)): cleaned_items: list[Any] = [] omitted = 0 for item in value: prepared, item_omitted = _strip_ephemeral_image_data_urls(item) omitted += item_omitted if prepared is not _OMITTED_EPHEMERAL_REFERENCE: cleaned_items.append(prepared) return cleaned_items, omitted return value, 0 def _provider_failure_from_assets(metadata_items: list[dict[str, Any]]) -> dict[str, Any] | None: for metadata in metadata_items: failure = metadata.get("provider_error") if isinstance(failure, dict): return failure return None def _planned_image_count(value: Any) -> int: try: count = int(value) except (TypeError, ValueError): count = 1 return min(max(count, 1), 8)