2026-07-05 09:28:39 +08:00
|
|
|
"""Generation orchestration service."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
import threading
|
2026-07-05 09:28:39 +08:00
|
|
|
from typing import Any
|
|
|
|
|
|
2026-07-05 18:23:35 +08:00
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
|
|
|
from backend.dto.job_dto import GenerationTask
|
2026-07-06 02:40:18 +08:00
|
|
|
from backend.services.cost_guard_service import CostGuardService
|
2026-07-11 21:50:34 +08:00
|
|
|
from backend.services.generation_reference_service import GenerationReferenceService, MaterialResolver
|
2026-07-05 09:28:39 +08:00
|
|
|
from backend.services.image_service import ImageService
|
2026-07-06 02:01:45 +08:00
|
|
|
from backend.services.model_provider_service import ModelProviderService
|
2026-07-10 01:25:15 +08:00
|
|
|
from backend.services.node_dependency_context_service import NodeDependencyContextService
|
2026-07-06 02:47:37 +08:00
|
|
|
from backend.services.prompt_expansion_service import PromptExpansionService
|
2026-07-05 09:28:39 +08:00
|
|
|
from backend.services.queue_service import QueueService
|
2026-07-08 02:25:50 +08:00
|
|
|
from backend.services.workflow_service import (
|
|
|
|
|
LOCKED_NODE_STATUSES,
|
|
|
|
|
WorkflowService,
|
2026-07-11 21:50:34 +08:00
|
|
|
is_repeatable_single_image_node,
|
2026-07-08 02:25:50 +08:00
|
|
|
locked_node_edit_error,
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
from core.event_bus import EventBus
|
2026-07-11 21:50:34 +08:00
|
|
|
from core.generation_errors import provider_failure_attribution
|
2026-07-05 09:28:39 +08:00
|
|
|
from core.identity import RequestContext
|
2026-07-11 21:50:34 +08:00
|
|
|
from engines.image_provider import ImageProviderError
|
2026-07-05 09:28:39 +08:00
|
|
|
from engines.single_flow import SingleFlowEngine
|
|
|
|
|
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
_NODE_GENERATION_LOCKS_GUARD = threading.Lock()
|
|
|
|
|
_NODE_GENERATION_LOCKS: dict[str, "_NodeGenerationLock"] = {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _NodeGenerationLock:
|
|
|
|
|
def __init__(self):
|
|
|
|
|
self.lock = threading.RLock()
|
|
|
|
|
self.users = 0
|
|
|
|
|
|
|
|
|
|
|
2026-07-05 09:28:39 +08:00
|
|
|
class GenerationService:
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
workflow_service: WorkflowService,
|
|
|
|
|
queue_service: QueueService,
|
|
|
|
|
image_service: ImageService,
|
|
|
|
|
single_flow_engine: SingleFlowEngine | None = None,
|
2026-07-06 02:01:45 +08:00
|
|
|
model_provider_service: ModelProviderService | None = None,
|
2026-07-06 02:40:18 +08:00
|
|
|
cost_guard_service: CostGuardService | None = None,
|
2026-07-06 02:47:37 +08:00
|
|
|
prompt_expansion_service: PromptExpansionService | None = None,
|
2026-07-05 15:54:48 +08:00
|
|
|
event_bus: EventBus | None = None,
|
2026-07-11 21:50:34 +08:00
|
|
|
default_model_route: str | None = None,
|
|
|
|
|
require_resolved_model_route: bool = False,
|
|
|
|
|
material_resolver: MaterialResolver | None = None,
|
2026-07-05 09:28:39 +08:00
|
|
|
):
|
|
|
|
|
self.workflow_service = workflow_service
|
|
|
|
|
self.queue_service = queue_service
|
|
|
|
|
self.image_service = image_service
|
|
|
|
|
self.single_flow_engine = single_flow_engine or SingleFlowEngine()
|
2026-07-11 21:50:34 +08:00
|
|
|
self.reference_service = GenerationReferenceService(
|
|
|
|
|
image_service,
|
|
|
|
|
self.single_flow_engine.storage,
|
|
|
|
|
material_resolver,
|
|
|
|
|
)
|
2026-07-06 02:01:45 +08:00
|
|
|
self.model_provider_service = model_provider_service
|
2026-07-06 02:40:18 +08:00
|
|
|
self.cost_guard_service = cost_guard_service
|
2026-07-06 02:47:37 +08:00
|
|
|
self.prompt_expansion_service = prompt_expansion_service
|
2026-07-05 15:54:48 +08:00
|
|
|
self.event_bus = event_bus
|
2026-07-11 21:50:34 +08:00
|
|
|
self.default_model_route = _optional_text(default_model_route)
|
|
|
|
|
self.require_resolved_model_route = require_resolved_model_route
|
2026-07-05 09:28:39 +08:00
|
|
|
|
|
|
|
|
def generate_node(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]:
|
2026-07-11 21:50:34 +08:00
|
|
|
lock_entry = _retain_node_generation_lock(node_id)
|
|
|
|
|
lock_entry.lock.acquire()
|
|
|
|
|
try:
|
|
|
|
|
return self._generate_node_locked(node_id, payload, ctx)
|
|
|
|
|
finally:
|
|
|
|
|
lock_entry.lock.release()
|
|
|
|
|
_release_node_generation_lock(node_id, lock_entry)
|
|
|
|
|
|
|
|
|
|
def _generate_node_locked(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]:
|
2026-07-05 09:28:39 +08:00
|
|
|
node = self.workflow_service.get_node(node_id)
|
2026-07-08 02:25:50 +08:00
|
|
|
idempotency_key = _optional_text(payload.get("idempotency_key"))
|
|
|
|
|
if idempotency_key:
|
|
|
|
|
existing_task = self.queue_service.get_job_by_idempotency_key(idempotency_key)
|
|
|
|
|
if existing_task is not None:
|
|
|
|
|
if existing_task.node_id != node_id:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"idempotency_key belongs to another node",
|
|
|
|
|
{
|
|
|
|
|
"idempotency_key": idempotency_key,
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"existing_node_id": existing_task.node_id,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return self._existing_generation_response(existing_task, node_id)
|
2026-07-11 21:50:34 +08:00
|
|
|
active_task = self._active_generation_task(node)
|
|
|
|
|
if node.status in {"queued", "running"} and active_task is not None:
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"generation is already in progress for this node",
|
|
|
|
|
{
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"status": node.status,
|
|
|
|
|
"last_task_id": active_task.task_id,
|
|
|
|
|
"task_status": active_task.status,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if node.status in LOCKED_NODE_STATUSES and not is_repeatable_single_image_node(node):
|
2026-07-08 02:25:50 +08:00
|
|
|
raise locked_node_edit_error(node)
|
2026-07-05 09:28:39 +08:00
|
|
|
workflow = self.workflow_service.get_workflow(node.workflow_id).workflow
|
2026-07-11 21:50:34 +08:00
|
|
|
if workflow.status == "cancelled":
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.VALIDATION_FAILED,
|
|
|
|
|
"generation cannot start for a cancelled workflow",
|
|
|
|
|
{"workflow_id": workflow.workflow_id, "status": workflow.status},
|
|
|
|
|
)
|
|
|
|
|
profile = self.workflow_service.profile_repository.get(workflow.profile_id)
|
|
|
|
|
profile_model_route = _optional_text(getattr(profile, "default_model_route", None))
|
|
|
|
|
model_route = str(
|
|
|
|
|
payload.get("model_route")
|
|
|
|
|
or node.model_route
|
|
|
|
|
or profile_model_route
|
|
|
|
|
or self.default_model_route
|
|
|
|
|
or "image_low"
|
|
|
|
|
)
|
2026-07-06 02:01:45 +08:00
|
|
|
route_resolution = self._resolve_model_route(model_route)
|
2026-07-11 21:50:34 +08:00
|
|
|
if self.require_resolved_model_route and route_resolution.get("status") != "resolved":
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.MODEL_CAPABILITY_UNCONFIRMED,
|
|
|
|
|
"未找到可用的生图模型路由,请先在模型配置中完成提供商与路由设置",
|
|
|
|
|
{"model_route": model_route, "resolution": route_resolution},
|
|
|
|
|
)
|
2026-07-06 02:40:18 +08:00
|
|
|
cost_guard = self._check_cost_budget(payload, model_route, route_resolution)
|
2026-07-11 21:50:34 +08:00
|
|
|
enriched_payload = self._enrich_generation_payload(node.node_id, payload)
|
|
|
|
|
prompt_expansion = self._expand_prompt(workflow.profile_id, node, enriched_payload)
|
|
|
|
|
reference_images = self.reference_service.resolve(workflow.workflow_id, enriched_payload)
|
|
|
|
|
runtime_generation_request = {
|
2026-07-06 02:01:45 +08:00
|
|
|
**payload,
|
|
|
|
|
"model_route": model_route,
|
|
|
|
|
"resolved_model_route": route_resolution,
|
2026-07-06 02:40:18 +08:00
|
|
|
**({"cost_guard": cost_guard} if cost_guard else {}),
|
2026-07-06 02:47:37 +08:00
|
|
|
**(
|
|
|
|
|
{
|
|
|
|
|
"prompt_expansion": prompt_expansion,
|
|
|
|
|
"expanded_prompt": prompt_expansion["prompt"],
|
|
|
|
|
"expanded_negative_prompt": prompt_expansion["negative_prompt"],
|
|
|
|
|
}
|
|
|
|
|
if prompt_expansion
|
|
|
|
|
else {}
|
|
|
|
|
),
|
2026-07-06 02:01:45 +08:00
|
|
|
}
|
2026-07-11 21:50:34 +08:00
|
|
|
generation_request, ephemeral_reference_count = _strip_ephemeral_image_data_urls(runtime_generation_request)
|
|
|
|
|
if ephemeral_reference_count:
|
|
|
|
|
generation_request["reference_persistence"] = {
|
|
|
|
|
"ephemeral_data_urls_omitted": ephemeral_reference_count,
|
|
|
|
|
}
|
2026-07-05 09:28:39 +08:00
|
|
|
task = self.queue_service.create_task(
|
|
|
|
|
{
|
|
|
|
|
"profile_id": workflow.profile_id,
|
|
|
|
|
"workflow_id": workflow.workflow_id,
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"status": "queued",
|
2026-07-06 02:01:45 +08:00
|
|
|
"model_route": model_route,
|
2026-07-05 18:23:35 +08:00
|
|
|
"idempotency_key": idempotency_key,
|
2026-07-06 02:01:45 +08:00
|
|
|
"request": generation_request,
|
|
|
|
|
"metadata": {
|
|
|
|
|
"source": "nodes.generate",
|
|
|
|
|
"model_route_resolution": route_resolution,
|
|
|
|
|
},
|
2026-07-05 09:28:39 +08:00
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish(
|
|
|
|
|
"generation.node_queued",
|
|
|
|
|
{"workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": task.task_id},
|
|
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
queued_metadata = {
|
|
|
|
|
**node.metadata,
|
|
|
|
|
"last_task_id": task.task_id,
|
|
|
|
|
"last_generate_request": generation_request,
|
|
|
|
|
}
|
|
|
|
|
execution_metadata = queued_metadata.get("execution")
|
|
|
|
|
if isinstance(execution_metadata, dict) and execution_metadata.get("prepared_only") is True:
|
|
|
|
|
queued_metadata["execution"] = {
|
|
|
|
|
**execution_metadata,
|
|
|
|
|
"prepared_only": False,
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
}
|
|
|
|
|
queued_node_payload = {
|
|
|
|
|
"status": "queued",
|
|
|
|
|
"model_route": model_route,
|
|
|
|
|
"metadata": queued_metadata,
|
|
|
|
|
}
|
|
|
|
|
try:
|
|
|
|
|
self._queue_generation_node(node, queued_node_payload, ctx)
|
|
|
|
|
except Exception:
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=[],
|
|
|
|
|
images=[],
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
try:
|
|
|
|
|
self.queue_service.update_task_if_status(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{"queued"},
|
|
|
|
|
{
|
|
|
|
|
"status": "failed",
|
|
|
|
|
"progress": 100,
|
|
|
|
|
"error_message": "node could not enter the generation queue",
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
pass
|
|
|
|
|
raise
|
|
|
|
|
running_task = self.queue_service.update_task_if_status(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{"queued"},
|
|
|
|
|
{"status": "running", "progress": 20},
|
2026-07-05 09:28:39 +08:00
|
|
|
ctx,
|
2026-07-11 21:50:34 +08:00
|
|
|
require_workflow_not_cancelled=True,
|
2026-07-05 09:28:39 +08:00
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
if running_task is None:
|
|
|
|
|
current_task = self.queue_service.get_job(task.task_id)
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=[],
|
|
|
|
|
images=[],
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.GENERATION_FAILED,
|
|
|
|
|
"generation task changed before provider execution",
|
|
|
|
|
{"task_id": task.task_id, "status": current_task.status},
|
|
|
|
|
)
|
|
|
|
|
task = running_task
|
2026-07-05 09:28:39 +08:00
|
|
|
running_node = self.workflow_service.update_node(node_id, {"status": "running"}, ctx)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish(
|
|
|
|
|
"generation.node_running",
|
|
|
|
|
{"workflow_id": workflow.workflow_id, "node_id": node_id, "task_id": task.task_id},
|
|
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=[],
|
|
|
|
|
images=[],
|
|
|
|
|
ctx=ctx,
|
2026-07-05 09:28:39 +08:00
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
assets: list[Any] = []
|
|
|
|
|
images: list[Any] = []
|
|
|
|
|
try:
|
|
|
|
|
runtime_generation_request = {
|
|
|
|
|
**runtime_generation_request,
|
|
|
|
|
"input_images": reference_images,
|
|
|
|
|
}
|
|
|
|
|
runtime_generation_request.pop("image", None)
|
|
|
|
|
assets = self.single_flow_engine.generate(running_node, task, runtime_generation_request)
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=assets,
|
|
|
|
|
images=images,
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
for asset in assets:
|
|
|
|
|
images.append(
|
|
|
|
|
self.image_service.create_image(
|
|
|
|
|
{
|
|
|
|
|
"profile_id": workflow.profile_id,
|
|
|
|
|
"workflow_id": workflow.workflow_id,
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
"image_uri": asset.image_uri,
|
|
|
|
|
"thumbnail_uri": asset.thumbnail_uri,
|
|
|
|
|
"positive_prompt": asset.positive_prompt,
|
|
|
|
|
"negative_prompt": asset.negative_prompt,
|
|
|
|
|
"model_route": asset.model_route,
|
|
|
|
|
"seed": asset.seed,
|
|
|
|
|
"width": asset.width,
|
|
|
|
|
"height": asset.height,
|
|
|
|
|
# Keep records unselected until the task completion CAS
|
|
|
|
|
# wins. A concurrent cancellation can then roll them
|
|
|
|
|
# back without disturbing an older selected image.
|
|
|
|
|
"is_selected": False,
|
|
|
|
|
"metadata": asset.metadata,
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=assets,
|
|
|
|
|
images=images,
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
self._cleanup_generation_outputs(assets=assets, images=images)
|
|
|
|
|
failure = self._record_generation_failure(
|
|
|
|
|
task=task,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
route_resolution=route_resolution,
|
|
|
|
|
exc=exc,
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=assets,
|
|
|
|
|
images=images,
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
if isinstance(exc, ApiError):
|
|
|
|
|
raise
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.GENERATION_FAILED,
|
|
|
|
|
"图像生成失败",
|
|
|
|
|
failure,
|
|
|
|
|
) from exc
|
2026-07-06 02:14:13 +08:00
|
|
|
provider_failure = _provider_failure_from_assets([asset.metadata for asset in assets])
|
|
|
|
|
result: dict[str, Any] = {"image_ids": [image.image_id for image in images]}
|
|
|
|
|
update_payload: dict[str, Any] = {
|
|
|
|
|
"status": "succeeded",
|
|
|
|
|
"progress": 100,
|
|
|
|
|
"result": result,
|
|
|
|
|
}
|
|
|
|
|
if provider_failure:
|
|
|
|
|
result["provider_failure"] = provider_failure
|
|
|
|
|
update_payload["metadata"] = {**task.metadata, "provider_failure": provider_failure}
|
2026-07-11 21:50:34 +08:00
|
|
|
completed_task = self.queue_service.update_task_if_status(
|
2026-07-05 09:28:39 +08:00
|
|
|
task.task_id,
|
2026-07-11 21:50:34 +08:00
|
|
|
{"running"},
|
2026-07-06 02:14:13 +08:00
|
|
|
update_payload,
|
2026-07-05 09:28:39 +08:00
|
|
|
ctx,
|
2026-07-11 21:50:34 +08:00
|
|
|
require_workflow_not_cancelled=True,
|
2026-07-05 09:28:39 +08:00
|
|
|
)
|
2026-07-11 21:50:34 +08:00
|
|
|
if completed_task is None:
|
|
|
|
|
current_task = self.queue_service.get_job(task.task_id)
|
|
|
|
|
if self._cancel_generation_if_requested(task.task_id, workflow.workflow_id, ctx):
|
|
|
|
|
return self._finalize_cancelled_generation(
|
|
|
|
|
task_id=task.task_id,
|
|
|
|
|
node_id=node_id,
|
|
|
|
|
workflow_id=workflow.workflow_id,
|
|
|
|
|
assets=assets,
|
|
|
|
|
images=images,
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
)
|
|
|
|
|
self._cleanup_generation_outputs(assets=assets, images=images)
|
|
|
|
|
raise ApiError(
|
|
|
|
|
ErrorCode.GENERATION_FAILED,
|
|
|
|
|
"generation task changed before completion",
|
|
|
|
|
{"task_id": task.task_id, "status": current_task.status},
|
|
|
|
|
)
|
|
|
|
|
if images:
|
|
|
|
|
images[0] = self.image_service.select_image(images[0].image_id, ctx)
|
2026-07-05 09:28:39 +08:00
|
|
|
completed_node = self.workflow_service.update_node(node_id, {"status": "completed"}, ctx)
|
2026-07-05 15:54:48 +08:00
|
|
|
self._publish(
|
|
|
|
|
"generation.node_completed",
|
|
|
|
|
{
|
|
|
|
|
"workflow_id": workflow.workflow_id,
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"task_id": completed_task.task_id,
|
|
|
|
|
"image_ids": [image.image_id for image in images],
|
|
|
|
|
},
|
|
|
|
|
)
|
2026-07-05 09:28:39 +08:00
|
|
|
return {
|
|
|
|
|
"task_id": completed_task.task_id,
|
|
|
|
|
"task_status": completed_task.status,
|
|
|
|
|
"node_status": completed_node.status,
|
|
|
|
|
"images": [image.to_dict() for image in images],
|
|
|
|
|
}
|
2026-07-05 15:54:48 +08:00
|
|
|
|
2026-07-05 18:23:35 +08:00
|
|
|
def _existing_generation_response(self, task: GenerationTask, node_id: str) -> dict[str, Any]:
|
|
|
|
|
node = self.workflow_service.get_node(node_id)
|
|
|
|
|
images = []
|
|
|
|
|
for image_id in task.result.get("image_ids", []):
|
|
|
|
|
try:
|
|
|
|
|
images.append(self.image_service.get_image(str(image_id)).to_dict())
|
|
|
|
|
except ApiError as exc:
|
|
|
|
|
if exc.error_code != ErrorCode.RESOURCE_NOT_FOUND:
|
|
|
|
|
raise
|
|
|
|
|
return {
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
"task_status": task.status,
|
|
|
|
|
"node_status": node.status,
|
|
|
|
|
"images": images,
|
|
|
|
|
"idempotent_replay": True,
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def _active_generation_task(self, node: Any) -> GenerationTask | None:
|
|
|
|
|
metadata = node.metadata if isinstance(node.metadata, dict) else {}
|
|
|
|
|
task_id = _optional_text(metadata.get("last_task_id"))
|
|
|
|
|
if not task_id:
|
|
|
|
|
return None
|
|
|
|
|
try:
|
|
|
|
|
task = self.queue_service.get_job(task_id)
|
|
|
|
|
except ApiError as exc:
|
|
|
|
|
if exc.error_code == ErrorCode.RESOURCE_NOT_FOUND:
|
|
|
|
|
return None
|
|
|
|
|
raise
|
|
|
|
|
if task.node_id == node.node_id and task.status in {"queued", "running"}:
|
|
|
|
|
return task
|
|
|
|
|
return None
|
|
|
|
|
|
2026-07-06 02:01:45 +08:00
|
|
|
def _resolve_model_route(self, route_name: str) -> dict[str, Any]:
|
|
|
|
|
if not self.model_provider_service:
|
|
|
|
|
return {"route_name": route_name, "status": "unresolved"}
|
|
|
|
|
resolved = self.model_provider_service.resolve_route(route_name, modality="image_generation")
|
|
|
|
|
if not resolved:
|
|
|
|
|
return {"route_name": route_name, "status": "unresolved"}
|
|
|
|
|
provider = resolved.get("provider") or {}
|
|
|
|
|
return {
|
|
|
|
|
"route_name": resolved["route_name"],
|
|
|
|
|
"route_id": resolved["route_id"],
|
|
|
|
|
"status": "resolved",
|
|
|
|
|
"provider_id": resolved["provider_id"],
|
|
|
|
|
"provider_type": provider.get("provider_type"),
|
|
|
|
|
"model_id": resolved["model_id"],
|
|
|
|
|
"modality": resolved["modality"],
|
|
|
|
|
"priority": resolved["priority"],
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-06 02:40:18 +08:00
|
|
|
def _check_cost_budget(
|
|
|
|
|
self,
|
|
|
|
|
payload: dict[str, Any],
|
|
|
|
|
model_route: str,
|
|
|
|
|
route_resolution: dict[str, Any],
|
|
|
|
|
) -> dict[str, Any] | None:
|
|
|
|
|
budget = payload.get("cost_budget") if isinstance(payload.get("cost_budget"), dict) else None
|
|
|
|
|
if not budget or not self.cost_guard_service:
|
|
|
|
|
return None
|
|
|
|
|
guard = self.cost_guard_service.check_generation_budget(
|
|
|
|
|
provider_id=route_resolution.get("provider_id"),
|
|
|
|
|
provider_model=route_resolution.get("model_id") or model_route,
|
|
|
|
|
model_route=model_route,
|
|
|
|
|
planned_task_count=1,
|
|
|
|
|
planned_image_count=_planned_image_count(payload.get("count")),
|
|
|
|
|
planned_total_tokens=int(budget.get("planned_total_tokens") or 0),
|
|
|
|
|
max_task_count=budget.get("max_task_count"),
|
|
|
|
|
max_image_count=budget.get("max_image_count"),
|
|
|
|
|
max_estimated_cost=budget.get("max_estimated_cost"),
|
|
|
|
|
days=budget.get("days"),
|
|
|
|
|
unit_prices=budget.get("unit_prices") if isinstance(budget.get("unit_prices"), dict) else None,
|
|
|
|
|
)
|
|
|
|
|
if not guard["allowed"]:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "generation budget exceeded", guard)
|
|
|
|
|
return guard
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def _queue_generation_node(
|
|
|
|
|
self,
|
|
|
|
|
original_node: Any,
|
|
|
|
|
payload: dict[str, Any],
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
) -> Any:
|
|
|
|
|
current = self.workflow_service.get_node(original_node.node_id)
|
|
|
|
|
if current.status not in LOCKED_NODE_STATUSES:
|
|
|
|
|
return self.workflow_service.update_node(current.node_id, payload, ctx)
|
|
|
|
|
if not is_repeatable_single_image_node(current):
|
|
|
|
|
raise locked_node_edit_error(current)
|
|
|
|
|
|
|
|
|
|
repeat_payload = dict(payload)
|
|
|
|
|
repeat_metadata = repeat_payload.get("metadata") if isinstance(repeat_payload.get("metadata"), dict) else {}
|
|
|
|
|
repeat_payload["metadata"] = {
|
|
|
|
|
**repeat_metadata,
|
|
|
|
|
"repeat_generation": {
|
|
|
|
|
"previous_status": current.status,
|
|
|
|
|
"previous_selected_image_id": current.selected_image_id,
|
|
|
|
|
"task_id": repeat_metadata.get("last_task_id"),
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
after = self.workflow_service.workflow_repository.update_node(current.node_id, repeat_payload)
|
|
|
|
|
if after is None:
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "node not found", {"node_id": current.node_id})
|
|
|
|
|
self.workflow_service.audit_service.record(
|
|
|
|
|
ctx=ctx,
|
|
|
|
|
action="update_node",
|
|
|
|
|
target_type="workflow_node",
|
|
|
|
|
target_id=current.node_id,
|
|
|
|
|
before_snapshot=current.to_dict(),
|
|
|
|
|
after_snapshot=after.to_dict(),
|
|
|
|
|
metadata={"source": "generation.repeatable_single_image"},
|
|
|
|
|
)
|
|
|
|
|
if self.workflow_service.event_bus:
|
|
|
|
|
self.workflow_service.event_bus.publish(
|
|
|
|
|
"workflow.node_status_changed",
|
|
|
|
|
{
|
|
|
|
|
"workflow_id": after.workflow_id,
|
|
|
|
|
"node_id": after.node_id,
|
|
|
|
|
"before_status": current.status,
|
|
|
|
|
"after_status": after.status,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
self.workflow_service.event_bus.publish(
|
|
|
|
|
"workflow.node_updated",
|
|
|
|
|
{"workflow_id": after.workflow_id, "node_id": after.node_id},
|
|
|
|
|
)
|
|
|
|
|
return after
|
|
|
|
|
|
2026-07-06 02:47:37 +08:00
|
|
|
def _expand_prompt(
|
|
|
|
|
self,
|
|
|
|
|
profile_id: str,
|
|
|
|
|
node,
|
|
|
|
|
payload: dict[str, Any],
|
|
|
|
|
) -> dict[str, Any] | None:
|
|
|
|
|
if not self.prompt_expansion_service:
|
|
|
|
|
return None
|
|
|
|
|
profile = self.workflow_service.profile_repository.get(profile_id)
|
|
|
|
|
if profile is None:
|
|
|
|
|
return None
|
2026-07-11 21:50:34 +08:00
|
|
|
return self.prompt_expansion_service.expand_for_node(profile, node, payload)
|
|
|
|
|
|
|
|
|
|
def _enrich_generation_payload(self, node_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
2026-07-10 01:25:15 +08:00
|
|
|
enriched_payload = dict(payload)
|
2026-07-11 21:50:34 +08:00
|
|
|
if not isinstance(enriched_payload.get("dependency_context"), dict):
|
2026-07-10 01:25:15 +08:00
|
|
|
enriched_payload["dependency_context"] = NodeDependencyContextService(
|
|
|
|
|
self.workflow_service,
|
|
|
|
|
self.image_service,
|
2026-07-11 21:50:34 +08:00
|
|
|
).get_context(node_id)
|
|
|
|
|
return enriched_payload
|
2026-07-06 02:47:37 +08:00
|
|
|
|
2026-07-05 15:54:48 +08:00
|
|
|
def _publish(self, event_type: str, payload: dict[str, Any]) -> None:
|
|
|
|
|
if self.event_bus:
|
|
|
|
|
self.event_bus.publish(event_type, payload)
|
2026-07-05 18:23:35 +08:00
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
def _cancel_generation_if_requested(
|
|
|
|
|
self,
|
|
|
|
|
task_id: str,
|
|
|
|
|
workflow_id: str,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
) -> bool:
|
|
|
|
|
task = self.queue_service.get_job(task_id)
|
|
|
|
|
if task.status == "cancelled":
|
|
|
|
|
return True
|
|
|
|
|
workflow = self.workflow_service.get_workflow(workflow_id).workflow
|
|
|
|
|
if workflow.status != "cancelled" or task.status not in {"queued", "running"}:
|
|
|
|
|
return False
|
|
|
|
|
try:
|
|
|
|
|
task = self.queue_service.cancel_job(
|
|
|
|
|
task_id,
|
|
|
|
|
ctx,
|
|
|
|
|
reason="owning workflow was cancelled",
|
|
|
|
|
)
|
|
|
|
|
except ApiError:
|
|
|
|
|
task = self.queue_service.get_job(task_id)
|
|
|
|
|
return task.status == "cancelled"
|
|
|
|
|
|
|
|
|
|
def _finalize_cancelled_generation(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
task_id: str,
|
|
|
|
|
node_id: str,
|
|
|
|
|
workflow_id: str,
|
|
|
|
|
assets: list[Any],
|
|
|
|
|
images: list[Any],
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
cleanup = self._cleanup_generation_outputs(assets=assets, images=images)
|
|
|
|
|
task = self.queue_service.get_job(task_id)
|
|
|
|
|
node = self.workflow_service.get_node(node_id)
|
|
|
|
|
node_metadata = node.metadata if isinstance(node.metadata, dict) else {}
|
|
|
|
|
node_task_id = _optional_text(node_metadata.get("last_task_id"))
|
|
|
|
|
if node_task_id == task_id and node.status in {"queued", "running", "cancelled"}:
|
|
|
|
|
cancellation = task.metadata.get("cancellation") if isinstance(task.metadata, dict) else None
|
|
|
|
|
cancellation_reason = (
|
|
|
|
|
_optional_text(cancellation.get("reason"))
|
|
|
|
|
if isinstance(cancellation, dict)
|
|
|
|
|
else None
|
|
|
|
|
)
|
|
|
|
|
node_update: dict[str, Any] = {
|
|
|
|
|
"metadata": {
|
|
|
|
|
**node_metadata,
|
|
|
|
|
"generation_cancellation": {
|
|
|
|
|
"task_id": task_id,
|
|
|
|
|
"reason": cancellation_reason or "cancelled",
|
|
|
|
|
"cleanup": cleanup,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
if node.status in {"queued", "running"}:
|
|
|
|
|
node_update["status"] = "cancelled"
|
|
|
|
|
node = self.workflow_service.update_node(
|
|
|
|
|
node_id,
|
|
|
|
|
node_update,
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
self._publish(
|
|
|
|
|
"generation.node_cancelled",
|
|
|
|
|
{
|
|
|
|
|
"workflow_id": workflow_id,
|
|
|
|
|
"node_id": node_id,
|
|
|
|
|
"task_id": task_id,
|
|
|
|
|
"node_status": node.status,
|
|
|
|
|
"cleanup": cleanup,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
return {
|
|
|
|
|
"task_id": task.task_id,
|
|
|
|
|
"task_status": task.status,
|
|
|
|
|
"node_status": node.status,
|
|
|
|
|
"images": [],
|
|
|
|
|
"cancelled": True,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def _cleanup_generation_outputs(self, *, assets: list[Any], images: list[Any]) -> dict[str, Any]:
|
|
|
|
|
removed_image_ids: list[str] = []
|
|
|
|
|
removed_uris: list[str] = []
|
|
|
|
|
errors: list[dict[str, str]] = []
|
|
|
|
|
for image in reversed(images):
|
|
|
|
|
image_id = _optional_text(getattr(image, "image_id", None))
|
|
|
|
|
if not image_id:
|
|
|
|
|
continue
|
|
|
|
|
try:
|
|
|
|
|
removed = self.image_service.image_repository.delete_metadata(image_id)
|
|
|
|
|
if removed is not None:
|
|
|
|
|
removed_image_ids.append(image_id)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
errors.append(
|
|
|
|
|
{
|
|
|
|
|
"target": image_id,
|
|
|
|
|
"error_type": exc.__class__.__name__,
|
|
|
|
|
"message": str(exc),
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
seen_uris: set[str] = set()
|
|
|
|
|
for asset in assets:
|
|
|
|
|
for field in ("image_uri", "thumbnail_uri"):
|
|
|
|
|
uri = _optional_text(getattr(asset, field, None))
|
|
|
|
|
if not uri or uri in seen_uris:
|
|
|
|
|
continue
|
|
|
|
|
seen_uris.add(uri)
|
|
|
|
|
try:
|
|
|
|
|
path = self.single_flow_engine.storage.resolve_uri(uri)
|
|
|
|
|
if path.exists() and path.is_file():
|
|
|
|
|
path.unlink()
|
|
|
|
|
removed_uris.append(uri)
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
errors.append(
|
|
|
|
|
{
|
|
|
|
|
"target": uri,
|
|
|
|
|
"error_type": exc.__class__.__name__,
|
|
|
|
|
"message": str(exc),
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
return {
|
|
|
|
|
"image_records_removed": removed_image_ids,
|
|
|
|
|
"files_removed": removed_uris,
|
|
|
|
|
"errors": errors,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def _record_generation_failure(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
task: GenerationTask,
|
|
|
|
|
node_id: str,
|
|
|
|
|
workflow_id: str,
|
|
|
|
|
route_resolution: dict[str, Any],
|
|
|
|
|
exc: Exception,
|
|
|
|
|
ctx: RequestContext,
|
|
|
|
|
) -> dict[str, Any]:
|
|
|
|
|
if isinstance(exc, ImageProviderError):
|
|
|
|
|
failure = provider_failure_attribution(exc, route_context=route_resolution, fallback_used=False)
|
|
|
|
|
else:
|
|
|
|
|
failure = {
|
|
|
|
|
"source": "generation_pipeline",
|
|
|
|
|
"category": "internal_error",
|
|
|
|
|
"error_type": exc.__class__.__name__,
|
|
|
|
|
"message": "generation pipeline failed",
|
|
|
|
|
"retryable": False,
|
|
|
|
|
"fallback_used": False,
|
|
|
|
|
}
|
|
|
|
|
failed_task = None
|
|
|
|
|
try:
|
|
|
|
|
failed_task = self.queue_service.update_task_if_status(
|
|
|
|
|
task.task_id,
|
|
|
|
|
{"queued", "running"},
|
|
|
|
|
{
|
|
|
|
|
"status": "failed",
|
|
|
|
|
"progress": 100,
|
|
|
|
|
"error_message": str(failure.get("message") or "generation failed"),
|
|
|
|
|
"metadata": {**task.metadata, "generation_failure": failure},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
pass
|
|
|
|
|
if failed_task is not None:
|
|
|
|
|
try:
|
|
|
|
|
node = self.workflow_service.get_node(node_id)
|
|
|
|
|
node_metadata = node.metadata if isinstance(node.metadata, dict) else {}
|
|
|
|
|
if (
|
|
|
|
|
_optional_text(node_metadata.get("last_task_id")) == task.task_id
|
|
|
|
|
and node.status in {"queued", "running"}
|
|
|
|
|
):
|
|
|
|
|
self.workflow_service.update_node(
|
|
|
|
|
node_id,
|
|
|
|
|
{
|
|
|
|
|
"status": "failed",
|
|
|
|
|
"metadata": {
|
|
|
|
|
**node_metadata,
|
|
|
|
|
"generation_failure": failure,
|
|
|
|
|
"last_task_id": task.task_id,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
ctx,
|
|
|
|
|
)
|
|
|
|
|
except Exception:
|
|
|
|
|
pass
|
|
|
|
|
self._publish(
|
|
|
|
|
"generation.node_failed",
|
|
|
|
|
{"workflow_id": workflow_id, "node_id": node_id, "task_id": task.task_id, "failure": failure},
|
|
|
|
|
)
|
|
|
|
|
return failure
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _retain_node_generation_lock(node_id: str) -> _NodeGenerationLock:
|
|
|
|
|
key = str(node_id)
|
|
|
|
|
with _NODE_GENERATION_LOCKS_GUARD:
|
|
|
|
|
entry = _NODE_GENERATION_LOCKS.get(key)
|
|
|
|
|
if entry is None:
|
|
|
|
|
entry = _NodeGenerationLock()
|
|
|
|
|
_NODE_GENERATION_LOCKS[key] = entry
|
|
|
|
|
entry.users += 1
|
|
|
|
|
return entry
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _release_node_generation_lock(node_id: str, entry: _NodeGenerationLock) -> None:
|
|
|
|
|
key = str(node_id)
|
|
|
|
|
with _NODE_GENERATION_LOCKS_GUARD:
|
|
|
|
|
entry.users -= 1
|
|
|
|
|
if entry.users == 0 and _NODE_GENERATION_LOCKS.get(key) is entry:
|
|
|
|
|
_NODE_GENERATION_LOCKS.pop(key, None)
|
|
|
|
|
|
2026-07-05 18:23:35 +08:00
|
|
|
|
|
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
if value is None:
|
|
|
|
|
return None
|
|
|
|
|
text = str(value).strip()
|
|
|
|
|
return text or None
|
2026-07-06 02:14:13 +08:00
|
|
|
|
|
|
|
|
|
2026-07-11 21:50:34 +08:00
|
|
|
_OMITTED_EPHEMERAL_REFERENCE = object()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _strip_ephemeral_image_data_urls(value: Any) -> tuple[Any, int]:
|
|
|
|
|
"""Remove inline image payloads before writing a generation request to storage."""
|
|
|
|
|
|
|
|
|
|
if isinstance(value, str):
|
|
|
|
|
if value.lstrip().lower().startswith("data:image/"):
|
|
|
|
|
return _OMITTED_EPHEMERAL_REFERENCE, 1
|
|
|
|
|
return value, 0
|
|
|
|
|
if isinstance(value, dict):
|
|
|
|
|
cleaned: dict[Any, Any] = {}
|
|
|
|
|
omitted = 0
|
|
|
|
|
for key, item in value.items():
|
|
|
|
|
prepared, item_omitted = _strip_ephemeral_image_data_urls(item)
|
|
|
|
|
omitted += item_omitted
|
|
|
|
|
if prepared is not _OMITTED_EPHEMERAL_REFERENCE:
|
|
|
|
|
cleaned[key] = prepared
|
|
|
|
|
return cleaned, omitted
|
|
|
|
|
if isinstance(value, (list, tuple)):
|
|
|
|
|
cleaned_items: list[Any] = []
|
|
|
|
|
omitted = 0
|
|
|
|
|
for item in value:
|
|
|
|
|
prepared, item_omitted = _strip_ephemeral_image_data_urls(item)
|
|
|
|
|
omitted += item_omitted
|
|
|
|
|
if prepared is not _OMITTED_EPHEMERAL_REFERENCE:
|
|
|
|
|
cleaned_items.append(prepared)
|
|
|
|
|
return cleaned_items, omitted
|
|
|
|
|
return value, 0
|
|
|
|
|
|
|
|
|
|
|
2026-07-06 02:14:13 +08:00
|
|
|
def _provider_failure_from_assets(metadata_items: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
|
|
|
for metadata in metadata_items:
|
|
|
|
|
failure = metadata.get("provider_error")
|
|
|
|
|
if isinstance(failure, dict):
|
|
|
|
|
return failure
|
|
|
|
|
return None
|
2026-07-06 02:40:18 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _planned_image_count(value: Any) -> int:
|
|
|
|
|
try:
|
|
|
|
|
count = int(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
count = 1
|
|
|
|
|
return min(max(count, 1), 8)
|