astrbot_plugin_imagestudio/backend/services/model_provider_runtime.py

406 行
20 KiB
Python

2026-07-06 02:08:07 +08:00
"""Runtime helpers for building provider clients from persisted routes."""
from __future__ import annotations
2026-07-12 02:54:45 +08:00
import base64
import json
import urllib.request
2026-07-06 02:08:07 +08:00
from typing import Any
from urllib.parse import urlsplit
2026-07-06 02:08:07 +08:00
from backend.services.model_provider_service import ModelProviderService
from core.model_route_chain import model_route_candidates
2026-07-06 02:08:07 +08:00
from core.secret_service import SecretStore, resolve_secret_value
2026-07-12 02:54:45 +08:00
from core.remote_fetch import read_http_response_limited, validate_http_url
from core.image_formats import detect_image_format
from core.storage import VfsStorage
from database.db_helper import load_config_payload
from engines.image_provider import FallbackImageProvider, ImageProvider, ImageProviderConfig, OpenAIImageProvider
from engines.vision_provider import FallbackVisionProvider, OpenAIVisionProvider, VisionProvider, VisionProviderConfig
2026-07-06 02:08:07 +08:00
class ModelProviderRuntime:
def __init__(
self,
model_provider_service: ModelProviderService,
*,
secret_store: SecretStore | None = None,
):
self.model_provider_service = model_provider_service
self.secret_store = secret_store
def image_provider_for_route(self, route_context: dict[str, Any]) -> ImageProvider | None:
route_name = str(route_context.get("route_name") or "").strip()
if not route_name:
return None
candidates: list[dict[str, Any]] = []
seen: set[str] = set()
for candidate_name in model_route_candidates(route_name):
if candidate_name.startswith("astrbot:"):
continue
candidate = self.model_provider_service.resolve_route(candidate_name, modality="image_generation")
if candidate and str(candidate.get("route_id")) not in seen:
candidates.append(candidate)
seen.add(str(candidate.get("route_id")))
if not candidates:
2026-07-06 02:08:07 +08:00
return None
resolved = candidates[0]
route_settings = resolved.get("settings") if isinstance(resolved.get("settings"), dict) else {}
# A model selected in the UI must only call its own provider. Generic
# cross-provider fallback can silently switch models and used to hide
# the selected provider's real error. Safety routing is handled by the
# generation governance layer instead.
fallback_enabled = len(candidates) > 1 or _configured_bool(
route_context.get("allow_provider_fallback"),
route_settings.get("allow_provider_fallback"),
default=False,
)
if fallback_enabled:
2026-07-12 02:54:45 +08:00
excluded_routes = {str(item).strip() for item in route_context.get("excluded_fallback_routes", []) if str(item).strip()} if isinstance(route_context.get("excluded_fallback_routes"), list) else set()
fallback_routes = sorted(
self.model_provider_service.list_routes(enabled=True),
key=lambda item: (int(item.get("priority") or 100), str(item.get("route_id") or "")),
)
for route in fallback_routes:
2026-07-12 02:54:45 +08:00
if str(route.get("route_name") or "") in excluded_routes:
continue
if route.get("modality") != "image_generation" or str(route.get("route_id")) in seen:
continue
candidate = self.model_provider_service.resolve_route(str(route.get("route_name")), modality="image_generation")
if candidate:
candidates.append(candidate); seen.add(str(candidate.get("route_id")))
providers = [provider for candidate in candidates if (provider := self._image_provider_from_resolved(candidate))]
if not providers:
return None
return providers[0] if len(providers) == 1 else FallbackImageProvider(providers)
def _image_provider_from_resolved(self, resolved: dict[str, Any]) -> ImageProvider | None:
2026-07-06 02:08:07 +08:00
provider = resolved.get("provider") or {}
if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible":
return None
secret = resolve_secret_value(
env_var=provider.get("api_key_env_var"),
secret_ref=provider.get("api_key_secret_ref"),
plaintext=_saved_provider_api_key(str(resolved.get("provider_id") or "")),
2026-07-06 02:08:07 +08:00
secret_store=self.secret_store,
)
secret_configured = bool(provider.get("api_key_env_var") or provider.get("api_key_secret_ref"))
if secret_configured and not secret.value:
2026-07-06 02:08:07 +08:00
return None
settings = dict(resolved.get("settings") or {})
capabilities = dict(provider.get("capabilities") or {})
base_url = str(provider.get("base_url") or "")
2026-07-06 02:08:07 +08:00
return OpenAIImageProvider(
ImageProviderConfig(
provider_id=str(resolved["provider_id"]),
base_url=base_url,
api_key=secret.value or "",
2026-07-06 02:08:07 +08:00
image_model=str(resolved["model_id"]),
image_endpoint=str(provider.get("image_endpoint") or "/v1/images/generations"),
provider_type=str(provider.get("provider_type") or "openai_compatible"),
timeout_seconds=int(provider.get("timeout_seconds") or 180),
extra_headers=dict(provider.get("extra_headers") or {}),
supports_negative_prompt=_configured_bool(
settings.get("supports_negative_prompt"),
capabilities.get("supports_negative_prompt"),
capabilities.get("negative_prompt"),
default=_default_negative_prompt_support(base_url),
),
2026-07-06 02:08:07 +08:00
)
)
2026-07-06 03:37:17 +08:00
def vision_provider_for_route(self, route_context: dict[str, Any]) -> VisionProvider | None:
route_name = str(route_context.get("route_name") or "").strip()
if not route_name:
return None
providers = [
provider
for candidate in model_route_candidates(route_name)
if (provider := self._vision_provider_for_single_route(candidate)) is not None
]
if not providers:
return None
return providers[0] if len(providers) == 1 else FallbackVisionProvider(providers)
def _vision_provider_for_single_route(self, route_name: str) -> VisionProvider | None:
if route_name.startswith("astrbot:"):
# AstrBot owns this provider's credentials and runtime client. Do
# not mirror those secrets into ImageStudio's provider tables.
from core.astrbot_provider_bridge import astrbot_vision_provider
return astrbot_vision_provider(route_name)
2026-07-06 03:37:17 +08:00
resolved = self.model_provider_service.resolve_route(route_name, modality="vision_recognition")
if not resolved:
return None
provider = resolved.get("provider") or {}
if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible":
return None
secret = resolve_secret_value(
env_var=provider.get("api_key_env_var"),
secret_ref=provider.get("api_key_secret_ref"),
plaintext=_saved_provider_api_key(str(resolved.get("provider_id") or "")),
2026-07-06 03:37:17 +08:00
secret_store=self.secret_store,
)
secret_configured = bool(provider.get("api_key_env_var") or provider.get("api_key_secret_ref"))
if secret_configured and not secret.value:
return None
base_url = str(provider.get("base_url") or "").strip()
if not base_url:
2026-07-06 03:37:17 +08:00
return None
settings = dict(resolved.get("settings") or {})
capabilities = dict(provider.get("capabilities") or {})
return OpenAIVisionProvider(
VisionProviderConfig(
provider_id=str(resolved["provider_id"]),
base_url=base_url,
api_key=secret.value or "",
2026-07-06 03:37:17 +08:00
vision_model=str(resolved["model_id"]),
vision_endpoint=str(
settings.get("vision_endpoint")
or capabilities.get("vision_endpoint")
or "/v1/chat/completions"
),
provider_type=str(provider.get("provider_type") or "openai_compatible"),
timeout_seconds=int(provider.get("timeout_seconds") or 180),
extra_headers=dict(provider.get("extra_headers") or {}),
)
)
2026-07-12 02:54:45 +08:00
def structured_text_callable(self, route_name: str):
"""Return an OpenAI-compatible JSON completion callable for a configured route."""
callables = [
completion
for candidate in model_route_candidates(route_name)
if (completion := self._structured_text_callable_for_single_route(candidate)) is not None
]
if not callables:
return None
if len(callables) == 1:
return callables[0]
def complete_with_fallback(request_payload: dict[str, Any]) -> dict[str, Any] | str:
failures: list[str] = []
for completion in callables:
try:
return completion(request_payload)
except Exception as exc:
failures.append(str(exc)[:500])
raise RuntimeError(f"all configured text routes failed: {' | '.join(failures)}")
return complete_with_fallback
def _structured_text_callable_for_single_route(self, route_name: str):
if route_name.startswith("astrbot:"):
from core.astrbot_provider_bridge import astrbot_structured_text_callable
return astrbot_structured_text_callable(route_name)
2026-07-12 02:54:45 +08:00
resolved = self.model_provider_service.resolve_route(route_name)
if not resolved:
return None
provider = resolved.get("provider") or {}
if str(provider.get("provider_type") or "openai_compatible") != "openai_compatible":
return None
secret = resolve_secret_value(
env_var=provider.get("api_key_env_var"),
secret_ref=provider.get("api_key_secret_ref"),
plaintext=_saved_provider_api_key(str(resolved.get("provider_id") or "")),
2026-07-12 02:54:45 +08:00
secret_store=self.secret_store,
)
if (provider.get("api_key_env_var") or provider.get("api_key_secret_ref")) and not secret.value:
return None
base_url = str(provider.get("base_url") or "").rstrip("/")
if not base_url:
return None
settings = dict(resolved.get("settings") or {})
capabilities = dict(provider.get("capabilities") or {})
endpoint = str(settings.get("chat_endpoint") or capabilities.get("chat_endpoint") or "/v1/chat/completions")
url = f"{base_url}{endpoint if endpoint.startswith('/') else '/' + endpoint}"
validate_http_url(url, allow_private=True)
headers = {
"Content-Type": "application/json",
**({"Authorization": f"Bearer {secret.value}"} if secret.value else {}),
**dict(provider.get("extra_headers") or {}),
}
timeout = int(provider.get("timeout_seconds") or 180)
model = str(resolved.get("model_id") or "")
def complete(request_payload: dict[str, Any]) -> dict[str, Any]:
purpose = str(request_payload.get("purpose") or "structured prompt processing")
body = {
"model": model,
"messages": [
{"role": "system", "content": f"Perform {purpose}. Return one JSON object only. Preserve safety-relevant meaning and do not invent facts."},
{"role": "user", "content": json.dumps(request_payload, ensure_ascii=False)},
],
"temperature": 0.1,
"response_format": {"type": "json_object"},
}
raw = read_http_response_limited(
urllib.request.Request(url, data=json.dumps(body, ensure_ascii=False).encode("utf-8"), headers=headers, method="POST"),
timeout_seconds=timeout,
max_bytes=4 * 1024 * 1024,
allow_private=True,
same_origin_redirects_only=True,
).decode("utf-8-sig")
response = json.loads(raw)
content = _completion_content(response)
parsed = json.loads(content)
if not isinstance(parsed, dict):
raise ValueError("structured completion must return an object")
return parsed
return complete
def structured_vision_callable(self, route_name: str, *, storage: VfsStorage | None = None):
callables = [
review
for candidate in model_route_candidates(route_name)
if (review := self._structured_vision_callable_for_single_route(candidate, storage=storage)) is not None
]
if not callables:
return None
if len(callables) == 1:
return callables[0]
def review_with_fallback(request_payload: dict[str, Any]) -> dict[str, Any]:
failures: list[str] = []
for review in callables:
try:
return review(request_payload)
except Exception as exc:
failures.append(str(exc)[:500])
raise RuntimeError(f"all configured vision review routes failed: {' | '.join(failures)}")
return review_with_fallback
def _structured_vision_callable_for_single_route(
self,
route_name: str,
*,
storage: VfsStorage | None = None,
):
if route_name.startswith("astrbot:"):
from core.astrbot_provider_bridge import astrbot_structured_vision_callable
return astrbot_structured_vision_callable(route_name, storage=storage)
2026-07-12 02:54:45 +08:00
resolved = self.model_provider_service.resolve_route(route_name)
if not resolved:
return None
provider = resolved.get("provider") or {}
secret = resolve_secret_value(
env_var=provider.get("api_key_env_var"),
secret_ref=provider.get("api_key_secret_ref"),
plaintext=_saved_provider_api_key(str(resolved.get("provider_id") or "")),
secret_store=self.secret_store,
)
2026-07-12 02:54:45 +08:00
if (provider.get("api_key_env_var") or provider.get("api_key_secret_ref")) and not secret.value:
return None
base_url = str(provider.get("base_url") or "").rstrip("/")
if not base_url:
return None
settings = dict(resolved.get("settings") or {})
capabilities = dict(provider.get("capabilities") or {})
endpoint = str(settings.get("vision_endpoint") or capabilities.get("vision_endpoint") or settings.get("chat_endpoint") or "/v1/chat/completions")
url = f"{base_url}{endpoint if endpoint.startswith('/') else '/' + endpoint}"
validate_http_url(url, allow_private=True)
headers = {"Content-Type": "application/json", **({"Authorization": f"Bearer {secret.value}"} if secret.value else {}), **dict(provider.get("extra_headers") or {})}
timeout = int(provider.get("timeout_seconds") or 180)
model = str(resolved.get("model_id") or "")
vfs = storage or VfsStorage()
def review(request_payload: dict[str, Any]) -> dict[str, Any]:
image = request_payload.get("image") if isinstance(request_payload.get("image"), dict) else {}
uri = str(image.get("image_uri") or image.get("thumbnail_uri") or "").strip()
if uri.startswith("storage://"):
path = vfs.ensure_file(uri)
body = path.read_bytes()
if len(body) > 25 * 1024 * 1024:
raise ValueError("image moderation input exceeds size limit")
image_format = detect_image_format(body)
if image_format is None:
raise ValueError("image moderation input is not a supported image")
uri = f"data:{image_format.mime_type};base64,{base64.b64encode(body).decode('ascii')}"
if not uri.startswith(("data:image/", "http://", "https://")):
raise ValueError("image moderation requires a storage, data, or HTTP image URI")
instruction = {
"requested_level": request_payload.get("requested_level"),
"route_kind": request_payload.get("route_kind"),
"reference_binding_mapping": (image.get("metadata") or {}).get("reference_binding_mapping") if isinstance(image.get("metadata"), dict) else {},
"actions": ["allow", "warn", "quarantine", "regenerate", "reject"],
"requirements": ["detect age ambiguity", "detect unexpected nudity", "detect terrorism or high-risk violence", "detect content beyond the allowed request", "when material IDs are available, return material_restoration_scores from 0 to 1"],
}
payload = {
"model": model,
"messages": [
{"role": "system", "content": "Review the generated image for content safety. Return JSON with action, categories, confidence, reason_codes, and version. Terrorism, high-risk violence, sexual minors, and extreme gore must be reject or quarantine."},
{"role": "user", "content": [{"type": "text", "text": json.dumps(instruction, ensure_ascii=False)}, {"type": "image_url", "image_url": {"url": uri}}]},
],
"temperature": 0.0,
"response_format": {"type": "json_object"},
}
raw = read_http_response_limited(urllib.request.Request(url, data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), headers=headers, method="POST"), timeout_seconds=timeout, max_bytes=4 * 1024 * 1024, allow_private=True, same_origin_redirects_only=True).decode("utf-8-sig")
parsed = json.loads(_completion_content(json.loads(raw)))
if not isinstance(parsed, dict):
raise ValueError("image moderation response must be an object")
return parsed
return review
def _saved_provider_api_key(provider_id: str) -> str | None:
"""Read the UI-saved adapter credential without copying it into the DB."""
if not provider_id:
return None
config = load_config_payload()
generation = config.get("generation") if isinstance(config.get("generation"), dict) else {}
adapters = generation.get("adapters") if isinstance(generation.get("adapters"), list) else []
candidates = adapters
if not candidates and isinstance(generation.get("provider"), dict):
candidates = [generation["provider"]]
for candidate in candidates:
if not isinstance(candidate, dict):
continue
candidate_id = str(candidate.get("id") or candidate.get("provider_id") or "").strip()
if candidate_id != provider_id:
continue
value = str(candidate.get("api_key_plaintext") or "").strip()
if value and value != "***MASKED***":
return value
return None
def _configured_bool(*values: Any, default: bool) -> bool:
for value in values:
if value is None:
continue
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on", "enabled"}:
return True
if normalized in {"0", "false", "no", "off", "disabled"}:
return False
return bool(value)
return default
def _default_negative_prompt_support(base_url: str) -> bool:
try:
hostname = str(urlsplit(base_url).hostname or "").lower()
except ValueError:
return True
return not (hostname == "api.openai.com" or hostname.endswith(".openai.azure.com"))
2026-07-12 02:54:45 +08:00
def _completion_content(response: dict[str, Any]) -> str:
if isinstance(response.get("output_text"), str):
return response["output_text"].strip()
choices = response.get("choices")
if isinstance(choices, list) and choices and isinstance(choices[0], dict):
message = choices[0].get("message") if isinstance(choices[0].get("message"), dict) else {}
content = message.get("content")
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
return " ".join(str(item.get("text") or "") for item in content if isinstance(item, dict)).strip()
raise ValueError("structured completion response has no text content")