2026-07-15 23:01:07 +08:00
|
|
|
"""Read-only bridge to AstrBot's configured chat-completion providers."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import asyncio
|
2026-07-18 23:20:06 +08:00
|
|
|
import base64
|
2026-07-15 23:01:07 +08:00
|
|
|
import concurrent.futures
|
|
|
|
|
import json
|
|
|
|
|
import re
|
|
|
|
|
from typing import Any, Mapping
|
|
|
|
|
|
2026-07-18 23:20:06 +08:00
|
|
|
from core.image_formats import detect_image_format
|
|
|
|
|
from core.storage import VfsStorage
|
2026-07-15 23:01:07 +08:00
|
|
|
from engines.vision_provider import (
|
|
|
|
|
VisionProviderError,
|
|
|
|
|
VisionRecognitionRequest,
|
|
|
|
|
VisionRecognitionResult,
|
2026-07-18 23:20:06 +08:00
|
|
|
vision_image_urls,
|
2026-07-15 23:01:07 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_context: Any | None = None
|
|
|
|
|
_runtime_loop: asyncio.AbstractEventLoop | None = None
|
|
|
|
|
ASTRBOT_VISION_TIMEOUT_SECONDS = 270
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def configure_astrbot_provider_bridge(context: Any | None, loop: asyncio.AbstractEventLoop | None) -> None:
|
|
|
|
|
global _context, _runtime_loop
|
|
|
|
|
_context = context
|
|
|
|
|
_runtime_loop = loop
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def list_astrbot_text_models() -> list[dict[str, str]]:
|
|
|
|
|
getter = getattr(_context, "get_all_providers", None)
|
|
|
|
|
if not callable(getter):
|
|
|
|
|
return []
|
|
|
|
|
try:
|
|
|
|
|
providers = getter() or []
|
|
|
|
|
except Exception:
|
|
|
|
|
return []
|
|
|
|
|
result: list[dict[str, str]] = []
|
|
|
|
|
for provider in providers:
|
|
|
|
|
config = getattr(provider, "provider_config", None)
|
|
|
|
|
config = config if isinstance(config, Mapping) else {}
|
|
|
|
|
provider_id = str(config.get("id") or _meta_value(provider, "id") or "").strip()
|
|
|
|
|
if not provider_id:
|
|
|
|
|
continue
|
|
|
|
|
model = str(_call_text(provider, "get_model") or config.get("model") or _meta_value(provider, "model") or "").strip()
|
|
|
|
|
name = str(config.get("name") or config.get("display_name") or provider_id).strip()
|
|
|
|
|
result.append(
|
|
|
|
|
{
|
|
|
|
|
"route": f"astrbot:{provider_id}",
|
|
|
|
|
"provider_id": provider_id,
|
|
|
|
|
"name": name,
|
|
|
|
|
"model": model,
|
|
|
|
|
"modality": "text_generation",
|
|
|
|
|
"source": "astrbot",
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def astrbot_structured_text_callable(route_name: str):
|
|
|
|
|
route = str(route_name or "").strip()
|
|
|
|
|
if not route.startswith("astrbot:"):
|
|
|
|
|
return None
|
|
|
|
|
provider_id = route.split(":", 1)[1].strip()
|
|
|
|
|
provider = _provider_by_id(provider_id)
|
|
|
|
|
loop = _runtime_loop
|
|
|
|
|
if provider is None or loop is None or loop.is_closed():
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def complete(request_payload: dict[str, Any]) -> dict[str, Any] | str:
|
|
|
|
|
purpose = str(request_payload.get("purpose") or "structured prompt processing")
|
|
|
|
|
|
|
|
|
|
async def invoke() -> str:
|
|
|
|
|
response = await provider.text_chat(
|
|
|
|
|
prompt=json.dumps(request_payload, ensure_ascii=False),
|
|
|
|
|
system_prompt=(
|
|
|
|
|
f"Perform {purpose}. Return one JSON object only. Preserve the user's visual intent, "
|
|
|
|
|
"preserve safety-relevant meaning, and do not invent facts."
|
|
|
|
|
),
|
|
|
|
|
)
|
|
|
|
|
return str(getattr(response, "completion_text", "") or "").strip()
|
|
|
|
|
|
|
|
|
|
future = asyncio.run_coroutine_threadsafe(invoke(), loop)
|
|
|
|
|
text = future.result(timeout=180)
|
|
|
|
|
if not text:
|
|
|
|
|
raise ValueError("AstrBot text provider returned an empty response")
|
|
|
|
|
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE).strip()
|
|
|
|
|
try:
|
|
|
|
|
parsed = json.loads(cleaned)
|
|
|
|
|
except json.JSONDecodeError:
|
|
|
|
|
return text
|
|
|
|
|
return parsed if isinstance(parsed, dict) else text
|
|
|
|
|
|
|
|
|
|
return complete
|
|
|
|
|
|
|
|
|
|
|
2026-07-18 23:20:06 +08:00
|
|
|
def astrbot_structured_vision_callable(
|
|
|
|
|
route_name: str,
|
|
|
|
|
*,
|
|
|
|
|
storage: VfsStorage | None = None,
|
|
|
|
|
):
|
|
|
|
|
"""Return a generic structured-image reviewer backed by AstrBot.
|
|
|
|
|
|
|
|
|
|
Unlike :func:`astrbot_vision_provider`, this adapter does not require the
|
|
|
|
|
response to contain a material ``prompt`` field. It is used by generated
|
|
|
|
|
image moderation and other JSON-only vision checks.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
route = str(route_name or "").strip()
|
|
|
|
|
if not route.startswith("astrbot:"):
|
|
|
|
|
return None
|
|
|
|
|
provider_id = route.split(":", 1)[1].strip()
|
|
|
|
|
provider = _provider_by_id(provider_id)
|
|
|
|
|
loop = _runtime_loop
|
|
|
|
|
if provider is None or loop is None or loop.is_closed():
|
|
|
|
|
return None
|
|
|
|
|
vfs = storage or VfsStorage()
|
|
|
|
|
model = str(
|
|
|
|
|
_call_text(provider, "get_model")
|
|
|
|
|
or _provider_config(provider).get("model")
|
|
|
|
|
or _meta_value(provider, "model")
|
|
|
|
|
or ""
|
|
|
|
|
).strip()
|
|
|
|
|
|
|
|
|
|
def review(request_payload: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
image = request_payload.get("image") if isinstance(request_payload.get("image"), dict) else {}
|
|
|
|
|
uri = str(image.get("image_uri") or image.get("thumbnail_uri") or "").strip()
|
|
|
|
|
if uri.startswith("storage://"):
|
|
|
|
|
path = vfs.ensure_file(uri)
|
|
|
|
|
body = path.read_bytes()
|
|
|
|
|
if len(body) > 25 * 1024 * 1024:
|
|
|
|
|
raise ValueError("image moderation input exceeds size limit")
|
|
|
|
|
image_format = detect_image_format(body)
|
|
|
|
|
if image_format is None:
|
|
|
|
|
raise ValueError("image moderation input is not a supported image")
|
|
|
|
|
uri = f"data:{image_format.mime_type};base64,{base64.b64encode(body).decode('ascii')}"
|
|
|
|
|
if not uri.startswith(("data:image/", "http://", "https://")):
|
|
|
|
|
raise ValueError("image moderation requires a storage, data, or HTTP image URI")
|
|
|
|
|
|
|
|
|
|
async def invoke() -> str:
|
|
|
|
|
response = await provider.text_chat(
|
|
|
|
|
prompt=json.dumps(request_payload, ensure_ascii=False),
|
|
|
|
|
image_urls=[uri],
|
|
|
|
|
system_prompt=(
|
|
|
|
|
"Review the attached image and return one JSON object only. Follow the requested actions and "
|
|
|
|
|
"requirements exactly. Detect age ambiguity, unexpected nudity, terrorism, high-risk violence, "
|
|
|
|
|
"extreme gore, and content beyond the allowed request. Never weaken a safety classification."
|
|
|
|
|
),
|
|
|
|
|
model=model or None,
|
|
|
|
|
)
|
|
|
|
|
return str(getattr(response, "completion_text", "") or "").strip()
|
|
|
|
|
|
|
|
|
|
future = asyncio.run_coroutine_threadsafe(invoke(), loop)
|
|
|
|
|
try:
|
|
|
|
|
text = future.result(timeout=ASTRBOT_VISION_TIMEOUT_SECONDS)
|
|
|
|
|
except concurrent.futures.TimeoutError as exc:
|
|
|
|
|
future.cancel()
|
|
|
|
|
raise RuntimeError("AstrBot structured vision provider timed out") from exc
|
|
|
|
|
if not text:
|
|
|
|
|
raise ValueError("AstrBot structured vision provider returned an empty response")
|
|
|
|
|
return _json_object_from_text(text)
|
|
|
|
|
|
|
|
|
|
return review
|
|
|
|
|
|
|
|
|
|
|
2026-07-15 23:01:07 +08:00
|
|
|
def astrbot_vision_provider(route_name: str):
|
|
|
|
|
"""Return a synchronous material-vision adapter backed by AstrBot.
|
|
|
|
|
|
|
|
|
|
ImageStudio's material service runs in AstrBot's web worker while provider
|
|
|
|
|
clients belong to AstrBot's main asyncio loop. The adapter keeps that
|
|
|
|
|
boundary explicit and uses AstrBot's public ``text_chat(image_urls=...)``
|
|
|
|
|
contract, so credentials remain owned by AstrBot rather than being copied
|
|
|
|
|
into the plugin configuration.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
route = str(route_name or "").strip()
|
|
|
|
|
if not route.startswith("astrbot:"):
|
|
|
|
|
return None
|
|
|
|
|
provider_id = route.split(":", 1)[1].strip()
|
|
|
|
|
provider = _provider_by_id(provider_id)
|
|
|
|
|
loop = _runtime_loop
|
|
|
|
|
if provider is None or loop is None or loop.is_closed():
|
|
|
|
|
return None
|
|
|
|
|
model = str(
|
|
|
|
|
_call_text(provider, "get_model")
|
|
|
|
|
or _provider_config(provider).get("model")
|
|
|
|
|
or _meta_value(provider, "model")
|
|
|
|
|
or ""
|
|
|
|
|
).strip()
|
|
|
|
|
return _AstrBotVisionProvider(
|
|
|
|
|
provider=provider,
|
|
|
|
|
provider_id=provider_id,
|
|
|
|
|
model=model,
|
|
|
|
|
loop=loop,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class _AstrBotVisionProvider:
|
|
|
|
|
def __init__(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
provider: Any,
|
|
|
|
|
provider_id: str,
|
|
|
|
|
model: str,
|
|
|
|
|
loop: asyncio.AbstractEventLoop,
|
|
|
|
|
) -> None:
|
|
|
|
|
self.provider = provider
|
|
|
|
|
self.provider_id = provider_id
|
|
|
|
|
self.model = model
|
|
|
|
|
self.loop = loop
|
|
|
|
|
|
|
|
|
|
def recognize_material(self, request: VisionRecognitionRequest) -> VisionRecognitionResult:
|
2026-07-18 23:20:06 +08:00
|
|
|
image_urls = list(vision_image_urls(request))
|
|
|
|
|
if not image_urls:
|
2026-07-15 23:01:07 +08:00
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision input must be a data or HTTP image URL",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="invalid_request",
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
async def invoke() -> str:
|
|
|
|
|
response = await self.provider.text_chat(
|
|
|
|
|
prompt=_material_user_prompt(request),
|
2026-07-18 23:20:06 +08:00
|
|
|
image_urls=image_urls,
|
2026-07-15 23:01:07 +08:00
|
|
|
system_prompt=_material_system_prompt(request),
|
|
|
|
|
model=request.model or self.model or None,
|
|
|
|
|
)
|
|
|
|
|
return str(getattr(response, "completion_text", "") or "").strip()
|
|
|
|
|
|
|
|
|
|
future = asyncio.run_coroutine_threadsafe(invoke(), self.loop)
|
|
|
|
|
try:
|
|
|
|
|
text = future.result(timeout=ASTRBOT_VISION_TIMEOUT_SECONDS)
|
|
|
|
|
except concurrent.futures.TimeoutError as exc:
|
|
|
|
|
future.cancel()
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision provider timed out",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="timeout",
|
|
|
|
|
) from exc
|
|
|
|
|
except Exception as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision provider request failed",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="provider_error",
|
|
|
|
|
details={"reason": _safe_error_reason(exc)},
|
|
|
|
|
) from exc
|
|
|
|
|
if not text:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision provider returned an empty response",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
)
|
|
|
|
|
try:
|
|
|
|
|
payload = _json_object_from_text(text)
|
|
|
|
|
except (json.JSONDecodeError, ValueError) as exc:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision provider returned invalid JSON",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"reason": str(exc), "content_preview": text[:500]},
|
|
|
|
|
) from exc
|
|
|
|
|
prompt = str(payload.get("prompt") or payload.get("positive_prompt") or payload.get("description") or "").strip()
|
|
|
|
|
if not prompt:
|
|
|
|
|
raise VisionProviderError(
|
|
|
|
|
"AstrBot vision response must include prompt or description",
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
error_type="invalid_response",
|
|
|
|
|
details={"structured_keys": sorted(str(key) for key in payload)},
|
|
|
|
|
)
|
|
|
|
|
return VisionRecognitionResult(
|
|
|
|
|
provider_id=self.provider_id,
|
|
|
|
|
model=request.model or self.model or "astrbot-provider-model",
|
|
|
|
|
prompt=prompt,
|
|
|
|
|
negative_prompt=_optional_text(payload.get("negative_prompt")),
|
|
|
|
|
tags=_string_list(payload.get("tags")),
|
|
|
|
|
confidence=_optional_confidence(payload.get("confidence")),
|
|
|
|
|
description=_optional_text(payload.get("description")),
|
|
|
|
|
structured_data=payload,
|
|
|
|
|
raw_metadata={
|
|
|
|
|
"source": "astrbot_provider_bridge",
|
|
|
|
|
"structured_keys": sorted(str(key) for key in payload),
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _provider_by_id(provider_id: str) -> Any | None:
|
|
|
|
|
getter = getattr(_context, "get_provider_by_id", None)
|
|
|
|
|
if callable(getter):
|
|
|
|
|
try:
|
|
|
|
|
return getter(provider_id)
|
|
|
|
|
except Exception:
|
|
|
|
|
return None
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _provider_config(provider: Any) -> Mapping[str, Any]:
|
|
|
|
|
config = getattr(provider, "provider_config", None)
|
|
|
|
|
return config if isinstance(config, Mapping) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _material_system_prompt(request: VisionRecognitionRequest) -> str:
|
|
|
|
|
configured = str(request.metadata.get("recognition_instruction") or "").strip()
|
|
|
|
|
return configured or (
|
|
|
|
|
"Analyze the visible reference material exhaustively and return one JSON object only. "
|
|
|
|
|
"Extract visible facts without presenting guesses as observations. Include prompt, negative_prompt, tags, "
|
|
|
|
|
"confidence, description, raw_analysis.attributes, normalized_asset_profile and optional generation_capsules. "
|
|
|
|
|
"Each raw attribute should include value, confidence and source_state."
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _material_user_prompt(request: VisionRecognitionRequest) -> str:
|
|
|
|
|
return (
|
|
|
|
|
f"Material name: {request.display_name}\n"
|
|
|
|
|
f"Known prompt: {request.prompt or ''}\n"
|
|
|
|
|
f"Known negative prompt: {request.negative_prompt or ''}\n"
|
|
|
|
|
f"Known tags: {', '.join(request.tags)}\n"
|
|
|
|
|
"Inspect the attached image and return the requested structured material record."
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _json_object_from_text(value: str) -> dict[str, Any]:
|
|
|
|
|
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", value.strip(), flags=re.IGNORECASE).strip()
|
|
|
|
|
candidates = [cleaned]
|
|
|
|
|
start = cleaned.find("{")
|
|
|
|
|
end = cleaned.rfind("}")
|
|
|
|
|
if start >= 0 and end > start:
|
|
|
|
|
candidates.append(cleaned[start : end + 1])
|
|
|
|
|
last_error: json.JSONDecodeError | None = None
|
|
|
|
|
for candidate in candidates:
|
|
|
|
|
try:
|
|
|
|
|
parsed = json.loads(re.sub(r",\s*([}\]])", r"\1", candidate))
|
|
|
|
|
except json.JSONDecodeError as exc:
|
|
|
|
|
last_error = exc
|
|
|
|
|
continue
|
|
|
|
|
if isinstance(parsed, dict):
|
|
|
|
|
return parsed
|
|
|
|
|
raise ValueError("vision response JSON must be an object")
|
|
|
|
|
raise last_error or ValueError("vision response did not contain a JSON object")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _string_list(value: Any) -> list[str]:
|
|
|
|
|
values = value.split(",") if isinstance(value, str) else value if isinstance(value, list) else []
|
|
|
|
|
return sorted({str(item).strip() for item in values if str(item).strip()})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _optional_text(value: Any) -> str | None:
|
|
|
|
|
text = str(value or "").strip()
|
|
|
|
|
return text or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _optional_confidence(value: Any) -> float | None:
|
|
|
|
|
try:
|
|
|
|
|
confidence = float(value)
|
|
|
|
|
except (TypeError, ValueError):
|
|
|
|
|
return None
|
|
|
|
|
return min(max(confidence, 0.0), 1.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _safe_error_reason(exc: Exception) -> str:
|
|
|
|
|
reason = str(exc).strip() or exc.__class__.__name__
|
|
|
|
|
return reason[:500]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _meta_value(provider: Any, key: str) -> Any:
|
|
|
|
|
meta = getattr(provider, "meta", None)
|
|
|
|
|
if not callable(meta):
|
|
|
|
|
return None
|
|
|
|
|
try:
|
|
|
|
|
return getattr(meta(), key, None)
|
|
|
|
|
except Exception:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _call_text(provider: Any, method_name: str) -> str:
|
|
|
|
|
method = getattr(provider, method_name, None)
|
|
|
|
|
if not callable(method):
|
|
|
|
|
return ""
|
|
|
|
|
try:
|
|
|
|
|
return str(method() or "")
|
|
|
|
|
except Exception:
|
|
|
|
|
return ""
|