From aa94d7c2ce5c6a2dde7fd3a0e440b3b3e3bdf929 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Sat, 11 Jul 2026 23:48:36 +0800 Subject: [PATCH] feat: add model routing and recognition prompt settings --- _conf_schema.json | 12 ++ backend/routes/material_routes.py | 5 +- backend/routes/node_routes.py | 2 + backend/services/generation_service.py | 24 ++++ backend/services/material_service.py | 26 +++- backend/services/settings_service.py | 27 +++++ config.yaml | 23 ++++ engines/vision_provider.py | 17 ++- .../settings/ModelAssignmentsPanel.vue | 113 ++++++++++++++++++ .../settings/RecognitionPromptsPanel.vue | 62 ++++++++++ .../src/features/settings/SettingsPage.vue | 15 ++- main.py | 31 +++++ pages/imagestudio/assets/index-BKG3cNQ-.js | 51 -------- pages/imagestudio/assets/index-DoRMtpTr.css | 1 + pages/imagestudio/assets/index-Dx9JF8Lp.css | 1 - pages/imagestudio/assets/index-S8PI1-k-.js | 51 ++++++++ pages/imagestudio/index.html | 4 +- tests/test_model_assignments.py | 43 +++++++ web/assets/index-BZytriHv.css | 1 - web/assets/index-Cm1uzzf8.js | 51 -------- web/assets/index-D1gSt_ux.css | 1 + web/assets/index-DyWLLdwX.js | 51 ++++++++ web/index.html | 4 +- 23 files changed, 499 insertions(+), 117 deletions(-) create mode 100644 frontend/standalone-web/src/features/settings/ModelAssignmentsPanel.vue create mode 100644 frontend/standalone-web/src/features/settings/RecognitionPromptsPanel.vue delete mode 100644 pages/imagestudio/assets/index-BKG3cNQ-.js create mode 100644 pages/imagestudio/assets/index-DoRMtpTr.css delete mode 100644 pages/imagestudio/assets/index-Dx9JF8Lp.css create mode 100644 pages/imagestudio/assets/index-S8PI1-k-.js create mode 100644 tests/test_model_assignments.py delete mode 100644 web/assets/index-BZytriHv.css delete mode 100644 web/assets/index-Cm1uzzf8.js create mode 100644 web/assets/index-D1gSt_ux.css create mode 100644 web/assets/index-DyWLLdwX.js diff --git a/_conf_schema.json b/_conf_schema.json index d4a8128..3f87ef5 100644 --- a/_conf_schema.json +++ b/_conf_schema.json @@ -79,6 +79,18 @@ "type": "bool", "default": true }, + "model_material_recognition": {"description":"素材导入解析使用的 Vision 路由。","type":"string","default":"material_vlm"}, + "model_profile_parse": {"description":"外部角色导入解析使用的文本模型路由。","type":"string","default":""}, + "model_persona_split": {"description":"角色设定拆解使用的文本模型路由。","type":"string","default":""}, + "model_prompt_planning": {"description":"提示词规划与扩写使用的文本模型路由。","type":"string","default":""}, + "model_cross_review_primary": {"description":"候选图交叉审核主 Vision 路由。","type":"string","default":"material_vlm"}, + "model_cross_review_secondary": {"description":"候选图交叉审核复核 Vision 路由。","type":"string","default":"material_vlm"}, + "model_quick_generation": {"description":"快速生图使用的模型路由。","type":"string","default":""}, + "model_portrait_generation": {"description":"头像草稿使用的生图路由。","type":"string","default":""}, + "model_halfbody_generation": {"description":"半身角色图使用的生图路由。","type":"string","default":""}, + "model_fullbody_generation": {"description":"全身角色图使用的生图路由。","type":"string","default":""}, + "model_sheet_generation": {"description":"设定图与姿势图使用的生图路由。","type":"string","default":""}, + "model_hires_fix": {"description":"高清修复使用的生图路由。","type":"string","default":""}, "enable_private_companion_integration": { "description": "允许 astrbot_plugin_private_companion 调用 ImageStudio 的穿搭和素材集成接口。", "type": "bool", diff --git a/backend/routes/material_routes.py b/backend/routes/material_routes.py index 7e47ff3..e63d2b0 100644 --- a/backend/routes/material_routes.py +++ b/backend/routes/material_routes.py @@ -238,12 +238,15 @@ def _create_material_service() -> MaterialService: sync_vision_provider_from_config(model_provider_service, config) vision = config.get("vision") if isinstance(config.get("vision"), dict) else {} provider = vision.get("provider") if isinstance(vision.get("provider"), dict) else {} - default_route = str(provider.get("route_name") or vision.get("default_route") or "material_vlm").strip() + assignments = config.get("model_assignments") if isinstance(config.get("model_assignments"), dict) else {} + recognition_prompts = config.get("recognition_prompts") if isinstance(config.get("recognition_prompts"), dict) else {} + default_route = str(assignments.get("material_recognition") or provider.get("route_name") or vision.get("default_route") or "material_vlm").strip() return MaterialService( repository=MaterialRepository(database), audit_service=AuditService(database), vision_provider_factory=model_provider_runtime.vision_provider_for_route, default_recognition_route=default_route, + recognition_prompt_config=recognition_prompts, ) diff --git a/backend/routes/node_routes.py b/backend/routes/node_routes.py index 4781071..bd579cb 100644 --- a/backend/routes/node_routes.py +++ b/backend/routes/node_routes.py @@ -294,6 +294,7 @@ def _create_generation_service() -> GenerationService: config_payload = load_config_payload() sync_generation_provider_from_config(model_provider_service, config_payload) generation_config = config_payload.get("generation") if isinstance(config_payload.get("generation"), dict) else {} + model_assignments = config_payload.get("model_assignments") if isinstance(config_payload.get("model_assignments"), dict) else {} return GenerationService( workflow_service=WorkflowService( workflow_repository=workflow_repository, @@ -322,6 +323,7 @@ def _create_generation_service() -> GenerationService: default_model_route=str(generation_config.get("default_model") or "").strip() or None, require_resolved_model_route=True, material_resolver=MaterialRepository(database), + model_assignments=model_assignments, ) diff --git a/backend/services/generation_service.py b/backend/services/generation_service.py index a3af309..1f33048 100644 --- a/backend/services/generation_service.py +++ b/backend/services/generation_service.py @@ -51,6 +51,7 @@ class GenerationService: default_model_route: str | None = None, require_resolved_model_route: bool = False, material_resolver: MaterialResolver | None = None, + model_assignments: dict[str, Any] | None = None, ): self.workflow_service = workflow_service self.queue_service = queue_service @@ -66,6 +67,7 @@ class GenerationService: self.prompt_expansion_service = prompt_expansion_service self.event_bus = event_bus self.default_model_route = _optional_text(default_model_route) + self.model_assignments = dict(model_assignments or {}) self.require_resolved_model_route = require_resolved_model_route def generate_node(self, node_id: str, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]: @@ -117,9 +119,11 @@ class GenerationService: ) profile = self.workflow_service.profile_repository.get(workflow.profile_id) profile_model_route = _optional_text(getattr(profile, "default_model_route", None)) + assigned_route = _optional_text(self.model_assignments.get(_assignment_key(node))) model_route = str( payload.get("model_route") or node.model_route + or assigned_route or profile_model_route or self.default_model_route or "image_low" @@ -749,6 +753,26 @@ def _release_node_generation_lock(node_id: str, entry: _NodeGenerationLock) -> N _NODE_GENERATION_LOCKS.pop(key, None) +def _assignment_key(node: Any) -> str: + node_key = str(getattr(node, "node_key", "") or "").strip() + node_type = str(getattr(node, "node_type", "") or "").strip() + mapping = { + "headshot": "portrait_generation", + "half_body": "halfbody_generation", + "full_body": "fullbody_generation", + "expression_sheet": "sheet_generation", + "pose_sheet": "sheet_generation", + "style_lock": "hires_fix", + "questionnaire": "quick_generation", + "preview": "quick_generation", + } + if node_key in mapping: + return mapping[node_key] + if node_type == "virtual_single_image": + return "quick_generation" + return "quick_generation" if node_type in {"profile_draft", "profile_preview"} else node_key + + def _optional_text(value: Any) -> str | None: if value is None: return None diff --git a/backend/services/material_service.py b/backend/services/material_service.py index 0cdb3b2..a880de3 100644 --- a/backend/services/material_service.py +++ b/backend/services/material_service.py @@ -43,12 +43,14 @@ class MaterialService: *, vision_provider_factory: Callable[[dict[str, Any]], VisionProvider | None] | None = None, default_recognition_route: str = "material_vlm", + recognition_prompt_config: dict[str, Any] | None = None, storage: VfsStorage | None = None, ): self.repository = repository self.audit_service = audit_service self.vision_provider_factory = vision_provider_factory self.default_recognition_route = default_recognition_route + self.recognition_prompt_config = recognition_prompt_config or {} self.storage = storage or VfsStorage() def create_category(self, payload: dict[str, Any], ctx: RequestContext) -> MaterialCategory: @@ -268,7 +270,7 @@ class MaterialService: }, ) - request = _vision_request(material, self.storage) + request = _vision_request(material, self.storage, self._recognition_instruction(material)) try: recognition = provider.recognize_material(request) except VisionProviderError as exc: @@ -313,6 +315,25 @@ class MaterialService: ) return after + def _recognition_instruction(self, material: Material) -> str: + base = str(self.recognition_prompt_config.get("default_instruction") or "").strip() + requirements = self.recognition_prompt_config.get("category_requirements") + if not isinstance(requirements, list): + return base + category_name = "" + if material.category_id: + category = self.repository.get_category(material.category_id) + category_name = category.name if category else "" + additions: list[str] = [] + for item in requirements: + if not isinstance(item, dict) or item.get("enabled") is False: + continue + category = str(item.get("category") or "").strip().lower() + requirement = str(item.get("requirement") or "").strip() + if requirement and (not category or category in {str(material.category_id or "").lower(), category_name.lower()}): + additions.append(requirement) + return "\n\n".join([part for part in [base, *additions] if part]) + def _vision_provider(self, material: Material, route_name: str) -> VisionProvider | None: if self.vision_provider_factory is None: return None @@ -418,7 +439,7 @@ def new_material_id() -> str: return f"mat_{uuid4().hex}" -def _vision_request(material: Material, storage: VfsStorage) -> VisionRecognitionRequest: +def _vision_request(material: Material, storage: VfsStorage, recognition_instruction: str = "") -> VisionRecognitionRequest: image_data_url, content_type, size_bytes = _storage_image_data_url( str(material.image_uri or ""), material.metadata, @@ -433,6 +454,7 @@ def _vision_request(material: Material, storage: VfsStorage) -> VisionRecognitio tags=tuple(material.tags), metadata={ **material.metadata, + "recognition_instruction": recognition_instruction, "vision_input": { "source_uri": material.image_uri, "content_type": content_type, diff --git a/backend/services/settings_service.py b/backend/services/settings_service.py index ca4d2cd..4cec6c1 100644 --- a/backend/services/settings_service.py +++ b/backend/services/settings_service.py @@ -96,6 +96,33 @@ class SettingsService: _field("provider.enabled", "bool", label="启用 Vision 路由"), ], }, + { + "key": "model_assignments", + "title": "模型分工", + "fields": [ + _field("material_recognition", "string", label="素材解析路由"), + _field("profile_parse", "string", label="角色导入解析路由"), + _field("persona_split", "string", label="人设拆解路由"), + _field("prompt_planning", "string", label="提示词规划路由"), + _field("cross_review_primary", "string", label="交叉审核主模型路由"), + _field("cross_review_secondary", "string", label="交叉审核复核模型路由"), + _field("cross_review_routes", "list", label="交叉审核模型路由列表"), + _field("quick_generation", "string", label="快速生图路由"), + _field("portrait_generation", "string", label="头像生成路由"), + _field("halfbody_generation", "string", label="半身生成路由"), + _field("fullbody_generation", "string", label="全身生成路由"), + _field("sheet_generation", "string", label="设定图生成路由"), + _field("hires_fix", "string", label="高清修复路由"), + ], + }, + { + "key": "recognition_prompts", + "title": "识别提示词", + "fields": [ + _field("default_instruction", "text", label="默认素材识别指令"), + _field("category_requirements", "list", label="分类附加要求"), + ], + }, { "key": "storage", "title": "存储", diff --git a/config.yaml b/config.yaml index 798a507..6a1cba0 100644 --- a/config.yaml +++ b/config.yaml @@ -43,6 +43,29 @@ vision: priority: 10 enabled: true +# 按功能分配模型路由。留空时回退到 generation.default_model 或 vision.default_route。 +model_assignments: + material_recognition: material_vlm + profile_parse: "" + persona_split: "" + prompt_planning: "" + cross_review_primary: material_vlm + cross_review_secondary: material_vlm + cross_review_routes: + - material_vlm + - material_vlm + quick_generation: "" + portrait_generation: "" + halfbody_generation: "" + fullbody_generation: "" + sheet_generation: "" + hires_fix: "" + +recognition_prompts: + default_instruction: >- + 请先判断图片中素材的准确分类,再尽可能完整地提取所有可见事实,包括物品类型、主色与辅色、材质、纹理、结构、轮廓、图案、裁剪、缝线、褶皱、装饰、配件、状态、视角、光照及可复用的设计特征。不要把无法确认的推测写成事实,并明确区分确定信息与低置信度判断。根据识别结果生成一段细节充分、可直接用于生图且能够保留原素材设计的正向提示词,同时生成用于排除错误类别、错误材质、结构畸变、文字、水印和无关背景的负向提示词。仅返回 JSON,字段必须包含 prompt、negative_prompt、tags、confidence、description;tags 应简短、可检索,并包含分类、颜色、材质和关键设计元素。 + category_requirements: [] + storage: # 这里使用逻辑 data/... 路径;AstrBot 运行时映射到 # data/plugin_data/astrbot_plugin_imagestudio/...,本地开发仍映射到仓库 data/。 diff --git a/engines/vision_provider.py b/engines/vision_provider.py index b4f05de..a569c7e 100644 --- a/engines/vision_provider.py +++ b/engines/vision_provider.py @@ -107,10 +107,7 @@ class OpenAIVisionProvider: "messages": [ { "role": "system", - "content": ( - "Return compact JSON for a reusable outfit material with keys: " - "prompt, negative_prompt, tags, confidence, description." - ), + "content": _system_instruction(request), }, {"role": "user", "content": _user_content(request)}, ], @@ -176,6 +173,18 @@ def _user_content(request: VisionRecognitionRequest) -> list[dict[str, Any]]: return content +def _system_instruction(request: VisionRecognitionRequest) -> str: + configured = str(request.metadata.get("recognition_instruction") or "").strip() + fallback = ( + "Analyze the visible material as exhaustively as possible. First identify its category, then extract only " + "visible facts including object type, colors, materials, texture, construction, silhouette, patterns, trims, " + "accessories, condition, viewpoint, lighting and reusable design features. Distinguish certainty from guesses. " + "Produce a detailed image-generation prompt that preserves the recognized design, plus a useful negative prompt. " + "Return JSON only with keys prompt, negative_prompt, tags, confidence, description. Tags must be concise and searchable." + ) + return configured or fallback + + def _remote_image_url(value: str | None) -> str | None: text = str(value or "").strip() if text.startswith(("http://", "https://", "data:image/")): diff --git a/frontend/standalone-web/src/features/settings/ModelAssignmentsPanel.vue b/frontend/standalone-web/src/features/settings/ModelAssignmentsPanel.vue new file mode 100644 index 0000000..af51405 --- /dev/null +++ b/frontend/standalone-web/src/features/settings/ModelAssignmentsPanel.vue @@ -0,0 +1,113 @@ + + + + + diff --git a/frontend/standalone-web/src/features/settings/RecognitionPromptsPanel.vue b/frontend/standalone-web/src/features/settings/RecognitionPromptsPanel.vue new file mode 100644 index 0000000..cebe7d2 --- /dev/null +++ b/frontend/standalone-web/src/features/settings/RecognitionPromptsPanel.vue @@ -0,0 +1,62 @@ + + + + + diff --git a/frontend/standalone-web/src/features/settings/SettingsPage.vue b/frontend/standalone-web/src/features/settings/SettingsPage.vue index aeeaf6e..5960f92 100644 --- a/frontend/standalone-web/src/features/settings/SettingsPage.vue +++ b/frontend/standalone-web/src/features/settings/SettingsPage.vue @@ -9,6 +9,8 @@ import StorageMaintenance from "./StorageMaintenance.vue" import ReviewPolicyPanel from "./ReviewPolicyPanel.vue" import CapabilityProfilesPanel from "./CapabilityProfilesPanel.vue" import SystemManagementPanel from "./SystemManagementPanel.vue" +import ModelAssignmentsPanel from "./ModelAssignmentsPanel.vue" +import RecognitionPromptsPanel from "./RecognitionPromptsPanel.vue" type ConfigValue = string | number | boolean | string[] | null type Adapter = { id:string; name:string; provider_type:string; base_url:string; image_endpoint:string; model_id:string; route_name:string; api_key_env_var:string; timeout_seconds:number; enabled:boolean } @@ -18,6 +20,8 @@ const emit = defineEmits<{refresh:[];validate:[config:Record];sa const previewSchema: SettingsSchema = {schema_version:"3.0-preview",sections:[ {key:"generation",title:"生成服务",fields:[]}, + {key:"model_assignments",title:"模型分工",fields:[]}, + {key:"recognition_prompts",title:"识别提示词",fields:[]}, {key:"features",title:"功能开关",fields:[ {key:"quick_profile",label:"快速角色",type:"bool",options:[],sensitive:false,readonly:false},{key:"workspace",label:"角色工作流",type:"bool",options:[],sensitive:false,readonly:false},{key:"material_library",label:"素材库",type:"bool",options:[],sensitive:false,readonly:false},{key:"wardrobe",label:"角色衣橱",type:"bool",options:[],sensitive:false,readonly:false},{key:"api_discovery",label:"能力探测",type:"bool",options:[],sensitive:false,readonly:false},{key:"private_companion_integration",label:"私伴插件集成",type:"bool",options:[],sensitive:false,readonly:false}, ]}, @@ -29,7 +33,7 @@ const previewSchema: SettingsSchema = {schema_version:"3.0-preview",sections:[ ]}, {key:"backup",title:"数据备份",fields:[]}, ]} -const previewConfig = computed>(()=>({generation:{default_model:"image_default",provider:{provider_id:"openai-compatible",base_url:"",image_endpoint:"/v1/images/generations",model_id:"",route_name:"image_default",api_key_env_var:"IMAGE_API_KEY",timeout_seconds:180}},features:{quick_profile:true,workspace:true,material_library:true,wardrobe:true,api_discovery:true,private_companion_integration:false},storage:{images_path:"data/images",thumbnails_path:"data/thumbnails",materials_path:"data/materials",styles_path:"data/styles",archives_path:"data/archives",temp_path:"data/temp"},database:{backend:"auto",sqlite:{path:"data/imagestudio.db"},postgresql:{enabled:false,host:"127.0.0.1",port:5432,database:"imagestudio",user:"",password:""}}})) +const previewConfig = computed>(()=>({generation:{default_model:"image_default",provider:{provider_id:"openai-compatible",base_url:"",image_endpoint:"/v1/images/generations",model_id:"",route_name:"image_default",api_key_env_var:"IMAGE_API_KEY",timeout_seconds:180}},model_assignments:{material_recognition:"material_vlm",quick_generation:"image_default",portrait_generation:"image_default",halfbody_generation:"image_default",fullbody_generation:"image_default",sheet_generation:"image_default",hires_fix:"image_default"},recognition_prompts:{default_instruction:"请先判断图片中素材的准确分类,再尽可能完整提取所有可见事实,包括颜色、材质、纹理、结构、轮廓、图案、裁剪、装饰、配件、视角和光照。不要把推测写成事实。根据结果生成可直接用于生图的详细正向提示词和负向提示词。仅返回 JSON:prompt、negative_prompt、tags、confidence、description。",category_requirements:[]},features:{quick_profile:true,workspace:true,material_library:true,wardrobe:true,api_discovery:true,private_companion_integration:false},storage:{images_path:"data/images",thumbnails_path:"data/thumbnails",materials_path:"data/materials",styles_path:"data/styles",archives_path:"data/archives",temp_path:"data/temp"},database:{backend:"auto",sqlite:{path:"data/imagestudio.db"},postgresql:{enabled:false,host:"127.0.0.1",port:5432,database:"imagestudio",user:"",password:""}}})) const effectiveSchema=computed(()=>{const schema=clone(props.schema??previewSchema);for(const section of [{key:"review",title:"审查策略",fields:[]},{key:"capabilities",title:"能力档案",fields:[]},{key:"integration",title:"私伴集成",fields:[]},{key:"cost_guard",title:"成本保护",fields:[]},{key:"diagnostics",title:"系统诊断",fields:[]},{key:"backup",title:"数据备份",fields:[]}])if(!schema.sections.some(item=>item.key===section.key))schema.sections.push(section);return schema}) const baseline=computed(()=>props.current?.config??previewConfig.value) const draft=ref>({}) @@ -51,6 +55,9 @@ const adapters=computed(()=>{ const provider=obj(generation.provider) return [normalizeAdapter({...provider,id:String(provider.provider_id||"primary"),name:String(provider.display_name||provider.provider_id||"主生成服务")})] }) +const modelAssignments=computed(()=>obj(draft.value.model_assignments)) +const modelAdapterOptions=computed(()=>adapters.value.map(item=>({name:item.name,route:item.route_name,model:item.model_id,enabled:item.enabled}))) +const visionRouteOption=computed(()=>String(obj(draft.value.vision).default_route||obj(obj(draft.value.vision).provider).route_name||"material_vlm")) watch(baseline,v=>{draft.value=clone(v);expandedAdapter.value=""},{immediate:true,deep:true}) watch(effectiveSchema,s=>{if(!s.sections.some(x=>x.key===activeSectionKey.value))activeSectionKey.value=s.sections[0]?.key||""},{immediate:true}) @@ -77,13 +84,15 @@ function resetDraft(){draft.value=clone(baseline.value)} function fieldPath(k:string){return activeSection.value?`${activeSection.value.key}.${k}`:k} function fieldValue(path:string):ConfigValue{let c:unknown=draft.value;for(const p of path.split(".")){if(!c||typeof c!=="object"||Array.isArray(c))return "";c=(c as Record)[p]}return Array.isArray(c)?c as string[]:typeof c==="string"||typeof c==="number"||typeof c==="boolean"||c===null?c:""} function updateField(path:string,value:ConfigValue){const next=clone(draft.value);const parts=path.split(".");let c=next;for(const p of parts.slice(0,-1)){if(!c[p]||typeof c[p]!=="object"||Array.isArray(c[p]))c[p]={};c=c[p] as Record}c[parts.at(-1)!]=value;draft.value=next} +function updateModelAssignments(value:Record){draft.value={...clone(draft.value),model_assignments:value}} +function updateRecognitionPrompts(value:Record){draft.value={...clone(draft.value),recognition_prompts:value}} function inputValue(e:Event,type:string):ConfigValue{const t=e.target as HTMLInputElement|HTMLSelectElement;if(type==="bool"||type==="boolean")return t.value==="true";if(type==="number")return t.value===""?null:Number(t.value);return t.value}