# -*- coding: utf-8 -*- from __future__ import annotations import json import re import sqlite3 import unicodedata from pathlib import Path from typing import Any from astrbot.api import logger from astrbot.core.utils.astrbot_path import get_astrbot_data_path from .helpers import _single_line class AstrBotKnowledgeMixin: """Read AstrBot knowledge-base metadata and selected chunks for roleplay context.""" @staticmethod def _normalize_roleplay_knowledge_source_ids(value: Any, *, limit: int = 80) -> list[str]: if isinstance(value, list): raw_items = value elif isinstance(value, tuple): raw_items = list(value) else: raw = str(value or "") raw_items = re.split(r"[\r\n,,;;]+", raw) ids: list[str] = [] seen: set[str] = set() for item in raw_items: text = str(item or "").strip() if not text or len(text) > 260: continue if not (text.startswith("kb:") or text.startswith("doc:")): continue if text in seen: continue seen.add(text) ids.append(text) if len(ids) >= limit: break return ids def _astrbot_knowledge_root(self) -> Path: try: return Path(get_astrbot_data_path()) / "knowledge_base" except Exception: data_dir = Path(str(getattr(self, "data_dir", "") or ".")).resolve() return data_dir.parent.parent / "knowledge_base" def _astrbot_knowledge_sources(self) -> list[dict[str, Any]]: root = self._astrbot_knowledge_root() db_path = root / "kb.db" if not db_path.exists(): return [] try: conn = sqlite3.connect(str(db_path)) conn.row_factory = sqlite3.Row bases = conn.execute( "select kb_id, kb_name, description, emoji, doc_count, chunk_count, updated_at " "from knowledge_bases order by updated_at desc, kb_name asc" ).fetchall() docs = conn.execute( "select kb_id, doc_id, doc_name, file_type, chunk_count, updated_at " "from kb_documents order by updated_at desc, doc_name asc" ).fetchall() conn.close() except Exception as exc: logger.warning(f"[PrivateCompanion] 读取 AstrBot 知识库列表失败: {exc}") return [] docs_by_kb: dict[str, list[dict[str, Any]]] = {} for row in docs: kb_id = str(row["kb_id"] or "").strip() doc_id = str(row["doc_id"] or "").strip() if not kb_id or not doc_id: continue docs_by_kb.setdefault(kb_id, []).append( { "id": f"doc:{kb_id}:{doc_id}", "type": "document", "kb_id": kb_id, "doc_id": doc_id, "name": str(row["doc_name"] or doc_id), "file_type": str(row["file_type"] or ""), "chunk_count": int(row["chunk_count"] or 0), "updated_at": str(row["updated_at"] or ""), } ) items: list[dict[str, Any]] = [] for row in bases: kb_id = str(row["kb_id"] or "").strip() if not kb_id: continue documents = docs_by_kb.get(kb_id, []) items.append( { "id": f"kb:{kb_id}", "type": "knowledge_base", "kb_id": kb_id, "name": str(row["kb_name"] or kb_id), "description": str(row["description"] or ""), "emoji": str(row["emoji"] or ""), "doc_count": int(row["doc_count"] or len(documents)), "chunk_count": int(row["chunk_count"] or sum(int(doc.get("chunk_count") or 0) for doc in documents)), "updated_at": str(row["updated_at"] or ""), "documents": documents, } ) return items def _roleplay_knowledge_summary(self) -> dict[str, Any]: selected = self._normalize_roleplay_knowledge_source_ids( getattr(self, "roleplay_knowledge_source_ids", []) ) sources = self._astrbot_knowledge_sources() selected_set = set(selected) available_ids = {str(item.get("id") or "") for item in sources} for item in sources: for doc in item.get("documents") or []: available_ids.add(str(doc.get("id") or "")) return { "available": bool(sources), "root": str(self._astrbot_knowledge_root()), "sources": sources, "selected_ids": selected, "selected_count": len(selected), "missing_ids": [item for item in selected if item not in available_ids], } def _format_roleplay_knowledge_context( self, *, query: str = "", purpose: str = "roleplay", max_chars: int = 3200, max_chunks: int = 18, ) -> str: selected = self._normalize_roleplay_knowledge_source_ids( getattr(self, "roleplay_knowledge_source_ids", []) ) if not selected: return "" sources = self._astrbot_knowledge_sources() if not sources: return "" kb_names = {str(item.get("kb_id") or ""): str(item.get("name") or "") for item in sources} doc_names: dict[tuple[str, str], str] = {} selected_kbs: set[str] = set() selected_docs: dict[str, set[str]] = {} for source in sources: kb_id = str(source.get("kb_id") or "") if f"kb:{kb_id}" in selected: selected_kbs.add(kb_id) for doc in source.get("documents") or []: doc_id = str(doc.get("doc_id") or "") doc_names[(kb_id, doc_id)] = str(doc.get("name") or doc_id) if str(doc.get("id") or "") in selected: selected_docs.setdefault(kb_id, set()).add(doc_id) if not selected_kbs and not selected_docs: return "" chunk_budget = max(1, int(max_chunks or 18)) char_budget = max(600, int(max_chars or 3200)) candidates: list[dict[str, Any]] = [] for kb_id in sorted(selected_kbs | set(selected_docs)): doc_filter = None if kb_id in selected_kbs else selected_docs.get(kb_id, set()) chunks = self._read_roleplay_knowledge_chunks(kb_id, doc_filter, scan_limit=max(200, chunk_budget * 80)) for chunk in chunks: doc_id = str(chunk.get("kb_doc_id") or "") chunk["kb_id"] = kb_id chunk["kb_name"] = kb_names.get(kb_id, kb_id) chunk["doc_name"] = doc_names.get((kb_id, doc_id), doc_id or "资料") candidates.append(chunk) if not candidates: return "" effective_query = query or self._build_roleplay_knowledge_query(purpose=purpose) selected_chunks = self._select_roleplay_knowledge_chunks( candidates, query=effective_query, max_chunks=chunk_budget, ) used_search = bool(selected_chunks) if not selected_chunks: selected_chunks = self._select_representative_roleplay_chunks(candidates, max_chunks=chunk_budget) snippets: list[str] = [] for chunk in selected_chunks: text = _single_line(chunk.get("text", ""), 700) if not text: continue source = f"{chunk.get('kb_name') or chunk.get('kb_id') or '知识库'} / {chunk.get('doc_name') or '资料'}" snippets.append(f"- {source}: {text}") if not snippets: return "" body = "\n".join(snippets) if len(body) > char_budget: body = body[:char_budget].rstrip() + "..." mode_text = "按当前用途检索匹配片段" if used_search else "未命中检索词,回退为文档代表性取样" return ( "【AstrBot 知识库世界观参考】\n" "以下内容来自拓展页角色设定中勾选的 AstrBot 知识库/文档,只作为日程模拟、生活背景和世界观适配参考;" "不要复述“知识库/文档/片段”等后台来源,不要把小说叙事逐字当成当前现实发言。\n" f"选取方式:{mode_text};用途:{_single_line(purpose, 40) or 'roleplay'}。\n" f"{body}" ) def _build_roleplay_knowledge_query(self, *, purpose: str = "roleplay") -> str: anchors = { "schedule": "日程 作息 生活 学校 工作 休息 住所 城市 关系 习惯 能力 身份 当前活动", "worldview": "世界观 时代 背景 法则 规则 能力 种族 组织 地点 关系网 术语 现代功能转译", } purpose_key = str(purpose or "").lower() anchor = anchors["worldview" if "world" in purpose_key or "适配" in purpose_key else "schedule"] parts = [ str(purpose or ""), anchor, getattr(self, "bot_name", ""), getattr(self, "schedule_persona_prompt", ""), getattr(self, "schedule_worldview_prompt", ""), getattr(self, "roleplay_user_profile_prompt", ""), getattr(self, "worldview_adaptation_prompt", ""), ] getter = getattr(self, "_get_default_persona_prompt", None) if callable(getter): try: parts.append(getter()) except Exception: pass return "\n".join(str(part or "") for part in parts if str(part or "").strip()) def _select_roleplay_knowledge_chunks( self, chunks: list[dict[str, Any]], *, query: str, max_chunks: int, ) -> list[dict[str, Any]]: terms = self._roleplay_knowledge_query_terms(query) if not terms: return [] scored: list[tuple[float, int, dict[str, Any]]] = [] for order, chunk in enumerate(chunks): text = self._knowledge_norm(chunk.get("text", "")) title = self._knowledge_norm(f"{chunk.get('kb_name', '')} {chunk.get('doc_name', '')}") if not text: continue score = 0.0 matched = 0 for term, weight in terms: text_count = text.count(term) title_count = title.count(term) if text_count or title_count: matched += 1 score += min(text_count, 4) * weight score += min(title_count, 2) * weight * 1.6 if score <= 0: continue try: chunk_index = int(chunk.get("chunk_index") or 0) except (TypeError, ValueError): chunk_index = 0 score += min(matched, 6) * 0.6 score += max(0.0, 0.35 - min(chunk_index, 12) * 0.02) scored.append((score, order, chunk)) if not scored: return [] scored.sort(key=lambda item: (-item[0], item[1])) selected: list[dict[str, Any]] = [] doc_counts: dict[tuple[str, str], int] = {} doc_indices: dict[tuple[str, str], set[int]] = {} text_fingerprints: set[str] = set() per_doc_limit = max(2, (max_chunks + 1) // 2) deferred: list[dict[str, Any]] = [] def add_chunk(chunk: dict[str, Any], *, allow_adjacent: bool) -> bool: doc_key = (str(chunk.get("kb_id") or ""), str(chunk.get("kb_doc_id") or "")) if doc_counts.get(doc_key, 0) >= per_doc_limit: return False try: chunk_index = int(chunk.get("chunk_index") or 0) except (TypeError, ValueError): chunk_index = 0 if not allow_adjacent and any(abs(chunk_index - used) <= 1 for used in doc_indices.get(doc_key, set())): return False text = _single_line(chunk.get("text", ""), 220) fingerprint = self._knowledge_norm(text)[:160] if fingerprint and fingerprint in text_fingerprints: return False if fingerprint: text_fingerprints.add(fingerprint) selected.append(chunk) doc_counts[doc_key] = doc_counts.get(doc_key, 0) + 1 doc_indices.setdefault(doc_key, set()).add(chunk_index) return True for _score, _order, chunk in scored: if not add_chunk(chunk, allow_adjacent=False): deferred.append(chunk) if len(selected) >= max_chunks: break for chunk in deferred: if len(selected) >= max_chunks: break add_chunk(chunk, allow_adjacent=True) return selected def _select_representative_roleplay_chunks( self, chunks: list[dict[str, Any]], *, max_chunks: int, ) -> list[dict[str, Any]]: grouped: dict[tuple[str, str], list[dict[str, Any]]] = {} for chunk in chunks: grouped.setdefault( (str(chunk.get("kb_id") or ""), str(chunk.get("kb_doc_id") or "")), [], ).append(chunk) for items in grouped.values(): items.sort(key=lambda item: int(item.get("chunk_index") or 0)) selected: list[dict[str, Any]] = [] max_len = max((len(items) for items in grouped.values()), default=0) for index in range(max_len): for key in sorted(grouped): items = grouped[key] if index < len(items): selected.append(items[index]) if len(selected) >= max_chunks: return selected return selected def _roleplay_knowledge_query_terms(self, query: str) -> list[tuple[str, float]]: text = self._knowledge_norm(query) if not text: return [] raw_terms = re.findall(r"[a-z][a-z0-9_+-]{2,}|[\u4e00-\u9fff]{2,12}", text) stop_terms = { "角色设定", "世界观", "生活背景", "其他补充", "用户补充", "当前使用", "日程专用", "适配模式", "自定义", "默认人格", "插件", "不要", "可以", "例如", "使用", "当前", "补充信息", "主要", "基本", "关系", } weighted: dict[str, float] = {} for raw in raw_terms: term = raw.strip() if not term or term in stop_terms: continue weight = 1.0 + min(len(term), 8) / 8.0 weighted[term] = max(weighted.get(term, 0.0), weight) if re.fullmatch(r"[\u4e00-\u9fff]{5,12}", term): for size in (2, 3, 4): for index in range(0, len(term) - size + 1): sub = term[index:index + size] if sub in stop_terms: continue weighted[sub] = max(weighted.get(sub, 0.0), 0.65 + size * 0.15) return sorted(weighted.items(), key=lambda item: (-item[1], item[0]))[:90] @staticmethod def _knowledge_norm(value: Any) -> str: text = unicodedata.normalize("NFKC", str(value or "")).lower() return re.sub(r"\s+", "", text) def _read_roleplay_knowledge_chunks( self, kb_id: str, doc_filter: set[str] | None, *, scan_limit: int = 1600, ) -> list[dict[str, Any]]: db_path = self._astrbot_knowledge_root() / kb_id / "doc.db" if not db_path.exists(): return [] try: conn = sqlite3.connect(str(db_path)) conn.row_factory = sqlite3.Row rows = conn.execute("select text, metadata from documents order by id asc").fetchall() conn.close() except Exception as exc: logger.warning(f"[PrivateCompanion] 读取 AstrBot 知识库片段失败 {kb_id}: {exc}") return [] chunks: list[dict[str, Any]] = [] for row in rows: try: metadata = json.loads(row["metadata"] or "{}") except Exception: metadata = {} kb_doc_id = str(metadata.get("kb_doc_id") or "") if doc_filter is not None and kb_doc_id not in doc_filter: continue try: chunk_index = int(metadata.get("chunk_index") or 0) except (TypeError, ValueError): chunk_index = 0 chunks.append( { "text": str(row["text"] or ""), "kb_doc_id": kb_doc_id, "chunk_index": chunk_index, } ) if len(chunks) >= max(1, int(scan_limit or 1600)): break chunks.sort(key=lambda item: (str(item.get("kb_doc_id") or ""), int(item.get("chunk_index") or 0))) return chunks