413 行
17 KiB
Python
413 行
17 KiB
Python
# -*- coding: utf-8 -*-
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import unicodedata
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from astrbot.api import logger
|
|
from astrbot.core.utils.astrbot_path import get_astrbot_data_path
|
|
|
|
from .helpers import _single_line
|
|
|
|
|
|
class AstrBotKnowledgeMixin:
|
|
"""Read AstrBot knowledge-base metadata and selected chunks for roleplay context."""
|
|
|
|
@staticmethod
|
|
def _normalize_roleplay_knowledge_source_ids(value: Any, *, limit: int = 80) -> list[str]:
|
|
if isinstance(value, list):
|
|
raw_items = value
|
|
elif isinstance(value, tuple):
|
|
raw_items = list(value)
|
|
else:
|
|
raw = str(value or "")
|
|
raw_items = re.split(r"[\r\n,,;;]+", raw)
|
|
ids: list[str] = []
|
|
seen: set[str] = set()
|
|
for item in raw_items:
|
|
text = str(item or "").strip()
|
|
if not text or len(text) > 260:
|
|
continue
|
|
if not (text.startswith("kb:") or text.startswith("doc:")):
|
|
continue
|
|
if text in seen:
|
|
continue
|
|
seen.add(text)
|
|
ids.append(text)
|
|
if len(ids) >= limit:
|
|
break
|
|
return ids
|
|
|
|
def _astrbot_knowledge_root(self) -> Path:
|
|
try:
|
|
return Path(get_astrbot_data_path()) / "knowledge_base"
|
|
except Exception:
|
|
data_dir = Path(str(getattr(self, "data_dir", "") or ".")).resolve()
|
|
return data_dir.parent.parent / "knowledge_base"
|
|
|
|
def _astrbot_knowledge_sources(self) -> list[dict[str, Any]]:
|
|
root = self._astrbot_knowledge_root()
|
|
db_path = root / "kb.db"
|
|
if not db_path.exists():
|
|
return []
|
|
try:
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
bases = conn.execute(
|
|
"select kb_id, kb_name, description, emoji, doc_count, chunk_count, updated_at "
|
|
"from knowledge_bases order by updated_at desc, kb_name asc"
|
|
).fetchall()
|
|
docs = conn.execute(
|
|
"select kb_id, doc_id, doc_name, file_type, chunk_count, updated_at "
|
|
"from kb_documents order by updated_at desc, doc_name asc"
|
|
).fetchall()
|
|
conn.close()
|
|
except Exception as exc:
|
|
logger.warning(f"[PrivateCompanion] 读取 AstrBot 知识库列表失败: {exc}")
|
|
return []
|
|
|
|
docs_by_kb: dict[str, list[dict[str, Any]]] = {}
|
|
for row in docs:
|
|
kb_id = str(row["kb_id"] or "").strip()
|
|
doc_id = str(row["doc_id"] or "").strip()
|
|
if not kb_id or not doc_id:
|
|
continue
|
|
docs_by_kb.setdefault(kb_id, []).append(
|
|
{
|
|
"id": f"doc:{kb_id}:{doc_id}",
|
|
"type": "document",
|
|
"kb_id": kb_id,
|
|
"doc_id": doc_id,
|
|
"name": str(row["doc_name"] or doc_id),
|
|
"file_type": str(row["file_type"] or ""),
|
|
"chunk_count": int(row["chunk_count"] or 0),
|
|
"updated_at": str(row["updated_at"] or ""),
|
|
}
|
|
)
|
|
|
|
items: list[dict[str, Any]] = []
|
|
for row in bases:
|
|
kb_id = str(row["kb_id"] or "").strip()
|
|
if not kb_id:
|
|
continue
|
|
documents = docs_by_kb.get(kb_id, [])
|
|
items.append(
|
|
{
|
|
"id": f"kb:{kb_id}",
|
|
"type": "knowledge_base",
|
|
"kb_id": kb_id,
|
|
"name": str(row["kb_name"] or kb_id),
|
|
"description": str(row["description"] or ""),
|
|
"emoji": str(row["emoji"] or ""),
|
|
"doc_count": int(row["doc_count"] or len(documents)),
|
|
"chunk_count": int(row["chunk_count"] or sum(int(doc.get("chunk_count") or 0) for doc in documents)),
|
|
"updated_at": str(row["updated_at"] or ""),
|
|
"documents": documents,
|
|
}
|
|
)
|
|
return items
|
|
|
|
def _roleplay_knowledge_summary(self) -> dict[str, Any]:
|
|
selected = self._normalize_roleplay_knowledge_source_ids(
|
|
getattr(self, "roleplay_knowledge_source_ids", [])
|
|
)
|
|
sources = self._astrbot_knowledge_sources()
|
|
selected_set = set(selected)
|
|
available_ids = {str(item.get("id") or "") for item in sources}
|
|
for item in sources:
|
|
for doc in item.get("documents") or []:
|
|
available_ids.add(str(doc.get("id") or ""))
|
|
return {
|
|
"available": bool(sources),
|
|
"root": str(self._astrbot_knowledge_root()),
|
|
"sources": sources,
|
|
"selected_ids": selected,
|
|
"selected_count": len(selected),
|
|
"missing_ids": [item for item in selected if item not in available_ids],
|
|
}
|
|
|
|
def _format_roleplay_knowledge_context(
|
|
self,
|
|
*,
|
|
query: str = "",
|
|
purpose: str = "roleplay",
|
|
max_chars: int = 3200,
|
|
max_chunks: int = 18,
|
|
) -> str:
|
|
selected = self._normalize_roleplay_knowledge_source_ids(
|
|
getattr(self, "roleplay_knowledge_source_ids", [])
|
|
)
|
|
if not selected:
|
|
return ""
|
|
sources = self._astrbot_knowledge_sources()
|
|
if not sources:
|
|
return ""
|
|
|
|
kb_names = {str(item.get("kb_id") or ""): str(item.get("name") or "") for item in sources}
|
|
doc_names: dict[tuple[str, str], str] = {}
|
|
selected_kbs: set[str] = set()
|
|
selected_docs: dict[str, set[str]] = {}
|
|
for source in sources:
|
|
kb_id = str(source.get("kb_id") or "")
|
|
if f"kb:{kb_id}" in selected:
|
|
selected_kbs.add(kb_id)
|
|
for doc in source.get("documents") or []:
|
|
doc_id = str(doc.get("doc_id") or "")
|
|
doc_names[(kb_id, doc_id)] = str(doc.get("name") or doc_id)
|
|
if str(doc.get("id") or "") in selected:
|
|
selected_docs.setdefault(kb_id, set()).add(doc_id)
|
|
|
|
if not selected_kbs and not selected_docs:
|
|
return ""
|
|
|
|
chunk_budget = max(1, int(max_chunks or 18))
|
|
char_budget = max(600, int(max_chars or 3200))
|
|
candidates: list[dict[str, Any]] = []
|
|
for kb_id in sorted(selected_kbs | set(selected_docs)):
|
|
doc_filter = None if kb_id in selected_kbs else selected_docs.get(kb_id, set())
|
|
chunks = self._read_roleplay_knowledge_chunks(kb_id, doc_filter, scan_limit=max(200, chunk_budget * 80))
|
|
for chunk in chunks:
|
|
doc_id = str(chunk.get("kb_doc_id") or "")
|
|
chunk["kb_id"] = kb_id
|
|
chunk["kb_name"] = kb_names.get(kb_id, kb_id)
|
|
chunk["doc_name"] = doc_names.get((kb_id, doc_id), doc_id or "资料")
|
|
candidates.append(chunk)
|
|
if not candidates:
|
|
return ""
|
|
|
|
effective_query = query or self._build_roleplay_knowledge_query(purpose=purpose)
|
|
selected_chunks = self._select_roleplay_knowledge_chunks(
|
|
candidates,
|
|
query=effective_query,
|
|
max_chunks=chunk_budget,
|
|
)
|
|
used_search = bool(selected_chunks)
|
|
if not selected_chunks:
|
|
selected_chunks = self._select_representative_roleplay_chunks(candidates, max_chunks=chunk_budget)
|
|
|
|
snippets: list[str] = []
|
|
for chunk in selected_chunks:
|
|
text = _single_line(chunk.get("text", ""), 700)
|
|
if not text:
|
|
continue
|
|
source = f"{chunk.get('kb_name') or chunk.get('kb_id') or '知识库'} / {chunk.get('doc_name') or '资料'}"
|
|
snippets.append(f"- {source}: {text}")
|
|
if not snippets:
|
|
return ""
|
|
body = "\n".join(snippets)
|
|
if len(body) > char_budget:
|
|
body = body[:char_budget].rstrip() + "..."
|
|
mode_text = "按当前用途检索匹配片段" if used_search else "未命中检索词,回退为文档代表性取样"
|
|
return (
|
|
"【AstrBot 知识库世界观参考】\n"
|
|
"以下内容来自拓展页角色设定中勾选的 AstrBot 知识库/文档,只作为日程模拟、生活背景和世界观适配参考;"
|
|
"不要复述“知识库/文档/片段”等后台来源,不要把小说叙事逐字当成当前现实发言。\n"
|
|
f"选取方式:{mode_text};用途:{_single_line(purpose, 40) or 'roleplay'}。\n"
|
|
f"{body}"
|
|
)
|
|
|
|
def _build_roleplay_knowledge_query(self, *, purpose: str = "roleplay") -> str:
|
|
anchors = {
|
|
"schedule": "日程 作息 生活 学校 工作 休息 住所 城市 关系 习惯 能力 身份 当前活动",
|
|
"worldview": "世界观 时代 背景 法则 规则 能力 种族 组织 地点 关系网 术语 现代功能转译",
|
|
}
|
|
purpose_key = str(purpose or "").lower()
|
|
anchor = anchors["worldview" if "world" in purpose_key or "适配" in purpose_key else "schedule"]
|
|
parts = [
|
|
str(purpose or ""),
|
|
anchor,
|
|
getattr(self, "bot_name", ""),
|
|
getattr(self, "schedule_persona_prompt", ""),
|
|
getattr(self, "schedule_worldview_prompt", ""),
|
|
getattr(self, "roleplay_user_profile_prompt", ""),
|
|
getattr(self, "worldview_adaptation_prompt", ""),
|
|
]
|
|
getter = getattr(self, "_get_default_persona_prompt", None)
|
|
if callable(getter):
|
|
try:
|
|
parts.append(getter())
|
|
except Exception:
|
|
pass
|
|
return "\n".join(str(part or "") for part in parts if str(part or "").strip())
|
|
|
|
def _select_roleplay_knowledge_chunks(
|
|
self,
|
|
chunks: list[dict[str, Any]],
|
|
*,
|
|
query: str,
|
|
max_chunks: int,
|
|
) -> list[dict[str, Any]]:
|
|
terms = self._roleplay_knowledge_query_terms(query)
|
|
if not terms:
|
|
return []
|
|
|
|
scored: list[tuple[float, int, dict[str, Any]]] = []
|
|
for order, chunk in enumerate(chunks):
|
|
text = self._knowledge_norm(chunk.get("text", ""))
|
|
title = self._knowledge_norm(f"{chunk.get('kb_name', '')} {chunk.get('doc_name', '')}")
|
|
if not text:
|
|
continue
|
|
score = 0.0
|
|
matched = 0
|
|
for term, weight in terms:
|
|
text_count = text.count(term)
|
|
title_count = title.count(term)
|
|
if text_count or title_count:
|
|
matched += 1
|
|
score += min(text_count, 4) * weight
|
|
score += min(title_count, 2) * weight * 1.6
|
|
if score <= 0:
|
|
continue
|
|
try:
|
|
chunk_index = int(chunk.get("chunk_index") or 0)
|
|
except (TypeError, ValueError):
|
|
chunk_index = 0
|
|
score += min(matched, 6) * 0.6
|
|
score += max(0.0, 0.35 - min(chunk_index, 12) * 0.02)
|
|
scored.append((score, order, chunk))
|
|
if not scored:
|
|
return []
|
|
|
|
scored.sort(key=lambda item: (-item[0], item[1]))
|
|
selected: list[dict[str, Any]] = []
|
|
doc_counts: dict[tuple[str, str], int] = {}
|
|
doc_indices: dict[tuple[str, str], set[int]] = {}
|
|
text_fingerprints: set[str] = set()
|
|
per_doc_limit = max(2, (max_chunks + 1) // 2)
|
|
deferred: list[dict[str, Any]] = []
|
|
|
|
def add_chunk(chunk: dict[str, Any], *, allow_adjacent: bool) -> bool:
|
|
doc_key = (str(chunk.get("kb_id") or ""), str(chunk.get("kb_doc_id") or ""))
|
|
if doc_counts.get(doc_key, 0) >= per_doc_limit:
|
|
return False
|
|
try:
|
|
chunk_index = int(chunk.get("chunk_index") or 0)
|
|
except (TypeError, ValueError):
|
|
chunk_index = 0
|
|
if not allow_adjacent and any(abs(chunk_index - used) <= 1 for used in doc_indices.get(doc_key, set())):
|
|
return False
|
|
text = _single_line(chunk.get("text", ""), 220)
|
|
fingerprint = self._knowledge_norm(text)[:160]
|
|
if fingerprint and fingerprint in text_fingerprints:
|
|
return False
|
|
if fingerprint:
|
|
text_fingerprints.add(fingerprint)
|
|
selected.append(chunk)
|
|
doc_counts[doc_key] = doc_counts.get(doc_key, 0) + 1
|
|
doc_indices.setdefault(doc_key, set()).add(chunk_index)
|
|
return True
|
|
|
|
for _score, _order, chunk in scored:
|
|
if not add_chunk(chunk, allow_adjacent=False):
|
|
deferred.append(chunk)
|
|
if len(selected) >= max_chunks:
|
|
break
|
|
for chunk in deferred:
|
|
if len(selected) >= max_chunks:
|
|
break
|
|
add_chunk(chunk, allow_adjacent=True)
|
|
return selected
|
|
|
|
def _select_representative_roleplay_chunks(
|
|
self,
|
|
chunks: list[dict[str, Any]],
|
|
*,
|
|
max_chunks: int,
|
|
) -> list[dict[str, Any]]:
|
|
grouped: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
for chunk in chunks:
|
|
grouped.setdefault(
|
|
(str(chunk.get("kb_id") or ""), str(chunk.get("kb_doc_id") or "")),
|
|
[],
|
|
).append(chunk)
|
|
for items in grouped.values():
|
|
items.sort(key=lambda item: int(item.get("chunk_index") or 0))
|
|
selected: list[dict[str, Any]] = []
|
|
max_len = max((len(items) for items in grouped.values()), default=0)
|
|
for index in range(max_len):
|
|
for key in sorted(grouped):
|
|
items = grouped[key]
|
|
if index < len(items):
|
|
selected.append(items[index])
|
|
if len(selected) >= max_chunks:
|
|
return selected
|
|
return selected
|
|
|
|
def _roleplay_knowledge_query_terms(self, query: str) -> list[tuple[str, float]]:
|
|
text = self._knowledge_norm(query)
|
|
if not text:
|
|
return []
|
|
raw_terms = re.findall(r"[a-z][a-z0-9_+-]{2,}|[\u4e00-\u9fff]{2,12}", text)
|
|
stop_terms = {
|
|
"角色设定", "世界观", "生活背景", "其他补充", "用户补充", "当前使用",
|
|
"日程专用", "适配模式", "自定义", "默认人格", "插件", "不要", "可以",
|
|
"例如", "使用", "当前", "补充信息", "主要", "基本", "关系",
|
|
}
|
|
weighted: dict[str, float] = {}
|
|
for raw in raw_terms:
|
|
term = raw.strip()
|
|
if not term or term in stop_terms:
|
|
continue
|
|
weight = 1.0 + min(len(term), 8) / 8.0
|
|
weighted[term] = max(weighted.get(term, 0.0), weight)
|
|
if re.fullmatch(r"[\u4e00-\u9fff]{5,12}", term):
|
|
for size in (2, 3, 4):
|
|
for index in range(0, len(term) - size + 1):
|
|
sub = term[index:index + size]
|
|
if sub in stop_terms:
|
|
continue
|
|
weighted[sub] = max(weighted.get(sub, 0.0), 0.65 + size * 0.15)
|
|
return sorted(weighted.items(), key=lambda item: (-item[1], item[0]))[:90]
|
|
|
|
@staticmethod
|
|
def _knowledge_norm(value: Any) -> str:
|
|
text = unicodedata.normalize("NFKC", str(value or "")).lower()
|
|
return re.sub(r"\s+", "", text)
|
|
|
|
def _read_roleplay_knowledge_chunks(
|
|
self,
|
|
kb_id: str,
|
|
doc_filter: set[str] | None,
|
|
*,
|
|
scan_limit: int = 1600,
|
|
) -> list[dict[str, Any]]:
|
|
db_path = self._astrbot_knowledge_root() / kb_id / "doc.db"
|
|
if not db_path.exists():
|
|
return []
|
|
try:
|
|
conn = sqlite3.connect(str(db_path))
|
|
conn.row_factory = sqlite3.Row
|
|
rows = conn.execute("select text, metadata from documents order by id asc").fetchall()
|
|
conn.close()
|
|
except Exception as exc:
|
|
logger.warning(f"[PrivateCompanion] 读取 AstrBot 知识库片段失败 {kb_id}: {exc}")
|
|
return []
|
|
|
|
chunks: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
try:
|
|
metadata = json.loads(row["metadata"] or "{}")
|
|
except Exception:
|
|
metadata = {}
|
|
kb_doc_id = str(metadata.get("kb_doc_id") or "")
|
|
if doc_filter is not None and kb_doc_id not in doc_filter:
|
|
continue
|
|
try:
|
|
chunk_index = int(metadata.get("chunk_index") or 0)
|
|
except (TypeError, ValueError):
|
|
chunk_index = 0
|
|
chunks.append(
|
|
{
|
|
"text": str(row["text"] or ""),
|
|
"kb_doc_id": kb_doc_id,
|
|
"chunk_index": chunk_index,
|
|
}
|
|
)
|
|
if len(chunks) >= max(1, int(scan_limit or 1600)):
|
|
break
|
|
chunks.sort(key=lambda item: (str(item.get("kb_doc_id") or ""), int(item.get("chunk_index") or 0)))
|
|
return chunks
|