fix(c7): bound import export and retrieval work
这个提交存在于:
父节点
4b0219eac2
当前提交
aae4565c84
共有 7 个文件被更改,包括 294 次插入 和 36 次删除
|
|
@ -637,6 +637,7 @@ class HistoricalChatImporter:
|
||||||
IDENTITY_LINKS_VERSION = 2
|
IDENTITY_LINKS_VERSION = 2
|
||||||
SUMMARY_PERSPECTIVE_VERSION = 1
|
SUMMARY_PERSPECTIVE_VERSION = 1
|
||||||
DETAIL_SCHEMA_VERSION = 1
|
DETAIL_SCHEMA_VERSION = 1
|
||||||
|
STAGED_DATA_ERROR = "历史导入暂存数据损坏,请重新上传或清理该预览"
|
||||||
|
|
||||||
def __init__(self, service: Any) -> None:
|
def __init__(self, service: Any) -> None:
|
||||||
self.service = service
|
self.service = service
|
||||||
|
|
@ -1276,17 +1277,32 @@ class HistoricalChatImporter:
|
||||||
manifest_path = self._upload_dir(upload_id) / "manifest.json"
|
manifest_path = self._upload_dir(upload_id) / "manifest.json"
|
||||||
if not manifest_path.is_file():
|
if not manifest_path.is_file():
|
||||||
raise ValueError("上传预览不存在或已被清理")
|
raise ValueError("上传预览不存在或已被清理")
|
||||||
return json.loads(manifest_path.read_text(encoding="utf-8"))
|
try:
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise ValueError(self.STAGED_DATA_ERROR) from exc
|
||||||
|
if not isinstance(manifest, dict):
|
||||||
|
raise ValueError(self.STAGED_DATA_ERROR)
|
||||||
|
if clean_text(manifest.get("upload_id"), 120) != clean_text(upload_id, 120):
|
||||||
|
raise ValueError(self.STAGED_DATA_ERROR)
|
||||||
|
return manifest
|
||||||
|
|
||||||
def _load_upload_messages(self, upload_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
def _load_upload_messages(self, upload_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||||
manifest = self.preview_upload(upload_id)
|
manifest = self.preview_upload(upload_id)
|
||||||
parsed_path = self._upload_dir(upload_id) / "parsed.jsonl"
|
parsed_path = self._upload_dir(upload_id) / "parsed.jsonl"
|
||||||
messages: list[dict[str, Any]] = []
|
messages: list[dict[str, Any]] = []
|
||||||
for raw in parsed_path.read_text(encoding="utf-8").splitlines():
|
try:
|
||||||
if raw.strip():
|
for line_no, raw in enumerate(parsed_path.read_text(encoding="utf-8").splitlines(), 1):
|
||||||
|
if not raw.strip():
|
||||||
|
continue
|
||||||
value = json.loads(raw)
|
value = json.loads(raw)
|
||||||
if isinstance(value, dict):
|
if not isinstance(value, dict):
|
||||||
messages.append(value)
|
raise ValueError(f"line={line_no}")
|
||||||
|
messages.append(value)
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
|
||||||
|
raise ValueError(self.STAGED_DATA_ERROR) from exc
|
||||||
|
if not messages:
|
||||||
|
raise ValueError(self.STAGED_DATA_ERROR)
|
||||||
return manifest, messages
|
return manifest, messages
|
||||||
|
|
||||||
def _private_api(self) -> Any | None:
|
def _private_api(self) -> Any | None:
|
||||||
|
|
@ -1986,7 +2002,12 @@ class HistoricalChatImporter:
|
||||||
batch["id"], statuses={"completed", "archived_only"}
|
batch["id"], statuses={"completed", "archived_only"}
|
||||||
)
|
)
|
||||||
memory_counts = await self.store.chat_import_memory_counts(batch["id"])
|
memory_counts = await self.store.chat_import_memory_counts(batch["id"])
|
||||||
checkpoint = max((int(item.get("segment_index") or 0) + 1 for item in completed_segments), default=0)
|
current_checkpoint = max(0, int(batch.get("checkpoint_segment") or 0))
|
||||||
|
computed_checkpoint = max(
|
||||||
|
(int(item.get("segment_index") or 0) + 1 for item in completed_segments),
|
||||||
|
default=0,
|
||||||
|
)
|
||||||
|
checkpoint = max(current_checkpoint, computed_checkpoint)
|
||||||
await self.store.update_chat_import_batch(
|
await self.store.update_chat_import_batch(
|
||||||
batch["id"],
|
batch["id"],
|
||||||
checkpoint_segment=checkpoint,
|
checkpoint_segment=checkpoint,
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ from .models import EntityRef, MemoryRecord, clean_text, json_loads
|
||||||
PORTABLE_FORMAT = "astrbot-memory-jsonl"
|
PORTABLE_FORMAT = "astrbot-memory-jsonl"
|
||||||
PORTABLE_VERSION = 1
|
PORTABLE_VERSION = 1
|
||||||
MAX_PORTABLE_BYTES = 64 * 1024 * 1024
|
MAX_PORTABLE_BYTES = 64 * 1024 * 1024
|
||||||
|
PORTABLE_EXPORT_PAGE_SIZE = 500
|
||||||
|
|
||||||
|
|
||||||
PRESETS: dict[str, dict[str, Any]] = {
|
PRESETS: dict[str, dict[str, Any]] = {
|
||||||
|
|
@ -302,17 +303,14 @@ class PortableMemoryArchive:
|
||||||
self.export_dir = self.data_dir / "exports"
|
self.export_dir = self.data_dir / "exports"
|
||||||
|
|
||||||
async def export(self) -> dict[str, Any]:
|
async def export(self) -> dict[str, Any]:
|
||||||
memories = await self.store.list_memories(limit=1_000_000)
|
stats = await self.store.stats()
|
||||||
identities = await self.store.list_identities(limit=1_000_000)
|
|
||||||
relationships = await self.store.list_relationships(limit=1_000_000)
|
|
||||||
timeline = await self.store.recent_timeline(limit=1_000_000)
|
|
||||||
acl_rules = await self.store.list_acl_rules()
|
acl_rules = await self.store.list_acl_rules()
|
||||||
acl_policies = await self.store.list_acl_policies()
|
acl_policies = await self.store.list_acl_policies()
|
||||||
counts = {
|
counts = {
|
||||||
"memory": len(memories),
|
"memory": int(stats.get("total_memories") or 0),
|
||||||
"identity": len(identities),
|
"identity": int(stats.get("identities") or 0),
|
||||||
"relationship": len(relationships),
|
"relationship": int(stats.get("relationships") or 0),
|
||||||
"timeline": len(timeline),
|
"timeline": int(stats.get("timeline_events") or 0),
|
||||||
"acl_rule": len(acl_rules),
|
"acl_rule": len(acl_rules),
|
||||||
"acl_policy": len(acl_policies),
|
"acl_policy": len(acl_policies),
|
||||||
}
|
}
|
||||||
|
|
@ -333,15 +331,26 @@ class PortableMemoryArchive:
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||||
self._write_line(handle, header)
|
self._write_line(handle, header)
|
||||||
for memory in memories:
|
for record_type, loader, normalizer in (
|
||||||
self._write_line(handle, {"record_type": "memory", "data": self._memory_payload(memory)})
|
("memory", self.store.list_memories, self._memory_payload),
|
||||||
for kind, rows in (
|
("identity", self.store.list_identities, self._normalized_row),
|
||||||
("identity", identities),
|
("relationship", self.store.list_relationships, self._normalized_row),
|
||||||
("relationship", relationships),
|
("timeline", self.store.recent_timeline, self._normalized_row),
|
||||||
("timeline", timeline),
|
|
||||||
("acl_rule", acl_rules),
|
|
||||||
("acl_policy", acl_policies),
|
|
||||||
):
|
):
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
rows = await loader(limit=PORTABLE_EXPORT_PAGE_SIZE, offset=offset)
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for row in rows:
|
||||||
|
self._write_line(
|
||||||
|
handle,
|
||||||
|
{"record_type": record_type, "data": normalizer(row)},
|
||||||
|
)
|
||||||
|
offset += len(rows)
|
||||||
|
if len(rows) < PORTABLE_EXPORT_PAGE_SIZE:
|
||||||
|
break
|
||||||
|
for kind, rows in (("acl_rule", acl_rules), ("acl_policy", acl_policies)):
|
||||||
for row in rows:
|
for row in rows:
|
||||||
self._write_line(handle, {"record_type": kind, "data": self._normalized_row(row)})
|
self._write_line(handle, {"record_type": kind, "data": self._normalized_row(row)})
|
||||||
os.replace(temp_name, path)
|
os.replace(temp_name, path)
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ from .visibility import VisibilityPolicy
|
||||||
|
|
||||||
|
|
||||||
class RetrievalEngine:
|
class RetrievalEngine:
|
||||||
|
DEFAULT_MATERIALIZE_LIMIT = 2000
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
store: MemoryStore,
|
store: MemoryStore,
|
||||||
|
|
@ -900,7 +901,11 @@ class RetrievalEngine:
|
||||||
)
|
)
|
||||||
expanded_terms = self._merge_terms(terms, graph_terms)
|
expanded_terms = self._merge_terms(terms, graph_terms)
|
||||||
include_pending = not self.policy.hide_pending_review
|
include_pending = not self.policy.hide_pending_review
|
||||||
ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending)
|
# Keep the broad priority scan explicitly bounded.
|
||||||
|
ranked_candidates = await self.store.list_candidate_memories(
|
||||||
|
limit=self.DEFAULT_MATERIALIZE_LIMIT,
|
||||||
|
include_pending=include_pending,
|
||||||
|
)
|
||||||
current_window_candidates = await self.store.list_current_window_candidate_memories(
|
current_window_candidates = await self.store.list_current_window_candidate_memories(
|
||||||
scope=ctx.scope,
|
scope=ctx.scope,
|
||||||
session_id=ctx.session_id,
|
session_id=ctx.session_id,
|
||||||
|
|
|
||||||
|
|
@ -2444,9 +2444,13 @@ class MemoryCompanionService:
|
||||||
provider_id = clean_text(provider_id, 160)
|
provider_id = clean_text(provider_id, 160)
|
||||||
indexed = 0
|
indexed = 0
|
||||||
try:
|
try:
|
||||||
|
batch_size = max(
|
||||||
|
1,
|
||||||
|
min(200, self.config.int("retrieval.embedding_backfill_batch_size", 50)),
|
||||||
|
)
|
||||||
records = await self.store.list_memories_missing_embeddings(
|
records = await self.store.list_memories_missing_embeddings(
|
||||||
provider_id=provider_id,
|
provider_id=provider_id,
|
||||||
limit=self.config.int("retrieval.embedding_backfill_batch_size", 50),
|
limit=batch_size,
|
||||||
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
|
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
|
||||||
)
|
)
|
||||||
for record in records:
|
for record in records:
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,8 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads,
|
||||||
|
|
||||||
|
|
||||||
class MemoryStore:
|
class MemoryStore:
|
||||||
|
EMBEDDING_CANDIDATE_CACHE_MAX = 64
|
||||||
|
|
||||||
def __init__(self, db_path: Path):
|
def __init__(self, db_path: Path):
|
||||||
self.db_path = Path(db_path)
|
self.db_path = Path(db_path)
|
||||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
@ -1421,14 +1423,14 @@ class MemoryStore:
|
||||||
self._conn.commit()
|
self._conn.commit()
|
||||||
return row_id
|
return row_id
|
||||||
|
|
||||||
async def list_identities(self, limit: int = 1000) -> list[dict[str, Any]]:
|
async def list_identities(self, limit: int = 1000, *, offset: int = 0) -> list[dict[str, Any]]:
|
||||||
return await asyncio.to_thread(self._list_identities_sync, limit)
|
return await asyncio.to_thread(self._list_identities_sync, limit, offset)
|
||||||
|
|
||||||
def _list_identities_sync(self, limit: int) -> list[dict[str, Any]]:
|
def _list_identities_sync(self, limit: int, offset: int) -> list[dict[str, Any]]:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
rows = self._conn.execute(
|
rows = self._conn.execute(
|
||||||
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ?",
|
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ? OFFSET ?",
|
||||||
(max(1, int(limit)),),
|
(max(1, int(limit)), max(0, int(offset))),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
result: list[dict[str, Any]] = []
|
result: list[dict[str, Any]] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
|
@ -1563,6 +1565,8 @@ class MemoryStore:
|
||||||
scope: str = "",
|
scope: str = "",
|
||||||
session_id: str = "",
|
session_id: str = "",
|
||||||
group_id: str = "",
|
group_id: str = "",
|
||||||
|
*,
|
||||||
|
offset: int = 0,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
return await asyncio.to_thread(
|
return await asyncio.to_thread(
|
||||||
self._list_relationships_sync,
|
self._list_relationships_sync,
|
||||||
|
|
@ -1571,6 +1575,7 @@ class MemoryStore:
|
||||||
scope,
|
scope,
|
||||||
session_id,
|
session_id,
|
||||||
group_id,
|
group_id,
|
||||||
|
offset,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _list_relationships_sync(
|
def _list_relationships_sync(
|
||||||
|
|
@ -1580,6 +1585,7 @@ class MemoryStore:
|
||||||
scope: str,
|
scope: str,
|
||||||
session_id: str,
|
session_id: str,
|
||||||
group_id: str,
|
group_id: str,
|
||||||
|
offset: int,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
params: list[Any] = []
|
params: list[Any] = []
|
||||||
where = "1=1"
|
where = "1=1"
|
||||||
|
|
@ -1601,9 +1607,9 @@ class MemoryStore:
|
||||||
SELECT * FROM relationship_edges
|
SELECT * FROM relationship_edges
|
||||||
WHERE {where}
|
WHERE {where}
|
||||||
ORDER BY updated_at DESC
|
ORDER BY updated_at DESC
|
||||||
LIMIT ?
|
LIMIT ? OFFSET ?
|
||||||
""",
|
""",
|
||||||
params + [max(1, int(limit))],
|
params + [max(1, int(limit)), max(0, int(offset))],
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
|
@ -2684,8 +2690,17 @@ class MemoryStore:
|
||||||
scope: str = "",
|
scope: str = "",
|
||||||
session_id: str = "",
|
session_id: str = "",
|
||||||
entity_id: str = "",
|
entity_id: str = "",
|
||||||
|
*,
|
||||||
|
offset: int = 0,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
return await asyncio.to_thread(self._recent_timeline_sync, limit, scope, session_id, entity_id)
|
return await asyncio.to_thread(
|
||||||
|
self._recent_timeline_sync,
|
||||||
|
limit,
|
||||||
|
scope,
|
||||||
|
session_id,
|
||||||
|
entity_id,
|
||||||
|
offset,
|
||||||
|
)
|
||||||
|
|
||||||
def _recent_timeline_sync(
|
def _recent_timeline_sync(
|
||||||
self,
|
self,
|
||||||
|
|
@ -2693,6 +2708,7 @@ class MemoryStore:
|
||||||
scope: str,
|
scope: str,
|
||||||
session_id: str,
|
session_id: str,
|
||||||
entity_id: str,
|
entity_id: str,
|
||||||
|
offset: int,
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
params: list[Any] = []
|
params: list[Any] = []
|
||||||
where = "1=1"
|
where = "1=1"
|
||||||
|
|
@ -2711,9 +2727,9 @@ class MemoryStore:
|
||||||
SELECT * FROM timeline
|
SELECT * FROM timeline
|
||||||
WHERE {where}
|
WHERE {where}
|
||||||
ORDER BY occurred_at DESC, created_at DESC
|
ORDER BY occurred_at DESC, created_at DESC
|
||||||
LIMIT ?
|
LIMIT ? OFFSET ?
|
||||||
""",
|
""",
|
||||||
params + [max(1, int(limit))],
|
params + [max(1, int(limit)), max(0, int(offset))],
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return [dict(row) for row in rows]
|
return [dict(row) for row in rows]
|
||||||
|
|
||||||
|
|
@ -3660,6 +3676,7 @@ class MemoryStore:
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
limit: int = 50,
|
limit: int = 50,
|
||||||
|
offset: int = 0,
|
||||||
include_pending: bool = True,
|
include_pending: bool = True,
|
||||||
query: str = "",
|
query: str = "",
|
||||||
memory_type: str = "",
|
memory_type: str = "",
|
||||||
|
|
@ -3674,6 +3691,7 @@ class MemoryStore:
|
||||||
return await self._run_recoverable_database_operation(
|
return await self._run_recoverable_database_operation(
|
||||||
self._list_memories_sync,
|
self._list_memories_sync,
|
||||||
limit,
|
limit,
|
||||||
|
offset,
|
||||||
include_pending,
|
include_pending,
|
||||||
query,
|
query,
|
||||||
memory_type,
|
memory_type,
|
||||||
|
|
@ -3689,6 +3707,7 @@ class MemoryStore:
|
||||||
def _list_memories_sync(
|
def _list_memories_sync(
|
||||||
self,
|
self,
|
||||||
limit: int,
|
limit: int,
|
||||||
|
offset: int,
|
||||||
include_pending: bool,
|
include_pending: bool,
|
||||||
query: str,
|
query: str,
|
||||||
memory_type: str,
|
memory_type: str,
|
||||||
|
|
@ -3743,9 +3762,9 @@ class MemoryStore:
|
||||||
SELECT * FROM memories
|
SELECT * FROM memories
|
||||||
WHERE {where}
|
WHERE {where}
|
||||||
ORDER BY occurred_at DESC, created_at DESC
|
ORDER BY occurred_at DESC, created_at DESC
|
||||||
LIMIT ?
|
LIMIT ? OFFSET ?
|
||||||
""",
|
""",
|
||||||
params + [max(1, int(limit))],
|
params + [max(1, int(limit)), max(0, int(offset))],
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return [MemoryRecord.from_row(row) for row in rows]
|
return [MemoryRecord.from_row(row) for row in rows]
|
||||||
|
|
||||||
|
|
@ -5243,6 +5262,8 @@ class MemoryStore:
|
||||||
result.append((MemoryRecord.from_row(row), vector, clean_text(row["embedding_text_hash"], 80)))
|
result.append((MemoryRecord.from_row(row), vector, clean_text(row["embedding_text_hash"], 80)))
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._embedding_candidate_cache[cache_key] = result
|
self._embedding_candidate_cache[cache_key] = result
|
||||||
|
while len(self._embedding_candidate_cache) > self.EMBEDDING_CANDIDATE_CACHE_MAX:
|
||||||
|
self._embedding_candidate_cache.pop(next(iter(self._embedding_candidate_cache)), None)
|
||||||
return deepcopy(result)
|
return deepcopy(result)
|
||||||
|
|
||||||
async def list_memories_missing_embeddings(
|
async def list_memories_missing_embeddings(
|
||||||
|
|
|
||||||
|
|
@ -192,6 +192,8 @@ const state = {
|
||||||
historicalChatPreviewSource: "",
|
historicalChatPreviewSource: "",
|
||||||
historicalChatBatchId: "",
|
historicalChatBatchId: "",
|
||||||
historicalChatPollTimer: 0,
|
historicalChatPollTimer: 0,
|
||||||
|
historicalChatPollAttempts: 0,
|
||||||
|
historicalChatPollStartedAt: 0,
|
||||||
historicalChatFile: null,
|
historicalChatFile: null,
|
||||||
conversationImportSource: "qq",
|
conversationImportSource: "qq",
|
||||||
qqHistoryCapabilities: null,
|
qqHistoryCapabilities: null,
|
||||||
|
|
@ -5607,7 +5609,16 @@ async function refreshHistoricalChatStatus() {
|
||||||
|
|
||||||
function scheduleHistoricalChatPoll() {
|
function scheduleHistoricalChatPoll() {
|
||||||
stopHistoricalChatPoll();
|
stopHistoricalChatPoll();
|
||||||
|
state.historicalChatPollAttempts = 0;
|
||||||
|
state.historicalChatPollStartedAt = Date.now();
|
||||||
state.historicalChatPollTimer = window.setInterval(() => {
|
state.historicalChatPollTimer = window.setInterval(() => {
|
||||||
|
state.historicalChatPollAttempts += 1;
|
||||||
|
const elapsed = Date.now() - state.historicalChatPollStartedAt;
|
||||||
|
if (state.historicalChatPollAttempts > 120 || elapsed > 10 * 60 * 1000) {
|
||||||
|
stopHistoricalChatPoll();
|
||||||
|
showToast("历史导入轮询已达到上限,请手动刷新任务状态", "error");
|
||||||
|
return;
|
||||||
|
}
|
||||||
refreshHistoricalChatStatus().catch(() => stopHistoricalChatPoll());
|
refreshHistoricalChatStatus().catch(() => stopHistoricalChatPoll());
|
||||||
}, 3000);
|
}, 3000);
|
||||||
}
|
}
|
||||||
|
|
@ -5615,6 +5626,7 @@ function scheduleHistoricalChatPoll() {
|
||||||
function stopHistoricalChatPoll() {
|
function stopHistoricalChatPoll() {
|
||||||
if (state.historicalChatPollTimer) window.clearInterval(state.historicalChatPollTimer);
|
if (state.historicalChatPollTimer) window.clearInterval(state.historicalChatPollTimer);
|
||||||
state.historicalChatPollTimer = 0;
|
state.historicalChatPollTimer = 0;
|
||||||
|
state.historicalChatPollStartedAt = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function historicalChatSegmentStatusLabel(value) {
|
function historicalChatSegmentStatusLabel(value) {
|
||||||
|
|
|
||||||
186
tests/test_c7_import_retrieval.py
普通文件
186
tests/test_c7_import_retrieval.py
普通文件
|
|
@ -0,0 +1,186 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import types
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if "astrbot_plugin_remember_you" not in sys.modules:
|
||||||
|
package = types.ModuleType("astrbot_plugin_remember_you")
|
||||||
|
package.__path__ = [str(ROOT)]
|
||||||
|
sys.modules["astrbot_plugin_remember_you"] = package
|
||||||
|
if str(ROOT.parent) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT.parent))
|
||||||
|
|
||||||
|
from astrbot_plugin_remember_you.core.chat_import import HistoricalChatImporter
|
||||||
|
from astrbot_plugin_remember_you.core.models import MemoryRecord
|
||||||
|
from astrbot_plugin_remember_you.core.operations import PORTABLE_EXPORT_PAGE_SIZE, PortableMemoryArchive
|
||||||
|
from astrbot_plugin_remember_you.core.retrieval import RetrievalEngine
|
||||||
|
from astrbot_plugin_remember_you.core.store import MemoryStore
|
||||||
|
|
||||||
|
|
||||||
|
class _Config:
|
||||||
|
def int(self, _key: str, default: int) -> int:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
class _ResumeStore:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.batch = {"id": "batch-1", "state": "running", "checkpoint_segment": 7}
|
||||||
|
self.segments = [{"id": "seg-1", "segment_index": 1, "status": "processing", "attempts": 2}]
|
||||||
|
self.updates: list[tuple[str, dict]] = []
|
||||||
|
|
||||||
|
async def get_chat_import_batch(self, _batch_id: str):
|
||||||
|
return dict(self.batch)
|
||||||
|
|
||||||
|
async def chat_import_segments(self, _batch_id: str, *, statuses=None):
|
||||||
|
if statuses is None:
|
||||||
|
return list(self.segments)
|
||||||
|
return [item for item in self.segments if item["status"] in statuses]
|
||||||
|
|
||||||
|
async def update_chat_import_segment(self, segment_id: str, **changes):
|
||||||
|
self.updates.append((segment_id, changes))
|
||||||
|
for item in self.segments:
|
||||||
|
if item["id"] == segment_id:
|
||||||
|
item.update(changes)
|
||||||
|
return dict(self.segments[0])
|
||||||
|
|
||||||
|
async def update_chat_import_batch(self, _batch_id: str, **changes):
|
||||||
|
self.batch.update(changes)
|
||||||
|
return dict(self.batch)
|
||||||
|
|
||||||
|
|
||||||
|
class C7ImportRecoveryTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
def test_corrupt_manifest_has_stable_redacted_error(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
|
||||||
|
upload_id = "chatup_" + "a" * 24
|
||||||
|
upload_dir = importer._upload_dir(upload_id)
|
||||||
|
upload_dir.mkdir(parents=True)
|
||||||
|
(upload_dir / "manifest.json").write_text("{broken", encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏") as raised:
|
||||||
|
importer.preview_upload(upload_id)
|
||||||
|
self.assertNotIn(str(upload_dir), str(raised.exception))
|
||||||
|
|
||||||
|
def test_corrupt_parsed_jsonl_does_not_become_empty_import(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
|
||||||
|
upload_id = "chatup_" + "b" * 24
|
||||||
|
upload_dir = importer._upload_dir(upload_id)
|
||||||
|
upload_dir.mkdir(parents=True)
|
||||||
|
(upload_dir / "manifest.json").write_text(
|
||||||
|
json.dumps({"upload_id": upload_id}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
(upload_dir / "parsed.jsonl").write_text('{"message_id":"ok"}\n{broken\n', encoding="utf-8")
|
||||||
|
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏"):
|
||||||
|
importer._load_upload_messages(upload_id)
|
||||||
|
|
||||||
|
async def test_processing_segment_is_retried_only_without_active_worker(self) -> None:
|
||||||
|
store = _ResumeStore()
|
||||||
|
service = SimpleNamespace(data_dir=Path(tempfile.gettempdir()), store=store, config=_Config())
|
||||||
|
importer = HistoricalChatImporter(service)
|
||||||
|
importer._start_worker = lambda _batch_id: None
|
||||||
|
result = await importer.resume_batch("batch-1")
|
||||||
|
self.assertEqual("retry", store.segments[0]["status"])
|
||||||
|
self.assertEqual("running", result["batch"]["state"])
|
||||||
|
self.assertEqual(0, store.segments[0]["attempts"])
|
||||||
|
|
||||||
|
|
||||||
|
class C7RetrievalBoundaryTests(unittest.TestCase):
|
||||||
|
def test_materialize_limit_is_explicit_and_bounded(self) -> None:
|
||||||
|
self.assertGreater(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 0)
|
||||||
|
self.assertLessEqual(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 2000)
|
||||||
|
|
||||||
|
def test_panel_polling_has_attempt_and_time_bounds(self) -> None:
|
||||||
|
panel = (ROOT / "pages" / "记忆面板" / "app.js").read_text(encoding="utf-8")
|
||||||
|
self.assertIn("historicalChatPollAttempts > 120", panel)
|
||||||
|
self.assertIn("elapsed > 10 * 60 * 1000", panel)
|
||||||
|
self.assertIn("历史导入轮询已达到上限", panel)
|
||||||
|
|
||||||
|
|
||||||
|
class C7PerformanceBoundaryTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_embedding_candidate_cache_is_bounded(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
store = MemoryStore(Path(temp) / "memory.db")
|
||||||
|
store.initialize()
|
||||||
|
try:
|
||||||
|
for index in range(store.EMBEDDING_CANDIDATE_CACHE_MAX + 1):
|
||||||
|
await store.list_embedding_candidate_rows(
|
||||||
|
provider_id=f"provider-{index}",
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
self.assertLessEqual(
|
||||||
|
len(store._embedding_candidate_cache),
|
||||||
|
store.EMBEDDING_CANDIDATE_CACHE_MAX,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
async def test_portable_export_pages_large_memory_sets_without_materializing_all_rows(self) -> None:
|
||||||
|
records = [
|
||||||
|
MemoryRecord(
|
||||||
|
id=f"memory-{index}",
|
||||||
|
memory_type="user_fact",
|
||||||
|
scope="private",
|
||||||
|
visibility="private_pair",
|
||||||
|
content=f"content-{index}",
|
||||||
|
)
|
||||||
|
for index in range(PORTABLE_EXPORT_PAGE_SIZE + 1)
|
||||||
|
]
|
||||||
|
|
||||||
|
class PagedStore:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.memory_calls: list[tuple[int, int]] = []
|
||||||
|
|
||||||
|
async def stats(self):
|
||||||
|
return {
|
||||||
|
"total_memories": len(records),
|
||||||
|
"identities": 0,
|
||||||
|
"relationships": 0,
|
||||||
|
"timeline_events": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def list_memories(self, *, limit, offset=0):
|
||||||
|
self.memory_calls.append((limit, offset))
|
||||||
|
return records[offset : offset + limit]
|
||||||
|
|
||||||
|
async def list_identities(self, *, limit, offset=0):
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def list_relationships(self, *, limit, offset=0):
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def recent_timeline(self, *, limit, offset=0):
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def list_acl_rules(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def list_acl_policies(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
store = PagedStore()
|
||||||
|
result = await PortableMemoryArchive(store, Path(temp)).export()
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
(PORTABLE_EXPORT_PAGE_SIZE, 0),
|
||||||
|
(PORTABLE_EXPORT_PAGE_SIZE, PORTABLE_EXPORT_PAGE_SIZE),
|
||||||
|
],
|
||||||
|
store.memory_calls,
|
||||||
|
)
|
||||||
|
self.assertEqual(len(records), result["counts"]["memory"])
|
||||||
|
self.assertEqual(
|
||||||
|
len(records) + 1,
|
||||||
|
len(Path(result["path"]).read_text(encoding="utf-8").splitlines()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
正在加载…
添加表格
在新工单中引用