diff --git a/core/chat_import.py b/core/chat_import.py index 9ab8559..96b77e3 100644 --- a/core/chat_import.py +++ b/core/chat_import.py @@ -637,6 +637,7 @@ class HistoricalChatImporter: IDENTITY_LINKS_VERSION = 2 SUMMARY_PERSPECTIVE_VERSION = 1 DETAIL_SCHEMA_VERSION = 1 + STAGED_DATA_ERROR = "历史导入暂存数据损坏,请重新上传或清理该预览" def __init__(self, service: Any) -> None: self.service = service @@ -1276,17 +1277,32 @@ class HistoricalChatImporter: manifest_path = self._upload_dir(upload_id) / "manifest.json" if not manifest_path.is_file(): raise ValueError("上传预览不存在或已被清理") - return json.loads(manifest_path.read_text(encoding="utf-8")) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(self.STAGED_DATA_ERROR) from exc + if not isinstance(manifest, dict): + raise ValueError(self.STAGED_DATA_ERROR) + if clean_text(manifest.get("upload_id"), 120) != clean_text(upload_id, 120): + raise ValueError(self.STAGED_DATA_ERROR) + return manifest def _load_upload_messages(self, upload_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]: manifest = self.preview_upload(upload_id) parsed_path = self._upload_dir(upload_id) / "parsed.jsonl" messages: list[dict[str, Any]] = [] - for raw in parsed_path.read_text(encoding="utf-8").splitlines(): - if raw.strip(): + try: + for line_no, raw in enumerate(parsed_path.read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue value = json.loads(raw) - if isinstance(value, dict): - messages.append(value) + if not isinstance(value, dict): + raise ValueError(f"line={line_no}") + messages.append(value) + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + raise ValueError(self.STAGED_DATA_ERROR) from exc + if not messages: + raise ValueError(self.STAGED_DATA_ERROR) return manifest, messages def _private_api(self) -> Any | None: @@ -1986,7 +2002,12 @@ class HistoricalChatImporter: batch["id"], statuses={"completed", "archived_only"} ) memory_counts = await self.store.chat_import_memory_counts(batch["id"]) - checkpoint = max((int(item.get("segment_index") or 0) + 1 for item in completed_segments), default=0) + current_checkpoint = max(0, int(batch.get("checkpoint_segment") or 0)) + computed_checkpoint = max( + (int(item.get("segment_index") or 0) + 1 for item in completed_segments), + default=0, + ) + checkpoint = max(current_checkpoint, computed_checkpoint) await self.store.update_chat_import_batch( batch["id"], checkpoint_segment=checkpoint, diff --git a/core/operations.py b/core/operations.py index bbd1385..6fb818d 100644 --- a/core/operations.py +++ b/core/operations.py @@ -15,6 +15,7 @@ from .models import EntityRef, MemoryRecord, clean_text, json_loads PORTABLE_FORMAT = "astrbot-memory-jsonl" PORTABLE_VERSION = 1 MAX_PORTABLE_BYTES = 64 * 1024 * 1024 +PORTABLE_EXPORT_PAGE_SIZE = 500 PRESETS: dict[str, dict[str, Any]] = { @@ -302,17 +303,14 @@ class PortableMemoryArchive: self.export_dir = self.data_dir / "exports" async def export(self) -> dict[str, Any]: - memories = await self.store.list_memories(limit=1_000_000) - identities = await self.store.list_identities(limit=1_000_000) - relationships = await self.store.list_relationships(limit=1_000_000) - timeline = await self.store.recent_timeline(limit=1_000_000) + stats = await self.store.stats() acl_rules = await self.store.list_acl_rules() acl_policies = await self.store.list_acl_policies() counts = { - "memory": len(memories), - "identity": len(identities), - "relationship": len(relationships), - "timeline": len(timeline), + "memory": int(stats.get("total_memories") or 0), + "identity": int(stats.get("identities") or 0), + "relationship": int(stats.get("relationships") or 0), + "timeline": int(stats.get("timeline_events") or 0), "acl_rule": len(acl_rules), "acl_policy": len(acl_policies), } @@ -333,15 +331,26 @@ class PortableMemoryArchive: try: with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: self._write_line(handle, header) - for memory in memories: - self._write_line(handle, {"record_type": "memory", "data": self._memory_payload(memory)}) - for kind, rows in ( - ("identity", identities), - ("relationship", relationships), - ("timeline", timeline), - ("acl_rule", acl_rules), - ("acl_policy", acl_policies), + for record_type, loader, normalizer in ( + ("memory", self.store.list_memories, self._memory_payload), + ("identity", self.store.list_identities, self._normalized_row), + ("relationship", self.store.list_relationships, self._normalized_row), + ("timeline", self.store.recent_timeline, self._normalized_row), ): + offset = 0 + while True: + rows = await loader(limit=PORTABLE_EXPORT_PAGE_SIZE, offset=offset) + if not rows: + break + for row in rows: + self._write_line( + handle, + {"record_type": record_type, "data": normalizer(row)}, + ) + offset += len(rows) + if len(rows) < PORTABLE_EXPORT_PAGE_SIZE: + break + for kind, rows in (("acl_rule", acl_rules), ("acl_policy", acl_policies)): for row in rows: self._write_line(handle, {"record_type": kind, "data": self._normalized_row(row)}) os.replace(temp_name, path) diff --git a/core/retrieval.py b/core/retrieval.py index 0dcee27..85df7d9 100644 --- a/core/retrieval.py +++ b/core/retrieval.py @@ -16,6 +16,7 @@ from .visibility import VisibilityPolicy class RetrievalEngine: + DEFAULT_MATERIALIZE_LIMIT = 2000 def __init__( self, store: MemoryStore, @@ -900,7 +901,11 @@ class RetrievalEngine: ) expanded_terms = self._merge_terms(terms, graph_terms) include_pending = not self.policy.hide_pending_review - ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending) + # Keep the broad priority scan explicitly bounded. + ranked_candidates = await self.store.list_candidate_memories( + limit=self.DEFAULT_MATERIALIZE_LIMIT, + include_pending=include_pending, + ) current_window_candidates = await self.store.list_current_window_candidate_memories( scope=ctx.scope, session_id=ctx.session_id, diff --git a/core/service.py b/core/service.py index e96554b..4bac0cf 100644 --- a/core/service.py +++ b/core/service.py @@ -2444,9 +2444,13 @@ class MemoryCompanionService: provider_id = clean_text(provider_id, 160) indexed = 0 try: + batch_size = max( + 1, + min(200, self.config.int("retrieval.embedding_backfill_batch_size", 50)), + ) records = await self.store.list_memories_missing_embeddings( provider_id=provider_id, - limit=self.config.int("retrieval.embedding_backfill_batch_size", 50), + limit=batch_size, include_pending=self.config.bool("retrieval.embedding_index_pending", False), ) for record in records: diff --git a/core/store.py b/core/store.py index dafd828..9440b27 100644 --- a/core/store.py +++ b/core/store.py @@ -14,6 +14,8 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads, class MemoryStore: + EMBEDDING_CANDIDATE_CACHE_MAX = 64 + def __init__(self, db_path: Path): self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) @@ -1421,14 +1423,14 @@ class MemoryStore: self._conn.commit() return row_id - async def list_identities(self, limit: int = 1000) -> list[dict[str, Any]]: - return await asyncio.to_thread(self._list_identities_sync, limit) + async def list_identities(self, limit: int = 1000, *, offset: int = 0) -> list[dict[str, Any]]: + return await asyncio.to_thread(self._list_identities_sync, limit, offset) - def _list_identities_sync(self, limit: int) -> list[dict[str, Any]]: + def _list_identities_sync(self, limit: int, offset: int) -> list[dict[str, Any]]: with self._lock: rows = self._conn.execute( - "SELECT * FROM identities ORDER BY updated_at DESC LIMIT ?", - (max(1, int(limit)),), + "SELECT * FROM identities ORDER BY updated_at DESC LIMIT ? OFFSET ?", + (max(1, int(limit)), max(0, int(offset))), ).fetchall() result: list[dict[str, Any]] = [] for row in rows: @@ -1563,6 +1565,8 @@ class MemoryStore: scope: str = "", session_id: str = "", group_id: str = "", + *, + offset: int = 0, ) -> list[dict[str, Any]]: return await asyncio.to_thread( self._list_relationships_sync, @@ -1571,6 +1575,7 @@ class MemoryStore: scope, session_id, group_id, + offset, ) def _list_relationships_sync( @@ -1580,6 +1585,7 @@ class MemoryStore: scope: str, session_id: str, group_id: str, + offset: int, ) -> list[dict[str, Any]]: params: list[Any] = [] where = "1=1" @@ -1601,9 +1607,9 @@ class MemoryStore: SELECT * FROM relationship_edges WHERE {where} ORDER BY updated_at DESC - LIMIT ? + LIMIT ? OFFSET ? """, - params + [max(1, int(limit))], + params + [max(1, int(limit)), max(0, int(offset))], ).fetchall() return [dict(row) for row in rows] @@ -2684,8 +2690,17 @@ class MemoryStore: scope: str = "", session_id: str = "", entity_id: str = "", + *, + offset: int = 0, ) -> list[dict[str, Any]]: - return await asyncio.to_thread(self._recent_timeline_sync, limit, scope, session_id, entity_id) + return await asyncio.to_thread( + self._recent_timeline_sync, + limit, + scope, + session_id, + entity_id, + offset, + ) def _recent_timeline_sync( self, @@ -2693,6 +2708,7 @@ class MemoryStore: scope: str, session_id: str, entity_id: str, + offset: int, ) -> list[dict[str, Any]]: params: list[Any] = [] where = "1=1" @@ -2711,9 +2727,9 @@ class MemoryStore: SELECT * FROM timeline WHERE {where} ORDER BY occurred_at DESC, created_at DESC - LIMIT ? + LIMIT ? OFFSET ? """, - params + [max(1, int(limit))], + params + [max(1, int(limit)), max(0, int(offset))], ).fetchall() return [dict(row) for row in rows] @@ -3660,6 +3676,7 @@ class MemoryStore: self, *, limit: int = 50, + offset: int = 0, include_pending: bool = True, query: str = "", memory_type: str = "", @@ -3674,6 +3691,7 @@ class MemoryStore: return await self._run_recoverable_database_operation( self._list_memories_sync, limit, + offset, include_pending, query, memory_type, @@ -3689,6 +3707,7 @@ class MemoryStore: def _list_memories_sync( self, limit: int, + offset: int, include_pending: bool, query: str, memory_type: str, @@ -3743,9 +3762,9 @@ class MemoryStore: SELECT * FROM memories WHERE {where} ORDER BY occurred_at DESC, created_at DESC - LIMIT ? + LIMIT ? OFFSET ? """, - params + [max(1, int(limit))], + params + [max(1, int(limit)), max(0, int(offset))], ).fetchall() return [MemoryRecord.from_row(row) for row in rows] @@ -5243,6 +5262,8 @@ class MemoryStore: result.append((MemoryRecord.from_row(row), vector, clean_text(row["embedding_text_hash"], 80))) with self._lock: self._embedding_candidate_cache[cache_key] = result + while len(self._embedding_candidate_cache) > self.EMBEDDING_CANDIDATE_CACHE_MAX: + self._embedding_candidate_cache.pop(next(iter(self._embedding_candidate_cache)), None) return deepcopy(result) async def list_memories_missing_embeddings( diff --git a/pages/记忆面板/app.js b/pages/记忆面板/app.js index cc97d3e..7d44ce8 100644 --- a/pages/记忆面板/app.js +++ b/pages/记忆面板/app.js @@ -192,6 +192,8 @@ const state = { historicalChatPreviewSource: "", historicalChatBatchId: "", historicalChatPollTimer: 0, + historicalChatPollAttempts: 0, + historicalChatPollStartedAt: 0, historicalChatFile: null, conversationImportSource: "qq", qqHistoryCapabilities: null, @@ -5607,7 +5609,16 @@ async function refreshHistoricalChatStatus() { function scheduleHistoricalChatPoll() { stopHistoricalChatPoll(); + state.historicalChatPollAttempts = 0; + state.historicalChatPollStartedAt = Date.now(); state.historicalChatPollTimer = window.setInterval(() => { + state.historicalChatPollAttempts += 1; + const elapsed = Date.now() - state.historicalChatPollStartedAt; + if (state.historicalChatPollAttempts > 120 || elapsed > 10 * 60 * 1000) { + stopHistoricalChatPoll(); + showToast("历史导入轮询已达到上限,请手动刷新任务状态", "error"); + return; + } refreshHistoricalChatStatus().catch(() => stopHistoricalChatPoll()); }, 3000); } @@ -5615,6 +5626,7 @@ function scheduleHistoricalChatPoll() { function stopHistoricalChatPoll() { if (state.historicalChatPollTimer) window.clearInterval(state.historicalChatPollTimer); state.historicalChatPollTimer = 0; + state.historicalChatPollStartedAt = 0; } function historicalChatSegmentStatusLabel(value) { diff --git a/tests/test_c7_import_retrieval.py b/tests/test_c7_import_retrieval.py new file mode 100644 index 0000000..f373932 --- /dev/null +++ b/tests/test_c7_import_retrieval.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import asyncio +import json +import sys +import tempfile +import types +import unittest +from pathlib import Path +from types import SimpleNamespace + + +ROOT = Path(__file__).resolve().parents[1] +if "astrbot_plugin_remember_you" not in sys.modules: + package = types.ModuleType("astrbot_plugin_remember_you") + package.__path__ = [str(ROOT)] + sys.modules["astrbot_plugin_remember_you"] = package +if str(ROOT.parent) not in sys.path: + sys.path.insert(0, str(ROOT.parent)) + +from astrbot_plugin_remember_you.core.chat_import import HistoricalChatImporter +from astrbot_plugin_remember_you.core.models import MemoryRecord +from astrbot_plugin_remember_you.core.operations import PORTABLE_EXPORT_PAGE_SIZE, PortableMemoryArchive +from astrbot_plugin_remember_you.core.retrieval import RetrievalEngine +from astrbot_plugin_remember_you.core.store import MemoryStore + + +class _Config: + def int(self, _key: str, default: int) -> int: + return default + + +class _ResumeStore: + def __init__(self) -> None: + self.batch = {"id": "batch-1", "state": "running", "checkpoint_segment": 7} + self.segments = [{"id": "seg-1", "segment_index": 1, "status": "processing", "attempts": 2}] + self.updates: list[tuple[str, dict]] = [] + + async def get_chat_import_batch(self, _batch_id: str): + return dict(self.batch) + + async def chat_import_segments(self, _batch_id: str, *, statuses=None): + if statuses is None: + return list(self.segments) + return [item for item in self.segments if item["status"] in statuses] + + async def update_chat_import_segment(self, segment_id: str, **changes): + self.updates.append((segment_id, changes)) + for item in self.segments: + if item["id"] == segment_id: + item.update(changes) + return dict(self.segments[0]) + + async def update_chat_import_batch(self, _batch_id: str, **changes): + self.batch.update(changes) + return dict(self.batch) + + +class C7ImportRecoveryTests(unittest.IsolatedAsyncioTestCase): + def test_corrupt_manifest_has_stable_redacted_error(self) -> None: + with tempfile.TemporaryDirectory() as temp: + importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object())) + upload_id = "chatup_" + "a" * 24 + upload_dir = importer._upload_dir(upload_id) + upload_dir.mkdir(parents=True) + (upload_dir / "manifest.json").write_text("{broken", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏") as raised: + importer.preview_upload(upload_id) + self.assertNotIn(str(upload_dir), str(raised.exception)) + + def test_corrupt_parsed_jsonl_does_not_become_empty_import(self) -> None: + with tempfile.TemporaryDirectory() as temp: + importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object())) + upload_id = "chatup_" + "b" * 24 + upload_dir = importer._upload_dir(upload_id) + upload_dir.mkdir(parents=True) + (upload_dir / "manifest.json").write_text( + json.dumps({"upload_id": upload_id}), encoding="utf-8" + ) + (upload_dir / "parsed.jsonl").write_text('{"message_id":"ok"}\n{broken\n', encoding="utf-8") + with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏"): + importer._load_upload_messages(upload_id) + + async def test_processing_segment_is_retried_only_without_active_worker(self) -> None: + store = _ResumeStore() + service = SimpleNamespace(data_dir=Path(tempfile.gettempdir()), store=store, config=_Config()) + importer = HistoricalChatImporter(service) + importer._start_worker = lambda _batch_id: None + result = await importer.resume_batch("batch-1") + self.assertEqual("retry", store.segments[0]["status"]) + self.assertEqual("running", result["batch"]["state"]) + self.assertEqual(0, store.segments[0]["attempts"]) + + +class C7RetrievalBoundaryTests(unittest.TestCase): + def test_materialize_limit_is_explicit_and_bounded(self) -> None: + self.assertGreater(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 0) + self.assertLessEqual(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 2000) + + def test_panel_polling_has_attempt_and_time_bounds(self) -> None: + panel = (ROOT / "pages" / "记忆面板" / "app.js").read_text(encoding="utf-8") + self.assertIn("historicalChatPollAttempts > 120", panel) + self.assertIn("elapsed > 10 * 60 * 1000", panel) + self.assertIn("历史导入轮询已达到上限", panel) + + +class C7PerformanceBoundaryTests(unittest.IsolatedAsyncioTestCase): + async def test_embedding_candidate_cache_is_bounded(self) -> None: + with tempfile.TemporaryDirectory() as temp: + store = MemoryStore(Path(temp) / "memory.db") + store.initialize() + try: + for index in range(store.EMBEDDING_CANDIDATE_CACHE_MAX + 1): + await store.list_embedding_candidate_rows( + provider_id=f"provider-{index}", + limit=1, + ) + self.assertLessEqual( + len(store._embedding_candidate_cache), + store.EMBEDDING_CANDIDATE_CACHE_MAX, + ) + finally: + store.close() + + async def test_portable_export_pages_large_memory_sets_without_materializing_all_rows(self) -> None: + records = [ + MemoryRecord( + id=f"memory-{index}", + memory_type="user_fact", + scope="private", + visibility="private_pair", + content=f"content-{index}", + ) + for index in range(PORTABLE_EXPORT_PAGE_SIZE + 1) + ] + + class PagedStore: + def __init__(self) -> None: + self.memory_calls: list[tuple[int, int]] = [] + + async def stats(self): + return { + "total_memories": len(records), + "identities": 0, + "relationships": 0, + "timeline_events": 0, + } + + async def list_memories(self, *, limit, offset=0): + self.memory_calls.append((limit, offset)) + return records[offset : offset + limit] + + async def list_identities(self, *, limit, offset=0): + return [] + + async def list_relationships(self, *, limit, offset=0): + return [] + + async def recent_timeline(self, *, limit, offset=0): + return [] + + async def list_acl_rules(self): + return [] + + async def list_acl_policies(self): + return [] + + with tempfile.TemporaryDirectory() as temp: + store = PagedStore() + result = await PortableMemoryArchive(store, Path(temp)).export() + self.assertEqual( + [ + (PORTABLE_EXPORT_PAGE_SIZE, 0), + (PORTABLE_EXPORT_PAGE_SIZE, PORTABLE_EXPORT_PAGE_SIZE), + ], + store.memory_calls, + ) + self.assertEqual(len(records), result["counts"]["memory"]) + self.assertEqual( + len(records) + 1, + len(Path(result["path"]).read_text(encoding="utf-8").splitlines()), + ) + + +if __name__ == "__main__": + unittest.main()