fix(c7): bound import export and retrieval work

这个提交存在于:
Codex Local 2026-07-30 08:24:09 +08:00
父节点 4b0219eac2
当前提交 aae4565c84
共有 7 个文件被更改,包括 294 次插入 和 36 次删除

查看文件

@ -637,6 +637,7 @@ class HistoricalChatImporter:
IDENTITY_LINKS_VERSION = 2
SUMMARY_PERSPECTIVE_VERSION = 1
DETAIL_SCHEMA_VERSION = 1
STAGED_DATA_ERROR = "历史导入暂存数据损坏,请重新上传或清理该预览"
def __init__(self, service: Any) -> None:
self.service = service
@ -1276,17 +1277,32 @@ class HistoricalChatImporter:
manifest_path = self._upload_dir(upload_id) / "manifest.json"
if not manifest_path.is_file():
raise ValueError("上传预览不存在或已被清理")
return json.loads(manifest_path.read_text(encoding="utf-8"))
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
raise ValueError(self.STAGED_DATA_ERROR) from exc
if not isinstance(manifest, dict):
raise ValueError(self.STAGED_DATA_ERROR)
if clean_text(manifest.get("upload_id"), 120) != clean_text(upload_id, 120):
raise ValueError(self.STAGED_DATA_ERROR)
return manifest
def _load_upload_messages(self, upload_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]:
manifest = self.preview_upload(upload_id)
parsed_path = self._upload_dir(upload_id) / "parsed.jsonl"
messages: list[dict[str, Any]] = []
for raw in parsed_path.read_text(encoding="utf-8").splitlines():
if raw.strip():
try:
for line_no, raw in enumerate(parsed_path.read_text(encoding="utf-8").splitlines(), 1):
if not raw.strip():
continue
value = json.loads(raw)
if isinstance(value, dict):
messages.append(value)
if not isinstance(value, dict):
raise ValueError(f"line={line_no}")
messages.append(value)
except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
raise ValueError(self.STAGED_DATA_ERROR) from exc
if not messages:
raise ValueError(self.STAGED_DATA_ERROR)
return manifest, messages
def _private_api(self) -> Any | None:
@ -1986,7 +2002,12 @@ class HistoricalChatImporter:
batch["id"], statuses={"completed", "archived_only"}
)
memory_counts = await self.store.chat_import_memory_counts(batch["id"])
checkpoint = max((int(item.get("segment_index") or 0) + 1 for item in completed_segments), default=0)
current_checkpoint = max(0, int(batch.get("checkpoint_segment") or 0))
computed_checkpoint = max(
(int(item.get("segment_index") or 0) + 1 for item in completed_segments),
default=0,
)
checkpoint = max(current_checkpoint, computed_checkpoint)
await self.store.update_chat_import_batch(
batch["id"],
checkpoint_segment=checkpoint,

查看文件

@ -15,6 +15,7 @@ from .models import EntityRef, MemoryRecord, clean_text, json_loads
PORTABLE_FORMAT = "astrbot-memory-jsonl"
PORTABLE_VERSION = 1
MAX_PORTABLE_BYTES = 64 * 1024 * 1024
PORTABLE_EXPORT_PAGE_SIZE = 500
PRESETS: dict[str, dict[str, Any]] = {
@ -302,17 +303,14 @@ class PortableMemoryArchive:
self.export_dir = self.data_dir / "exports"
async def export(self) -> dict[str, Any]:
memories = await self.store.list_memories(limit=1_000_000)
identities = await self.store.list_identities(limit=1_000_000)
relationships = await self.store.list_relationships(limit=1_000_000)
timeline = await self.store.recent_timeline(limit=1_000_000)
stats = await self.store.stats()
acl_rules = await self.store.list_acl_rules()
acl_policies = await self.store.list_acl_policies()
counts = {
"memory": len(memories),
"identity": len(identities),
"relationship": len(relationships),
"timeline": len(timeline),
"memory": int(stats.get("total_memories") or 0),
"identity": int(stats.get("identities") or 0),
"relationship": int(stats.get("relationships") or 0),
"timeline": int(stats.get("timeline_events") or 0),
"acl_rule": len(acl_rules),
"acl_policy": len(acl_policies),
}
@ -333,15 +331,26 @@ class PortableMemoryArchive:
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
self._write_line(handle, header)
for memory in memories:
self._write_line(handle, {"record_type": "memory", "data": self._memory_payload(memory)})
for kind, rows in (
("identity", identities),
("relationship", relationships),
("timeline", timeline),
("acl_rule", acl_rules),
("acl_policy", acl_policies),
for record_type, loader, normalizer in (
("memory", self.store.list_memories, self._memory_payload),
("identity", self.store.list_identities, self._normalized_row),
("relationship", self.store.list_relationships, self._normalized_row),
("timeline", self.store.recent_timeline, self._normalized_row),
):
offset = 0
while True:
rows = await loader(limit=PORTABLE_EXPORT_PAGE_SIZE, offset=offset)
if not rows:
break
for row in rows:
self._write_line(
handle,
{"record_type": record_type, "data": normalizer(row)},
)
offset += len(rows)
if len(rows) < PORTABLE_EXPORT_PAGE_SIZE:
break
for kind, rows in (("acl_rule", acl_rules), ("acl_policy", acl_policies)):
for row in rows:
self._write_line(handle, {"record_type": kind, "data": self._normalized_row(row)})
os.replace(temp_name, path)

查看文件

@ -16,6 +16,7 @@ from .visibility import VisibilityPolicy
class RetrievalEngine:
DEFAULT_MATERIALIZE_LIMIT = 2000
def __init__(
self,
store: MemoryStore,
@ -900,7 +901,11 @@ class RetrievalEngine:
)
expanded_terms = self._merge_terms(terms, graph_terms)
include_pending = not self.policy.hide_pending_review
ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending)
# Keep the broad priority scan explicitly bounded.
ranked_candidates = await self.store.list_candidate_memories(
limit=self.DEFAULT_MATERIALIZE_LIMIT,
include_pending=include_pending,
)
current_window_candidates = await self.store.list_current_window_candidate_memories(
scope=ctx.scope,
session_id=ctx.session_id,

查看文件

@ -2444,9 +2444,13 @@ class MemoryCompanionService:
provider_id = clean_text(provider_id, 160)
indexed = 0
try:
batch_size = max(
1,
min(200, self.config.int("retrieval.embedding_backfill_batch_size", 50)),
)
records = await self.store.list_memories_missing_embeddings(
provider_id=provider_id,
limit=self.config.int("retrieval.embedding_backfill_batch_size", 50),
limit=batch_size,
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
)
for record in records:

查看文件

@ -14,6 +14,8 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads,
class MemoryStore:
EMBEDDING_CANDIDATE_CACHE_MAX = 64
def __init__(self, db_path: Path):
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True)
@ -1421,14 +1423,14 @@ class MemoryStore:
self._conn.commit()
return row_id
async def list_identities(self, limit: int = 1000) -> list[dict[str, Any]]:
return await asyncio.to_thread(self._list_identities_sync, limit)
async def list_identities(self, limit: int = 1000, *, offset: int = 0) -> list[dict[str, Any]]:
return await asyncio.to_thread(self._list_identities_sync, limit, offset)
def _list_identities_sync(self, limit: int) -> list[dict[str, Any]]:
def _list_identities_sync(self, limit: int, offset: int) -> list[dict[str, Any]]:
with self._lock:
rows = self._conn.execute(
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ?",
(max(1, int(limit)),),
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ? OFFSET ?",
(max(1, int(limit)), max(0, int(offset))),
).fetchall()
result: list[dict[str, Any]] = []
for row in rows:
@ -1563,6 +1565,8 @@ class MemoryStore:
scope: str = "",
session_id: str = "",
group_id: str = "",
*,
offset: int = 0,
) -> list[dict[str, Any]]:
return await asyncio.to_thread(
self._list_relationships_sync,
@ -1571,6 +1575,7 @@ class MemoryStore:
scope,
session_id,
group_id,
offset,
)
def _list_relationships_sync(
@ -1580,6 +1585,7 @@ class MemoryStore:
scope: str,
session_id: str,
group_id: str,
offset: int,
) -> list[dict[str, Any]]:
params: list[Any] = []
where = "1=1"
@ -1601,9 +1607,9 @@ class MemoryStore:
SELECT * FROM relationship_edges
WHERE {where}
ORDER BY updated_at DESC
LIMIT ?
LIMIT ? OFFSET ?
""",
params + [max(1, int(limit))],
params + [max(1, int(limit)), max(0, int(offset))],
).fetchall()
return [dict(row) for row in rows]
@ -2684,8 +2690,17 @@ class MemoryStore:
scope: str = "",
session_id: str = "",
entity_id: str = "",
*,
offset: int = 0,
) -> list[dict[str, Any]]:
return await asyncio.to_thread(self._recent_timeline_sync, limit, scope, session_id, entity_id)
return await asyncio.to_thread(
self._recent_timeline_sync,
limit,
scope,
session_id,
entity_id,
offset,
)
def _recent_timeline_sync(
self,
@ -2693,6 +2708,7 @@ class MemoryStore:
scope: str,
session_id: str,
entity_id: str,
offset: int,
) -> list[dict[str, Any]]:
params: list[Any] = []
where = "1=1"
@ -2711,9 +2727,9 @@ class MemoryStore:
SELECT * FROM timeline
WHERE {where}
ORDER BY occurred_at DESC, created_at DESC
LIMIT ?
LIMIT ? OFFSET ?
""",
params + [max(1, int(limit))],
params + [max(1, int(limit)), max(0, int(offset))],
).fetchall()
return [dict(row) for row in rows]
@ -3660,6 +3676,7 @@ class MemoryStore:
self,
*,
limit: int = 50,
offset: int = 0,
include_pending: bool = True,
query: str = "",
memory_type: str = "",
@ -3674,6 +3691,7 @@ class MemoryStore:
return await self._run_recoverable_database_operation(
self._list_memories_sync,
limit,
offset,
include_pending,
query,
memory_type,
@ -3689,6 +3707,7 @@ class MemoryStore:
def _list_memories_sync(
self,
limit: int,
offset: int,
include_pending: bool,
query: str,
memory_type: str,
@ -3743,9 +3762,9 @@ class MemoryStore:
SELECT * FROM memories
WHERE {where}
ORDER BY occurred_at DESC, created_at DESC
LIMIT ?
LIMIT ? OFFSET ?
""",
params + [max(1, int(limit))],
params + [max(1, int(limit)), max(0, int(offset))],
).fetchall()
return [MemoryRecord.from_row(row) for row in rows]
@ -5243,6 +5262,8 @@ class MemoryStore:
result.append((MemoryRecord.from_row(row), vector, clean_text(row["embedding_text_hash"], 80)))
with self._lock:
self._embedding_candidate_cache[cache_key] = result
while len(self._embedding_candidate_cache) > self.EMBEDDING_CANDIDATE_CACHE_MAX:
self._embedding_candidate_cache.pop(next(iter(self._embedding_candidate_cache)), None)
return deepcopy(result)
async def list_memories_missing_embeddings(

查看文件

@ -192,6 +192,8 @@ const state = {
historicalChatPreviewSource: "",
historicalChatBatchId: "",
historicalChatPollTimer: 0,
historicalChatPollAttempts: 0,
historicalChatPollStartedAt: 0,
historicalChatFile: null,
conversationImportSource: "qq",
qqHistoryCapabilities: null,
@ -5607,7 +5609,16 @@ async function refreshHistoricalChatStatus() {
function scheduleHistoricalChatPoll() {
stopHistoricalChatPoll();
state.historicalChatPollAttempts = 0;
state.historicalChatPollStartedAt = Date.now();
state.historicalChatPollTimer = window.setInterval(() => {
state.historicalChatPollAttempts += 1;
const elapsed = Date.now() - state.historicalChatPollStartedAt;
if (state.historicalChatPollAttempts > 120 || elapsed > 10 * 60 * 1000) {
stopHistoricalChatPoll();
showToast("历史导入轮询已达到上限,请手动刷新任务状态", "error");
return;
}
refreshHistoricalChatStatus().catch(() => stopHistoricalChatPoll());
}, 3000);
}
@ -5615,6 +5626,7 @@ function scheduleHistoricalChatPoll() {
function stopHistoricalChatPoll() {
if (state.historicalChatPollTimer) window.clearInterval(state.historicalChatPollTimer);
state.historicalChatPollTimer = 0;
state.historicalChatPollStartedAt = 0;
}
function historicalChatSegmentStatusLabel(value) {

查看文件

@ -0,0 +1,186 @@
from __future__ import annotations
import asyncio
import json
import sys
import tempfile
import types
import unittest
from pathlib import Path
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
if "astrbot_plugin_remember_you" not in sys.modules:
package = types.ModuleType("astrbot_plugin_remember_you")
package.__path__ = [str(ROOT)]
sys.modules["astrbot_plugin_remember_you"] = package
if str(ROOT.parent) not in sys.path:
sys.path.insert(0, str(ROOT.parent))
from astrbot_plugin_remember_you.core.chat_import import HistoricalChatImporter
from astrbot_plugin_remember_you.core.models import MemoryRecord
from astrbot_plugin_remember_you.core.operations import PORTABLE_EXPORT_PAGE_SIZE, PortableMemoryArchive
from astrbot_plugin_remember_you.core.retrieval import RetrievalEngine
from astrbot_plugin_remember_you.core.store import MemoryStore
class _Config:
def int(self, _key: str, default: int) -> int:
return default
class _ResumeStore:
def __init__(self) -> None:
self.batch = {"id": "batch-1", "state": "running", "checkpoint_segment": 7}
self.segments = [{"id": "seg-1", "segment_index": 1, "status": "processing", "attempts": 2}]
self.updates: list[tuple[str, dict]] = []
async def get_chat_import_batch(self, _batch_id: str):
return dict(self.batch)
async def chat_import_segments(self, _batch_id: str, *, statuses=None):
if statuses is None:
return list(self.segments)
return [item for item in self.segments if item["status"] in statuses]
async def update_chat_import_segment(self, segment_id: str, **changes):
self.updates.append((segment_id, changes))
for item in self.segments:
if item["id"] == segment_id:
item.update(changes)
return dict(self.segments[0])
async def update_chat_import_batch(self, _batch_id: str, **changes):
self.batch.update(changes)
return dict(self.batch)
class C7ImportRecoveryTests(unittest.IsolatedAsyncioTestCase):
def test_corrupt_manifest_has_stable_redacted_error(self) -> None:
with tempfile.TemporaryDirectory() as temp:
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
upload_id = "chatup_" + "a" * 24
upload_dir = importer._upload_dir(upload_id)
upload_dir.mkdir(parents=True)
(upload_dir / "manifest.json").write_text("{broken", encoding="utf-8")
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏") as raised:
importer.preview_upload(upload_id)
self.assertNotIn(str(upload_dir), str(raised.exception))
def test_corrupt_parsed_jsonl_does_not_become_empty_import(self) -> None:
with tempfile.TemporaryDirectory() as temp:
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
upload_id = "chatup_" + "b" * 24
upload_dir = importer._upload_dir(upload_id)
upload_dir.mkdir(parents=True)
(upload_dir / "manifest.json").write_text(
json.dumps({"upload_id": upload_id}), encoding="utf-8"
)
(upload_dir / "parsed.jsonl").write_text('{"message_id":"ok"}\n{broken\n', encoding="utf-8")
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏"):
importer._load_upload_messages(upload_id)
async def test_processing_segment_is_retried_only_without_active_worker(self) -> None:
store = _ResumeStore()
service = SimpleNamespace(data_dir=Path(tempfile.gettempdir()), store=store, config=_Config())
importer = HistoricalChatImporter(service)
importer._start_worker = lambda _batch_id: None
result = await importer.resume_batch("batch-1")
self.assertEqual("retry", store.segments[0]["status"])
self.assertEqual("running", result["batch"]["state"])
self.assertEqual(0, store.segments[0]["attempts"])
class C7RetrievalBoundaryTests(unittest.TestCase):
def test_materialize_limit_is_explicit_and_bounded(self) -> None:
self.assertGreater(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 0)
self.assertLessEqual(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 2000)
def test_panel_polling_has_attempt_and_time_bounds(self) -> None:
panel = (ROOT / "pages" / "记忆面板" / "app.js").read_text(encoding="utf-8")
self.assertIn("historicalChatPollAttempts > 120", panel)
self.assertIn("elapsed > 10 * 60 * 1000", panel)
self.assertIn("历史导入轮询已达到上限", panel)
class C7PerformanceBoundaryTests(unittest.IsolatedAsyncioTestCase):
async def test_embedding_candidate_cache_is_bounded(self) -> None:
with tempfile.TemporaryDirectory() as temp:
store = MemoryStore(Path(temp) / "memory.db")
store.initialize()
try:
for index in range(store.EMBEDDING_CANDIDATE_CACHE_MAX + 1):
await store.list_embedding_candidate_rows(
provider_id=f"provider-{index}",
limit=1,
)
self.assertLessEqual(
len(store._embedding_candidate_cache),
store.EMBEDDING_CANDIDATE_CACHE_MAX,
)
finally:
store.close()
async def test_portable_export_pages_large_memory_sets_without_materializing_all_rows(self) -> None:
records = [
MemoryRecord(
id=f"memory-{index}",
memory_type="user_fact",
scope="private",
visibility="private_pair",
content=f"content-{index}",
)
for index in range(PORTABLE_EXPORT_PAGE_SIZE + 1)
]
class PagedStore:
def __init__(self) -> None:
self.memory_calls: list[tuple[int, int]] = []
async def stats(self):
return {
"total_memories": len(records),
"identities": 0,
"relationships": 0,
"timeline_events": 0,
}
async def list_memories(self, *, limit, offset=0):
self.memory_calls.append((limit, offset))
return records[offset : offset + limit]
async def list_identities(self, *, limit, offset=0):
return []
async def list_relationships(self, *, limit, offset=0):
return []
async def recent_timeline(self, *, limit, offset=0):
return []
async def list_acl_rules(self):
return []
async def list_acl_policies(self):
return []
with tempfile.TemporaryDirectory() as temp:
store = PagedStore()
result = await PortableMemoryArchive(store, Path(temp)).export()
self.assertEqual(
[
(PORTABLE_EXPORT_PAGE_SIZE, 0),
(PORTABLE_EXPORT_PAGE_SIZE, PORTABLE_EXPORT_PAGE_SIZE),
],
store.memory_calls,
)
self.assertEqual(len(records), result["counts"]["memory"])
self.assertEqual(
len(records) + 1,
len(Path(result["path"]).read_text(encoding="utf-8").splitlines()),
)
if __name__ == "__main__":
unittest.main()