fix(c7): bound import export and retrieval work
这个提交存在于:
父节点
4b0219eac2
当前提交
aae4565c84
共有 7 个文件被更改,包括 294 次插入 和 36 次删除
|
|
@ -637,6 +637,7 @@ class HistoricalChatImporter:
|
|||
IDENTITY_LINKS_VERSION = 2
|
||||
SUMMARY_PERSPECTIVE_VERSION = 1
|
||||
DETAIL_SCHEMA_VERSION = 1
|
||||
STAGED_DATA_ERROR = "历史导入暂存数据损坏,请重新上传或清理该预览"
|
||||
|
||||
def __init__(self, service: Any) -> None:
|
||||
self.service = service
|
||||
|
|
@ -1276,17 +1277,32 @@ class HistoricalChatImporter:
|
|||
manifest_path = self._upload_dir(upload_id) / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise ValueError("上传预览不存在或已被清理")
|
||||
return json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(self.STAGED_DATA_ERROR) from exc
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError(self.STAGED_DATA_ERROR)
|
||||
if clean_text(manifest.get("upload_id"), 120) != clean_text(upload_id, 120):
|
||||
raise ValueError(self.STAGED_DATA_ERROR)
|
||||
return manifest
|
||||
|
||||
def _load_upload_messages(self, upload_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
manifest = self.preview_upload(upload_id)
|
||||
parsed_path = self._upload_dir(upload_id) / "parsed.jsonl"
|
||||
messages: list[dict[str, Any]] = []
|
||||
for raw in parsed_path.read_text(encoding="utf-8").splitlines():
|
||||
if raw.strip():
|
||||
try:
|
||||
for line_no, raw in enumerate(parsed_path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if not raw.strip():
|
||||
continue
|
||||
value = json.loads(raw)
|
||||
if isinstance(value, dict):
|
||||
messages.append(value)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"line={line_no}")
|
||||
messages.append(value)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
|
||||
raise ValueError(self.STAGED_DATA_ERROR) from exc
|
||||
if not messages:
|
||||
raise ValueError(self.STAGED_DATA_ERROR)
|
||||
return manifest, messages
|
||||
|
||||
def _private_api(self) -> Any | None:
|
||||
|
|
@ -1986,7 +2002,12 @@ class HistoricalChatImporter:
|
|||
batch["id"], statuses={"completed", "archived_only"}
|
||||
)
|
||||
memory_counts = await self.store.chat_import_memory_counts(batch["id"])
|
||||
checkpoint = max((int(item.get("segment_index") or 0) + 1 for item in completed_segments), default=0)
|
||||
current_checkpoint = max(0, int(batch.get("checkpoint_segment") or 0))
|
||||
computed_checkpoint = max(
|
||||
(int(item.get("segment_index") or 0) + 1 for item in completed_segments),
|
||||
default=0,
|
||||
)
|
||||
checkpoint = max(current_checkpoint, computed_checkpoint)
|
||||
await self.store.update_chat_import_batch(
|
||||
batch["id"],
|
||||
checkpoint_segment=checkpoint,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from .models import EntityRef, MemoryRecord, clean_text, json_loads
|
|||
PORTABLE_FORMAT = "astrbot-memory-jsonl"
|
||||
PORTABLE_VERSION = 1
|
||||
MAX_PORTABLE_BYTES = 64 * 1024 * 1024
|
||||
PORTABLE_EXPORT_PAGE_SIZE = 500
|
||||
|
||||
|
||||
PRESETS: dict[str, dict[str, Any]] = {
|
||||
|
|
@ -302,17 +303,14 @@ class PortableMemoryArchive:
|
|||
self.export_dir = self.data_dir / "exports"
|
||||
|
||||
async def export(self) -> dict[str, Any]:
|
||||
memories = await self.store.list_memories(limit=1_000_000)
|
||||
identities = await self.store.list_identities(limit=1_000_000)
|
||||
relationships = await self.store.list_relationships(limit=1_000_000)
|
||||
timeline = await self.store.recent_timeline(limit=1_000_000)
|
||||
stats = await self.store.stats()
|
||||
acl_rules = await self.store.list_acl_rules()
|
||||
acl_policies = await self.store.list_acl_policies()
|
||||
counts = {
|
||||
"memory": len(memories),
|
||||
"identity": len(identities),
|
||||
"relationship": len(relationships),
|
||||
"timeline": len(timeline),
|
||||
"memory": int(stats.get("total_memories") or 0),
|
||||
"identity": int(stats.get("identities") or 0),
|
||||
"relationship": int(stats.get("relationships") or 0),
|
||||
"timeline": int(stats.get("timeline_events") or 0),
|
||||
"acl_rule": len(acl_rules),
|
||||
"acl_policy": len(acl_policies),
|
||||
}
|
||||
|
|
@ -333,15 +331,26 @@ class PortableMemoryArchive:
|
|||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
self._write_line(handle, header)
|
||||
for memory in memories:
|
||||
self._write_line(handle, {"record_type": "memory", "data": self._memory_payload(memory)})
|
||||
for kind, rows in (
|
||||
("identity", identities),
|
||||
("relationship", relationships),
|
||||
("timeline", timeline),
|
||||
("acl_rule", acl_rules),
|
||||
("acl_policy", acl_policies),
|
||||
for record_type, loader, normalizer in (
|
||||
("memory", self.store.list_memories, self._memory_payload),
|
||||
("identity", self.store.list_identities, self._normalized_row),
|
||||
("relationship", self.store.list_relationships, self._normalized_row),
|
||||
("timeline", self.store.recent_timeline, self._normalized_row),
|
||||
):
|
||||
offset = 0
|
||||
while True:
|
||||
rows = await loader(limit=PORTABLE_EXPORT_PAGE_SIZE, offset=offset)
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
self._write_line(
|
||||
handle,
|
||||
{"record_type": record_type, "data": normalizer(row)},
|
||||
)
|
||||
offset += len(rows)
|
||||
if len(rows) < PORTABLE_EXPORT_PAGE_SIZE:
|
||||
break
|
||||
for kind, rows in (("acl_rule", acl_rules), ("acl_policy", acl_policies)):
|
||||
for row in rows:
|
||||
self._write_line(handle, {"record_type": kind, "data": self._normalized_row(row)})
|
||||
os.replace(temp_name, path)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from .visibility import VisibilityPolicy
|
|||
|
||||
|
||||
class RetrievalEngine:
|
||||
DEFAULT_MATERIALIZE_LIMIT = 2000
|
||||
def __init__(
|
||||
self,
|
||||
store: MemoryStore,
|
||||
|
|
@ -900,7 +901,11 @@ class RetrievalEngine:
|
|||
)
|
||||
expanded_terms = self._merge_terms(terms, graph_terms)
|
||||
include_pending = not self.policy.hide_pending_review
|
||||
ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending)
|
||||
# Keep the broad priority scan explicitly bounded.
|
||||
ranked_candidates = await self.store.list_candidate_memories(
|
||||
limit=self.DEFAULT_MATERIALIZE_LIMIT,
|
||||
include_pending=include_pending,
|
||||
)
|
||||
current_window_candidates = await self.store.list_current_window_candidate_memories(
|
||||
scope=ctx.scope,
|
||||
session_id=ctx.session_id,
|
||||
|
|
|
|||
|
|
@ -2444,9 +2444,13 @@ class MemoryCompanionService:
|
|||
provider_id = clean_text(provider_id, 160)
|
||||
indexed = 0
|
||||
try:
|
||||
batch_size = max(
|
||||
1,
|
||||
min(200, self.config.int("retrieval.embedding_backfill_batch_size", 50)),
|
||||
)
|
||||
records = await self.store.list_memories_missing_embeddings(
|
||||
provider_id=provider_id,
|
||||
limit=self.config.int("retrieval.embedding_backfill_batch_size", 50),
|
||||
limit=batch_size,
|
||||
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
|
||||
)
|
||||
for record in records:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads,
|
|||
|
||||
|
||||
class MemoryStore:
|
||||
EMBEDDING_CANDIDATE_CACHE_MAX = 64
|
||||
|
||||
def __init__(self, db_path: Path):
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -1421,14 +1423,14 @@ class MemoryStore:
|
|||
self._conn.commit()
|
||||
return row_id
|
||||
|
||||
async def list_identities(self, limit: int = 1000) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(self._list_identities_sync, limit)
|
||||
async def list_identities(self, limit: int = 1000, *, offset: int = 0) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(self._list_identities_sync, limit, offset)
|
||||
|
||||
def _list_identities_sync(self, limit: int) -> list[dict[str, Any]]:
|
||||
def _list_identities_sync(self, limit: int, offset: int) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ?",
|
||||
(max(1, int(limit)),),
|
||||
"SELECT * FROM identities ORDER BY updated_at DESC LIMIT ? OFFSET ?",
|
||||
(max(1, int(limit)), max(0, int(offset))),
|
||||
).fetchall()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
|
|
@ -1563,6 +1565,8 @@ class MemoryStore:
|
|||
scope: str = "",
|
||||
session_id: str = "",
|
||||
group_id: str = "",
|
||||
*,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(
|
||||
self._list_relationships_sync,
|
||||
|
|
@ -1571,6 +1575,7 @@ class MemoryStore:
|
|||
scope,
|
||||
session_id,
|
||||
group_id,
|
||||
offset,
|
||||
)
|
||||
|
||||
def _list_relationships_sync(
|
||||
|
|
@ -1580,6 +1585,7 @@ class MemoryStore:
|
|||
scope: str,
|
||||
session_id: str,
|
||||
group_id: str,
|
||||
offset: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
params: list[Any] = []
|
||||
where = "1=1"
|
||||
|
|
@ -1601,9 +1607,9 @@ class MemoryStore:
|
|||
SELECT * FROM relationship_edges
|
||||
WHERE {where}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ?
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
params + [max(1, int(limit))],
|
||||
params + [max(1, int(limit)), max(0, int(offset))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
|
@ -2684,8 +2690,17 @@ class MemoryStore:
|
|||
scope: str = "",
|
||||
session_id: str = "",
|
||||
entity_id: str = "",
|
||||
*,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(self._recent_timeline_sync, limit, scope, session_id, entity_id)
|
||||
return await asyncio.to_thread(
|
||||
self._recent_timeline_sync,
|
||||
limit,
|
||||
scope,
|
||||
session_id,
|
||||
entity_id,
|
||||
offset,
|
||||
)
|
||||
|
||||
def _recent_timeline_sync(
|
||||
self,
|
||||
|
|
@ -2693,6 +2708,7 @@ class MemoryStore:
|
|||
scope: str,
|
||||
session_id: str,
|
||||
entity_id: str,
|
||||
offset: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
params: list[Any] = []
|
||||
where = "1=1"
|
||||
|
|
@ -2711,9 +2727,9 @@ class MemoryStore:
|
|||
SELECT * FROM timeline
|
||||
WHERE {where}
|
||||
ORDER BY occurred_at DESC, created_at DESC
|
||||
LIMIT ?
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
params + [max(1, int(limit))],
|
||||
params + [max(1, int(limit)), max(0, int(offset))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
|
@ -3660,6 +3676,7 @@ class MemoryStore:
|
|||
self,
|
||||
*,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
include_pending: bool = True,
|
||||
query: str = "",
|
||||
memory_type: str = "",
|
||||
|
|
@ -3674,6 +3691,7 @@ class MemoryStore:
|
|||
return await self._run_recoverable_database_operation(
|
||||
self._list_memories_sync,
|
||||
limit,
|
||||
offset,
|
||||
include_pending,
|
||||
query,
|
||||
memory_type,
|
||||
|
|
@ -3689,6 +3707,7 @@ class MemoryStore:
|
|||
def _list_memories_sync(
|
||||
self,
|
||||
limit: int,
|
||||
offset: int,
|
||||
include_pending: bool,
|
||||
query: str,
|
||||
memory_type: str,
|
||||
|
|
@ -3743,9 +3762,9 @@ class MemoryStore:
|
|||
SELECT * FROM memories
|
||||
WHERE {where}
|
||||
ORDER BY occurred_at DESC, created_at DESC
|
||||
LIMIT ?
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
params + [max(1, int(limit))],
|
||||
params + [max(1, int(limit)), max(0, int(offset))],
|
||||
).fetchall()
|
||||
return [MemoryRecord.from_row(row) for row in rows]
|
||||
|
||||
|
|
@ -5243,6 +5262,8 @@ class MemoryStore:
|
|||
result.append((MemoryRecord.from_row(row), vector, clean_text(row["embedding_text_hash"], 80)))
|
||||
with self._lock:
|
||||
self._embedding_candidate_cache[cache_key] = result
|
||||
while len(self._embedding_candidate_cache) > self.EMBEDDING_CANDIDATE_CACHE_MAX:
|
||||
self._embedding_candidate_cache.pop(next(iter(self._embedding_candidate_cache)), None)
|
||||
return deepcopy(result)
|
||||
|
||||
async def list_memories_missing_embeddings(
|
||||
|
|
|
|||
|
|
@ -192,6 +192,8 @@ const state = {
|
|||
historicalChatPreviewSource: "",
|
||||
historicalChatBatchId: "",
|
||||
historicalChatPollTimer: 0,
|
||||
historicalChatPollAttempts: 0,
|
||||
historicalChatPollStartedAt: 0,
|
||||
historicalChatFile: null,
|
||||
conversationImportSource: "qq",
|
||||
qqHistoryCapabilities: null,
|
||||
|
|
@ -5607,7 +5609,16 @@ async function refreshHistoricalChatStatus() {
|
|||
|
||||
function scheduleHistoricalChatPoll() {
|
||||
stopHistoricalChatPoll();
|
||||
state.historicalChatPollAttempts = 0;
|
||||
state.historicalChatPollStartedAt = Date.now();
|
||||
state.historicalChatPollTimer = window.setInterval(() => {
|
||||
state.historicalChatPollAttempts += 1;
|
||||
const elapsed = Date.now() - state.historicalChatPollStartedAt;
|
||||
if (state.historicalChatPollAttempts > 120 || elapsed > 10 * 60 * 1000) {
|
||||
stopHistoricalChatPoll();
|
||||
showToast("历史导入轮询已达到上限,请手动刷新任务状态", "error");
|
||||
return;
|
||||
}
|
||||
refreshHistoricalChatStatus().catch(() => stopHistoricalChatPoll());
|
||||
}, 3000);
|
||||
}
|
||||
|
|
@ -5615,6 +5626,7 @@ function scheduleHistoricalChatPoll() {
|
|||
function stopHistoricalChatPoll() {
|
||||
if (state.historicalChatPollTimer) window.clearInterval(state.historicalChatPollTimer);
|
||||
state.historicalChatPollTimer = 0;
|
||||
state.historicalChatPollStartedAt = 0;
|
||||
}
|
||||
|
||||
function historicalChatSegmentStatusLabel(value) {
|
||||
|
|
|
|||
186
tests/test_c7_import_retrieval.py
普通文件
186
tests/test_c7_import_retrieval.py
普通文件
|
|
@ -0,0 +1,186 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if "astrbot_plugin_remember_you" not in sys.modules:
|
||||
package = types.ModuleType("astrbot_plugin_remember_you")
|
||||
package.__path__ = [str(ROOT)]
|
||||
sys.modules["astrbot_plugin_remember_you"] = package
|
||||
if str(ROOT.parent) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT.parent))
|
||||
|
||||
from astrbot_plugin_remember_you.core.chat_import import HistoricalChatImporter
|
||||
from astrbot_plugin_remember_you.core.models import MemoryRecord
|
||||
from astrbot_plugin_remember_you.core.operations import PORTABLE_EXPORT_PAGE_SIZE, PortableMemoryArchive
|
||||
from astrbot_plugin_remember_you.core.retrieval import RetrievalEngine
|
||||
from astrbot_plugin_remember_you.core.store import MemoryStore
|
||||
|
||||
|
||||
class _Config:
|
||||
def int(self, _key: str, default: int) -> int:
|
||||
return default
|
||||
|
||||
|
||||
class _ResumeStore:
|
||||
def __init__(self) -> None:
|
||||
self.batch = {"id": "batch-1", "state": "running", "checkpoint_segment": 7}
|
||||
self.segments = [{"id": "seg-1", "segment_index": 1, "status": "processing", "attempts": 2}]
|
||||
self.updates: list[tuple[str, dict]] = []
|
||||
|
||||
async def get_chat_import_batch(self, _batch_id: str):
|
||||
return dict(self.batch)
|
||||
|
||||
async def chat_import_segments(self, _batch_id: str, *, statuses=None):
|
||||
if statuses is None:
|
||||
return list(self.segments)
|
||||
return [item for item in self.segments if item["status"] in statuses]
|
||||
|
||||
async def update_chat_import_segment(self, segment_id: str, **changes):
|
||||
self.updates.append((segment_id, changes))
|
||||
for item in self.segments:
|
||||
if item["id"] == segment_id:
|
||||
item.update(changes)
|
||||
return dict(self.segments[0])
|
||||
|
||||
async def update_chat_import_batch(self, _batch_id: str, **changes):
|
||||
self.batch.update(changes)
|
||||
return dict(self.batch)
|
||||
|
||||
|
||||
class C7ImportRecoveryTests(unittest.IsolatedAsyncioTestCase):
|
||||
def test_corrupt_manifest_has_stable_redacted_error(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
|
||||
upload_id = "chatup_" + "a" * 24
|
||||
upload_dir = importer._upload_dir(upload_id)
|
||||
upload_dir.mkdir(parents=True)
|
||||
(upload_dir / "manifest.json").write_text("{broken", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏") as raised:
|
||||
importer.preview_upload(upload_id)
|
||||
self.assertNotIn(str(upload_dir), str(raised.exception))
|
||||
|
||||
def test_corrupt_parsed_jsonl_does_not_become_empty_import(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
importer = HistoricalChatImporter(SimpleNamespace(data_dir=Path(temp), store=object()))
|
||||
upload_id = "chatup_" + "b" * 24
|
||||
upload_dir = importer._upload_dir(upload_id)
|
||||
upload_dir.mkdir(parents=True)
|
||||
(upload_dir / "manifest.json").write_text(
|
||||
json.dumps({"upload_id": upload_id}), encoding="utf-8"
|
||||
)
|
||||
(upload_dir / "parsed.jsonl").write_text('{"message_id":"ok"}\n{broken\n', encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, "历史导入暂存数据损坏"):
|
||||
importer._load_upload_messages(upload_id)
|
||||
|
||||
async def test_processing_segment_is_retried_only_without_active_worker(self) -> None:
|
||||
store = _ResumeStore()
|
||||
service = SimpleNamespace(data_dir=Path(tempfile.gettempdir()), store=store, config=_Config())
|
||||
importer = HistoricalChatImporter(service)
|
||||
importer._start_worker = lambda _batch_id: None
|
||||
result = await importer.resume_batch("batch-1")
|
||||
self.assertEqual("retry", store.segments[0]["status"])
|
||||
self.assertEqual("running", result["batch"]["state"])
|
||||
self.assertEqual(0, store.segments[0]["attempts"])
|
||||
|
||||
|
||||
class C7RetrievalBoundaryTests(unittest.TestCase):
|
||||
def test_materialize_limit_is_explicit_and_bounded(self) -> None:
|
||||
self.assertGreater(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 0)
|
||||
self.assertLessEqual(RetrievalEngine.DEFAULT_MATERIALIZE_LIMIT, 2000)
|
||||
|
||||
def test_panel_polling_has_attempt_and_time_bounds(self) -> None:
|
||||
panel = (ROOT / "pages" / "记忆面板" / "app.js").read_text(encoding="utf-8")
|
||||
self.assertIn("historicalChatPollAttempts > 120", panel)
|
||||
self.assertIn("elapsed > 10 * 60 * 1000", panel)
|
||||
self.assertIn("历史导入轮询已达到上限", panel)
|
||||
|
||||
|
||||
class C7PerformanceBoundaryTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_embedding_candidate_cache_is_bounded(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
store = MemoryStore(Path(temp) / "memory.db")
|
||||
store.initialize()
|
||||
try:
|
||||
for index in range(store.EMBEDDING_CANDIDATE_CACHE_MAX + 1):
|
||||
await store.list_embedding_candidate_rows(
|
||||
provider_id=f"provider-{index}",
|
||||
limit=1,
|
||||
)
|
||||
self.assertLessEqual(
|
||||
len(store._embedding_candidate_cache),
|
||||
store.EMBEDDING_CANDIDATE_CACHE_MAX,
|
||||
)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
async def test_portable_export_pages_large_memory_sets_without_materializing_all_rows(self) -> None:
|
||||
records = [
|
||||
MemoryRecord(
|
||||
id=f"memory-{index}",
|
||||
memory_type="user_fact",
|
||||
scope="private",
|
||||
visibility="private_pair",
|
||||
content=f"content-{index}",
|
||||
)
|
||||
for index in range(PORTABLE_EXPORT_PAGE_SIZE + 1)
|
||||
]
|
||||
|
||||
class PagedStore:
|
||||
def __init__(self) -> None:
|
||||
self.memory_calls: list[tuple[int, int]] = []
|
||||
|
||||
async def stats(self):
|
||||
return {
|
||||
"total_memories": len(records),
|
||||
"identities": 0,
|
||||
"relationships": 0,
|
||||
"timeline_events": 0,
|
||||
}
|
||||
|
||||
async def list_memories(self, *, limit, offset=0):
|
||||
self.memory_calls.append((limit, offset))
|
||||
return records[offset : offset + limit]
|
||||
|
||||
async def list_identities(self, *, limit, offset=0):
|
||||
return []
|
||||
|
||||
async def list_relationships(self, *, limit, offset=0):
|
||||
return []
|
||||
|
||||
async def recent_timeline(self, *, limit, offset=0):
|
||||
return []
|
||||
|
||||
async def list_acl_rules(self):
|
||||
return []
|
||||
|
||||
async def list_acl_policies(self):
|
||||
return []
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
store = PagedStore()
|
||||
result = await PortableMemoryArchive(store, Path(temp)).export()
|
||||
self.assertEqual(
|
||||
[
|
||||
(PORTABLE_EXPORT_PAGE_SIZE, 0),
|
||||
(PORTABLE_EXPORT_PAGE_SIZE, PORTABLE_EXPORT_PAGE_SIZE),
|
||||
],
|
||||
store.memory_calls,
|
||||
)
|
||||
self.assertEqual(len(records), result["counts"]["memory"])
|
||||
self.assertEqual(
|
||||
len(records) + 1,
|
||||
len(Path(result["path"]).read_text(encoding="utf-8").splitlines()),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
正在加载…
添加表格
在新工单中引用