diff --git a/_conf_schema.json b/_conf_schema.json index d7d2c54..7915a97 100644 --- a/_conf_schema.json +++ b/_conf_schema.json @@ -875,6 +875,12 @@ "hint": "开启后,睡眠维护会筛选长期未访问、重要度较低的稳定记忆,先压缩成高层摘要,再归档原碎片。", "type": "bool", "default": true + }, + "memory_decay_dry_run": { + "description": "自然衰减试运行", + "hint": "开启后睡眠维护照常筛选候选并生成摘要,但不写入摘要也不归档原碎片,只在睡眠报告里输出本轮会压缩和归档哪些记忆。用于上线前核对影响面。注意仍会调用模型生成摘要,会消耗 token。", + "type": "bool", + "default": false } } }, diff --git a/core/service.py b/core/service.py index da07e3b..76324df 100644 --- a/core/service.py +++ b/core/service.py @@ -4115,6 +4115,8 @@ class MemoryCompanionService: return {"enabled": True, "reason": "already_running", "candidates": 0, "archived": 0, "summaries": 0} async with self._decay_lock: + # 试运行:照常筛候选、分桶、生成摘要,但不写库不归档,用于上线前核对影响面。 + dry_run = self.config.bool("maintenance.memory_decay_dry_run", False) max_candidates = max(1, self.config.int("maintenance.memory_decay_max_candidates", 120)) scan_limit = max(max_candidates * 12, self.config.int("maintenance.memory_decay_scan_limit", 2000)) pool = await self.store.list_decay_candidate_pool(limit=scan_limit) @@ -4130,6 +4132,7 @@ class MemoryCompanionService: if not candidates: return { "enabled": True, + "dry_run": dry_run, "scanned": len(pool), "candidates": 0, "archived": 0, @@ -4152,7 +4155,7 @@ class MemoryCompanionService: skipped_groups += 1 continue try: - result = await self._summarize_decay_group(group) + result = await self._summarize_decay_group(group, dry_run=dry_run) if result.get("summary_id"): summaries += 1 archived += int(result.get("archived") or 0) @@ -4169,6 +4172,7 @@ class MemoryCompanionService: return { "enabled": True, + "dry_run": dry_run, "scanned": len(pool), "candidates": len(candidates), "groups": len(groups), @@ -4289,6 +4293,9 @@ class MemoryCompanionService: clean_text(record.scope, 40) or "unknown", clean_text(owner, 160), clean_text(record.visibility, 40), + # 域必须进桶键:摘要要继承来源域才能保住可召回性, + # 同一个桶里混域会让「摘要该落哪个域」没有确定答案。 + clean_text(getattr(record, "memory_domain", ""), 60) or "unknown", ] ) buckets[key].append(item) @@ -4308,7 +4315,7 @@ class MemoryCompanionService: return entity.id return record.session_id or record.group_id or record.object.id or record.subject.id - async def _summarize_decay_group(self, group: dict[str, Any]) -> dict[str, Any]: + async def _summarize_decay_group(self, group: dict[str, Any], *, dry_run: bool = False) -> dict[str, Any]: items = list(group.get("items") or []) if not items: return {"bucket": group.get("bucket"), "summary_id": "", "archived": 0, "reason": "empty_group"} @@ -4346,9 +4353,16 @@ class MemoryCompanionService: first_at = clean_text(min((record.occurred_at or record.created_at for record in records if record.occurred_at or record.created_at), default=""), 80) last_at = clean_text(max((record.occurred_at or record.created_at for record in records if record.occurred_at or record.created_at), default=""), 80) evidence = "\n".join(clean_text(record.content, 220) for record in records[:8] if clean_text(record.content, 220)) + # 摘要必须显式继承来源片段的域。MemoryRecord.memory_domain 默认值 "unknown" 是真值, + # ensure_defaults 的分类分支(core/models.py 的 `if not self.memory_domain:`)只在调用方 + # 显式传 "" 时才跑,所以不传域就会落库成 unknown,被 _memory_domain_block_reason 判成 + # missing_user_subject 永久屏蔽;而此时源片段已被归档,等于把这条主线整段从召回面抹掉。 + summary_domain = clean_text(getattr(sample, "memory_domain", ""), 60) or "unknown" + source_block_reason = RetrievalEngine._memory_domain_block_reason(sample) summary_record = MemoryRecord( id=self.stable_id("decay_summary", group.get("bucket", ""), first_at, last_at, summary), memory_type="memory_decay_summary", + memory_domain=summary_domain, subject=self._decay_subject(sample), object=self._decay_object(sample), scope=sample.scope, @@ -4382,17 +4396,61 @@ class MemoryCompanionService: }, source_plugin="memory_companion", ) + if dry_run: + # 试运行只算不写:不落摘要、不归档原片段,只把这一轮会做什么原样报出来。 + logger.info( + "[MemoryCompanion] 睡眠衰减试运行(未写库未归档): bucket=%s domain=%s sources=%s", + group.get("bucket"), + summary_domain, + len(records), + ) + return { + "bucket": group.get("bucket"), + "summary_id": "", + "archived": 0, + "source_count": len(records), + "dry_run": True, + "memory_domain": summary_domain, + "planned_summary_id": summary_record.id, + "summary_preview": clean_text(summary, 240), + "would_archive_ids": [record.id for record in records], + } + self.importance.calibrate(summary_record, source="memory_decay_summary") summary_id = await self.store.insert_memory(summary_record) self._schedule_memory_embedding(summary_id, summary_record) + # 归档前先回读确认摘要真的召回得到:insert_memory 会按 content_fingerprint 把新摘要 + # 合并进已有行并返回那一行的 id,且合并不更新 memory_domain,历史遗留的 unknown 域旧摘要 + # 仍可能把这条新摘要吃掉。只要来源本来可召回而摘要不可召回,就宁可不归档, + # 也不能把主线记忆变成死数据。 + stored = (await self.store.get_memories_by_ids([summary_id])).get(summary_id) + summary_block_reason = ( + RetrievalEngine._memory_domain_block_reason(stored) if stored is not None else "summary_missing" + ) + if summary_block_reason and not source_block_reason: + logger.warning( + "[MemoryCompanion] 睡眠衰减摘要不可召回,已跳过归档: bucket=%s summary=%s reason=%s", + group.get("bucket"), + summary_id, + summary_block_reason, + ) + return { + "bucket": group.get("bucket"), + "summary_id": summary_id, + "archived": 0, + "source_count": len(records), + "memory_domain": summary_domain, + "reason": f"summary_not_retrievable:{summary_block_reason}", + } archived = await self.store.archive_memories( [record.id for record in records], reason="sleep_decay_consolidated", supersedes_id=summary_id, ) logger.info( - "[MemoryCompanion] 睡眠衰减已压缩归档: bucket=%s summary=%s archived=%s", + "[MemoryCompanion] 睡眠衰减已压缩归档: bucket=%s domain=%s summary=%s archived=%s", group.get("bucket"), + summary_domain, summary_id, archived, ) @@ -4401,6 +4459,7 @@ class MemoryCompanionService: "summary_id": summary_id, "archived": archived, "source_count": len(records), + "memory_domain": summary_domain, } def _decay_context(self, sample: MemoryRecord) -> SessionContext: diff --git a/docs/运维记忆错题本.md b/docs/运维记忆错题本.md index a2f3d7e..ff7c02e 100644 --- a/docs/运维记忆错题本.md +++ b/docs/运维记忆错题本.md @@ -288,3 +288,14 @@ - 修复或规避:`core/bot_profiles.py` 把七件事(域/类型/时间范围/条数/排序/注入字段/权限)全部固定在记忆侧;调用方只能选 Profile 名,`limit` 只能往小调不能往大调;注入字段走白名单投影,`serialize_memory` 带出的人格权重、指纹、幂等键一律不出现在结果里;capability 公布契约,两侧不必各自猜测限制。 - 预防规则:给出「专用入口」时,先列清楚哪些维度必须由提供方固定;每个维度都要有一条「调用方试图突破 → 被夹住」的测试,否则契约只是文档里的一句话。 - 验证证据:`test_every_profile_pins_all_seven_dimensions`、`test_limit_can_be_lowered_but_not_raised`、`test_projection_drops_internal_fields`、`test_profiles_never_return_user_memories`;联合 E2E S 组 13 项。 + +## 2026-07-27 — 默认值是真值,于是「没传就自动分类」那条分支从来没跑过 + +- 阶段:P0 生产缺陷清理 +- 类型:数据一致性 / 检索可用性 +- 现象:默认开启的睡眠衰减把一批稳定记忆压成一条 `memory_decay_summary` 落库、再把原片段改成 `archived`。摘要落库的 `memory_domain` 恒为 `unknown`,被 `RetrievalEngine._memory_domain_block_reason` 判成 `missing_user_subject`、在打分之前就被前置过滤;而源片段已归档、被 `lifecycle != 'archived'` 的候选查询排除。摘要召回不到 + 原文召回不到 = 整段主线消失,无告警、无回滚、无 dry-run,每跑一晚制造一批死数据。 +- 根因:两层。表层是 `_summarize_decay_group` 构造 `MemoryRecord` 时没传 `memory_domain`。深层是 `MemoryRecord.memory_domain` 的 dataclass 默认值写的是 `"unknown"`——**一个真值**,于是 `ensure_defaults` 里 `if not self.memory_domain:` 那条自动分类分支成了死分支,只有调用方显式传 `""` 才会执行。「不传就自动分类」这个心智模型与实际行为相反,而两者都不报错。 +- 修复或规避:摘要显式继承来源片段的域(`sample.memory_domain`),来源是 `user_memory` 就产出 `user_memory` 摘要,来源本来是 `bot_self_legacy`/`unknown` 则原样继承,隔离边界一点不放宽;分桶键加上域,避免同一个桶混域导致「摘要该落哪个域」没有确定答案;**归档前回读摘要、用检索侧同一个闸门校验可召回性**,只要来源本来召回得到而摘要召回不到就跳过归档并 WARNING——把「静默产生死数据」换成「有告警、原文仍在」;新增 `maintenance.memory_decay_dry_run` 试运行开关,照常筛选与生成摘要但不写库不归档。**没有动 `ensure_defaults` 的默认值**:那会改变所有未显式传域的写入点的归类,爆炸半径太大,且现有测试已把当前行为钉死。 +- 附带收获:`tests/test_security_cache.py` 连续五轮被排除的那 2 个失败用例,根因就是同一个陷阱——夹具 `group_memory()` 构造记录不传域 → 落库 `unknown` → `search` 恒返回空集。它们要测的是 ACL 与缓存失效,不是域分类;夹具补上显式域后 6 项全通过,该文件不必再排除在测试口径外。 +- 预防规则:给「缺省时自动推断」的字段设默认值,默认值必须是**假值**(`""`/`None`),否则推断分支永远不会执行,而且不报错。凡是「压缩/合并/归档」这类会让原始数据不可访问的操作,都要在丢弃原文之前回读一次产物、用**消费侧同一个判据**确认它真的可用——不能只断言「写进去了」。 +- 验证证据:`tests/test_sleep_decay_recall.py` 5 项(含端到端召回、按域分桶、非用户域仍正常清理且隔离不放宽、摘要不可召回则跳过归档并告警、dry-run 零写入)。反证:临时摘掉域继承后,端到端召回那条必红(`reason=missing_user_subject`),且新守卫当场拦下归档。全量 330 项通过(此前 323 通过 / 2 失败)。 diff --git a/tests/test_security_cache.py b/tests/test_security_cache.py index ec2f54d..e403ac6 100644 --- a/tests/test_security_cache.py +++ b/tests/test_security_cache.py @@ -47,6 +47,12 @@ class SecurityAndCacheTests(unittest.IsolatedAsyncioTestCase): def group_memory(content: str = "唯一锚点蓝风铃") -> MemoryRecord: return MemoryRecord( memory_type="conversation_summary", + # 必须显式给域:MemoryRecord.memory_domain 的默认值 "unknown" 是真值, + # ensure_defaults 里 `if not self.memory_domain:` 那条分类分支因此永不执行, + # 不传域就落库成 unknown,被 RetrievalEngine._memory_domain_block_reason + # 判成 missing_user_subject 整体屏蔽,search 恒返回空集——本组两个用例 + # 曾因此连续五轮被排除在测试口径外,而它们要测的是 ACL 与缓存失效,不是域分类。 + memory_domain="user_memory", subject=EntityRef(kind="user", id="group-user", name="群成员"), object=EntityRef(kind="group", id="g1", name="测试群"), scope="group", diff --git a/tests/test_sleep_decay_recall.py b/tests/test_sleep_decay_recall.py new file mode 100644 index 0000000..6503235 --- /dev/null +++ b/tests/test_sleep_decay_recall.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +"""睡眠衰减压缩后主线仍可召回(P0)。 + +默认开启的睡眠衰减把一批稳定记忆压成一条 `memory_decay_summary` 落库、 +再把原片段改成 `archived`。但构造摘要时没传 `memory_domain`,而 +`MemoryRecord.memory_domain` 的默认值 `"unknown"` 是**真值**—— +`ensure_defaults` 里 `if not self.memory_domain:` 那条分类分支因此是死分支, +摘要以 `unknown` 落库;检索侧 `_memory_domain_block_reason` 只放行 `user_memory`, +`unknown` 在打分之前就被前置过滤。而源片段此时已被归档、被候选查询排除。 + +结果:摘要召回不到 + 原文召回不到 = 整段主线消失,无告警、无回滚、无 dry-run。 +每跑一晚就制造一批死数据。 + +这一组的关键是**端到端召回断言**:不是「摘要写进去了」,而是「压缩之后还搜得到」。 +""" +from __future__ import annotations + +import logging +import sys +import tempfile +import types +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT.parent) not in sys.path: + sys.path.insert(0, str(ROOT.parent)) + +if "quart" not in sys.modules: + quart_stub = types.ModuleType("quart") + quart_stub.jsonify = lambda payload=None, **kwargs: payload or kwargs + quart_stub.request = SimpleNamespace(args={}, method="GET") + + async def _send_file(path): + return path + + quart_stub.send_file = _send_file + sys.modules["quart"] = quart_stub + +from core.models import EntityRef, MemoryRecord, SessionContext # noqa: E402 +from core.retrieval import RetrievalEngine # noqa: E402 +from core.service import MemoryCompanionService # noqa: E402 + +SESSION = "qq:FriendMessage:u1" +# 四条内容必须互不相同:insert_memory 按 content_fingerprint 去重合并, +# 内容相同会塌成一条,凑不满一个桶。每条各带一个唯一关键词。 +CONTENTS = [ + "主人昨天说最喜欢桂花糕,配茶刚好", + "主人提到藕粉要冲得稠一点才好喝", + "主人说杏仁酥容易碎,得小心拿", + "主人讲桂圆羹要炖够时间才出味", +] + + +def old_ts(days: int) -> str: + return (datetime.now(timezone.utc) - timedelta(days=days)).isoformat() + + +class SleepDecayRecallTests(unittest.IsolatedAsyncioTestCase): + def make_service(self, config: dict | None = None) -> MemoryCompanionService: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + service = MemoryCompanionService( + context=None, config=config or {}, plugin_root=ROOT, data_dir=Path(temp_dir.name)) + self.addCleanup(service.close) + self.stub_summary_provider(service) + return service + + @staticmethod + def stub_summary_provider(service) -> None: + async def attempts(*args, **kwargs): + return [{"provider": object(), "provider_id": "p1", "source": "test"}] + + async def summarize(*args, **kwargs): + return "主人爱吃桂花糕这类甜点,口味偏清甜" + + service._summary_provider_attempts = attempts + service._summarize_decay_records_with_provider = summarize + + @staticmethod + def source_record(content: str, domain: str = "user_memory") -> MemoryRecord: + return MemoryRecord( + memory_type="user_event", + memory_domain=domain, + subject=EntityRef(kind="user", id="u1", name="主人"), + scope="private", + session_id=SESSION, + platform="qq", + visibility="private_pair", + lifecycle="stable_memory", + content=content, + importance=0.4, + confidence=0.5, + occurred_at=old_ts(400), + last_accessed_at=old_ts(400), + ) + + async def seed(self, service, domain: str = "user_memory") -> list[str]: + ids = [] + for content in CONTENTS: + ids.append(await service.store.insert_memory(self.source_record(content, domain))) + return ids + + async def lifecycles(self, service, ids): + rows = await service.store.get_memories_by_ids(ids) + return {rows[i].lifecycle for i in ids if i in rows} + + async def test_summary_is_still_retrievable_after_compression(self): + """改前必红:摘要落 unknown 被域闸门屏蔽,源片段已归档,搜出来是空集。""" + service = self.make_service() + ids = await self.seed(service) + result = await service._run_memory_decay() + + self.assertEqual(result.get("summaries"), 1, result) + self.assertEqual(result.get("archived"), 4, result) + self.assertFalse(result.get("dry_run")) + + summary_id = result["reports"][0]["summary_id"] + stored = (await service.store.get_memories_by_ids([summary_id])).get(summary_id) + self.assertIsNotNone(stored) + self.assertEqual(stored.memory_domain, "user_memory") + self.assertEqual(RetrievalEngine._memory_domain_block_reason(stored), "") + + ctx = SessionContext(session_id=SESSION, scope="private", platform="qq", user_id="u1") + found = await service.search("桂花糕", ctx) + self.assertIn(summary_id, {item.memory.id for item in found}, + "压缩后的摘要召回不到,主线记忆等于消失") + + self.assertEqual(await self.lifecycles(service, ids), {"archived"}) + + async def test_groups_are_split_by_memory_domain(self): + """同 scope/owner/visibility 但不同域的候选不能混进一个桶, + 否则「摘要该继承哪个域」没有确定答案。""" + service = self.make_service() + records = [self.source_record(CONTENTS[0], "user_memory"), + self.source_record(CONTENTS[1], "bot_self_legacy")] + groups = service._decay_groups([{"record": r, "score": 2.0} for r in records]) + self.assertEqual(len(groups), 2) + for group in groups: + domains = {item["record"].memory_domain for item in group["items"]} + self.assertEqual(len(domains), 1, domains) + + async def test_non_user_domain_still_archives_and_stays_isolated(self): + """来源本来就召回不到的,正常清理不受守卫影响;隔离边界零放宽。""" + service = self.make_service() + await self.seed(service, "bot_self_legacy") + result = await service._run_memory_decay() + self.assertEqual(result.get("archived"), 4, result) + summary_id = result["reports"][0]["summary_id"] + stored = (await service.store.get_memories_by_ids([summary_id])).get(summary_id) + self.assertEqual(stored.memory_domain, "bot_self_legacy") + self.assertEqual(RetrievalEngine._memory_domain_block_reason(stored), "bot_self_domain") + + async def test_unretrievable_summary_skips_archive_and_warns(self): + """摘要被历史同指纹旧记录合并、域仍是 unknown 时,宁可不归档也不制造死数据。""" + service = self.make_service() + ids = await self.seed(service) + + async def fake_get(memory_ids): + return {mid: MemoryRecord(id=mid, memory_domain="unknown") for mid in memory_ids} + + original = service.store.get_memories_by_ids + service.store.get_memories_by_ids = fake_get + try: + with self.assertLogs("MemoryCompanion", level="WARNING") as captured: + result = await service._run_memory_decay() + finally: + service.store.get_memories_by_ids = original + + self.assertEqual(result.get("archived"), 0, result) + self.assertTrue(str(result["reports"][0].get("reason", "")).startswith("summary_not_retrievable:"), + result["reports"][0]) + self.assertTrue(any("不可召回" in line for line in captured.output), captured.output) + self.assertEqual(await self.lifecycles(service, ids), {"stable_memory"}, + "守卫没拦住:源片段已被归档,主线已经变成死数据") + + async def test_dry_run_writes_nothing(self): + service = self.make_service({"maintenance": {"memory_decay_dry_run": True}}) + ids = await self.seed(service) + result = await service._run_memory_decay() + + self.assertIs(result.get("dry_run"), True, result) + self.assertEqual(result.get("archived"), 0) + self.assertEqual(result.get("summaries"), 0) + self.assertEqual(await self.lifecycles(service, ids), {"stable_memory"}) + + report = result["reports"][0] + self.assertEqual(set(report["would_archive_ids"]), set(ids)) + self.assertTrue(report["summary_preview"]) + self.assertTrue(report["planned_summary_id"]) + + rows = await service.store.list_memories(memory_type="memory_decay_summary", limit=10) + self.assertEqual(list(rows), [], "试运行不该往库里写摘要") + + +if __name__ == "__main__": + unittest.main()