From 809a1ed3bba56c5df70a668df6c96b84fb98359e Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Fri, 24 Jul 2026 21:11:02 +0800 Subject: [PATCH] feat: add explicit memory data migration --- core/commands.py | 16 ++- core/ops_migration.py | 232 ++++++++++++++++++++++++++++++++++++ docs/错题本.md | 10 ++ main.py | 6 + tests/test_ops_migration.py | 127 ++++++++++++++++++++ 5 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 core/ops_migration.py create mode 100644 tests/test_ops_migration.py diff --git a/core/commands.py b/core/commands.py index 185fbe9..b40bdab 100644 --- a/core/commands.py +++ b/core/commands.py @@ -1,8 +1,10 @@ from __future__ import annotations +from pathlib import Path from typing import Any from .models import clean_text +from . import ops_migration from .service import MemoryCompanionService @@ -308,6 +310,17 @@ class MemoryCompanionCommandHandler: ) return "用法:/mcomp preset status|apply light|standard|companion" + async def ops_migration(self, action: str = "preview", source: str = "") -> str: + action = clean_text(action, 40).lower() or "preview" + source_path = Path(source).expanduser() if source.strip() else self.service.data_dir.parent / "astrbot_plugin_memory_companion" + target_path = self.service.data_dir + if action in {"preview", "check", "dry"}: + report = ops_migration.preview(source_path, target_path) + return ops_migration.format_report(report) + if action in {"run", "execute", "import"}: + report = ops_migration.execute(source_path, target_path) + return ops_migration.format_report(report) + return "用法:/mcomp ops_migration preview|run [旧数据目录]" async def portable_data(self, action: str = "help", path: str = "") -> str: action = clean_text(action, 40).lower() or "help" path = str(path or "").strip() @@ -435,7 +448,8 @@ class MemoryCompanionCommandHandler: "/mcomp data export|preview|import [jsonl_path]\n" "/mcomp sleep status|run\n" "/mcomp delete \n" - "/mcomp import_livingmemory preview|run|detail [db_path]" + "/mcomp import_livingmemory preview|run|detail [db_path]\n" + "/mcomp ops_migration preview|run [旧数据目录]" ) def _format_livingmemory_preview(self, report: dict[str, Any], *, verbose: bool = False) -> str: diff --git a/core/ops_migration.py b/core/ops_migration.py new file mode 100644 index 0000000..a86fbc1 --- /dev/null +++ b/core/ops_migration.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import tempfile +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +TARGET_PLUGIN_NAME = "astrbot_plugin_memory_companion_ops" +MIGRATION_VERSION = 1 +MARKER_NAME = "ops_migration_v1.json" +DATABASE_NAME = "memory_companion.db" +STATE_FILES = ( + "memory_companion_sleep_state.json", + "memory_companion_token_usage.json", + "memory_companion_relationship_phase.json", +) +OPTIONAL_DIRS = ("historical_chat_imports", "exports") +REQUIRED_TABLES = ("memories", "identities", "timeline") + + +@dataclass(frozen=True) +class MigrationFile: + source: str + target: str + kind: str + bytes: int + sha256: str + + +@dataclass +class MigrationReport: + ok: bool + mode: str + source_dir: str + target_dir: str + source_plugin: str = "astrbot_plugin_memory_companion" + target_plugin: str = TARGET_PLUGIN_NAME + source_files: list[MigrationFile] = field(default_factory=list) + retained_tables: list[str] = field(default_factory=list) + retained_state_files: list[str] = field(default_factory=list) + skipped_paths: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + marker: str = "" + + def to_dict(self) -> dict[str, Any]: + result = asdict(self) + result["source_files"] = [asdict(item) for item in self.source_files] + return result + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _file(path: Path, target_name: str, kind: str) -> MigrationFile: + return MigrationFile(str(path), target_name, kind, path.stat().st_size, _sha256(path)) + + +def _validate_json(path: Path) -> None: + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError("状态文件顶层必须是对象") + + +def _validate_sqlite(path: Path) -> tuple[list[str], dict[str, int]]: + connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True, timeout=3) + try: + result = connection.execute("PRAGMA integrity_check").fetchone() + if not result or str(result[0]).lower() != "ok": + raise ValueError(f"SQLite integrity_check 失败: {result[0] if result else 'unknown'}") + tables = [row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")] + missing = [name for name in REQUIRED_TABLES if name not in tables] + if missing: + raise ValueError(f"缺少必要表:{', '.join(missing)}") + counts = {name: int(connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]) for name in tables} + return tables, counts + finally: + connection.close() + + +def _empty_target(target: Path) -> tuple[bool, str]: + if not target.exists(): + return True, "" + if not target.is_dir(): + return False, "目标路径不是目录" + entries = list(target.iterdir()) + if not entries: + return True, "" + if (target / MARKER_NAME).is_file(): + return False, "目标目录已有迁移标记,拒绝重复迁移" + return False, "目标目录非空,拒绝覆盖" + + +def preview(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]: + source = Path(source_dir).expanduser().resolve() + target = Path(target_dir).expanduser().resolve() + report = MigrationReport(ok=False, mode="preview", source_dir=str(source), target_dir=str(target)) + report.marker = str(target / MARKER_NAME) + if not source.is_dir(): + report.errors.append("源数据目录不存在") + return report.to_dict() + empty, reason = _empty_target(target) + if not empty: + report.errors.append(reason) + return report.to_dict() + database = source / DATABASE_NAME + if not database.is_file(): + report.errors.append(f"未找到 {DATABASE_NAME}") + return report.to_dict() + try: + tables, counts = _validate_sqlite(database) + except Exception as exc: + report.errors.append(f"记忆数据库校验失败:{exc}") + return report.to_dict() + report.source_files.append(_file(database, DATABASE_NAME, "sqlite")) + report.retained_tables = sorted(tables) + report.warnings.append(f"SQLite 表记录数:{counts}") + for name in STATE_FILES: + path = source / name + if not path.is_file(): + continue + try: + _validate_json(path) + except Exception as exc: + report.warnings.append(f"状态文件 {name} 无法读取,将跳过:{exc}") + report.skipped_paths.append(name) + continue + report.source_files.append(_file(path, name, "state")) + report.retained_state_files.append(name) + for name in OPTIONAL_DIRS: + path = source / name + if path.is_dir(): + report.warnings.append(f"保留附属目录:{name}") + report.skipped_paths.append(f"directory:{name}") + report.ok = not report.errors + return report.to_dict() + + +def _backup_sqlite(source: Path, target: Path) -> None: + source_conn = sqlite3.connect(f"file:{source.resolve()}?mode=ro", uri=True, timeout=3) + try: + target_conn = sqlite3.connect(str(target), timeout=3) + try: + source_conn.backup(target_conn) + target_conn.commit() + finally: + target_conn.close() + finally: + source_conn.close() + + +def _report_from_dict(payload: dict[str, Any]) -> MigrationReport: + fields = {key: value for key, value in payload.items() if key in MigrationReport.__dataclass_fields__} + fields["source_files"] = [MigrationFile(**item) for item in fields.get("source_files", [])] + return MigrationReport(**fields) + + +def execute(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]: + report = _report_from_dict(preview(source_dir, target_dir)) + report.mode = "execute" + if not report.ok: + return report.to_dict() + source = Path(report.source_dir) + target = Path(report.target_dir) + target.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".ops-migration-", dir=str(target.parent))) + moved: list[Path] = [] + try: + _backup_sqlite(source / DATABASE_NAME, staging / DATABASE_NAME) + for name in report.retained_state_files: + shutil.copy2(source / name, staging / name) + marker_payload = { + "migration_version": MIGRATION_VERSION, + "source_plugin": report.source_plugin, + "target_plugin": TARGET_PLUGIN_NAME, + "source_dir": str(source), + "target_dir": str(target), + "created_at": datetime.now(timezone.utc).isoformat(), + "files": [asdict(item) for item in report.source_files], + } + (staging / MARKER_NAME).write_text( + json.dumps(marker_payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) + for item in staging.iterdir(): + destination = target / item.name + if destination.exists(): + raise FileExistsError(f"目标文件已存在:{destination}") + os.replace(item, destination) + moved.append(destination) + report.marker = str(target / MARKER_NAME) + report.warnings.append("迁移完成;源目录保持不变,删除旧目录前请先做 VM 验收") + return report.to_dict() + except Exception as exc: + for destination in reversed(moved): + try: + destination.unlink() + except OSError: + pass + report.ok = False + report.errors.append(f"迁移失败,源目录未修改:{exc}") + return report.to_dict() + finally: + shutil.rmtree(staging, ignore_errors=True) + + +def format_report(report: dict[str, Any]) -> str: + lines = [ + f"迁移状态:{'可执行' if report.get('ok') else '拒绝/失败'}", + f"模式:{report.get('mode')}", + f"来源:{report.get('source_dir')}", + f"目标:{report.get('target_dir')}", + f"保留表:{len(report.get('retained_tables', []))} 个", + f"保留状态文件:{len(report.get('retained_state_files', []))} 个", + ] + for error in report.get("errors", []): + lines.append("错误:" + str(error)) + for warning in report.get("warnings", []): + lines.append("提示:" + str(warning)) + return "\n".join(lines) + diff --git a/docs/错题本.md b/docs/错题本.md index eb94ecb..9588c31 100644 --- a/docs/错题本.md +++ b/docs/错题本.md @@ -122,3 +122,13 @@ - 修复或规避:建立 companion-main 基线、upstream-sync 和 ops-main;Git 使用仓库级 safe.directory;测试使用 Codex bundled Python;benchmark 改用 astrbot_plugin_memory_companion_ops。此前关于“全部测试因旧包名无法收集”的记录在当前基线已被本条取代,当前测试直接引用 core 包。 - 预防规则:后续版本改名必须同步检查 main、metadata、页面路由、Bridge、历史导入、运维诊断、benchmark 和测试;测试结果必须来自当前分支实际执行。 - 验证证据:companion-main 基线提交为 2f24824,当前分支为 ops-main;compileall 通过;pytest 结果 128 passed in 13.51s。 + +## 2026-07-24 — Q1 迁移必须与插件启动解耦 + +- 阶段:Q1 数据迁移与启动安全 +- 类型:功能行为 / 数据安全 +- 现象:运维版数据目录已与普通版分离,但没有迁移器时,旧记忆不会自动出现;启动时直接搬迁又可能覆盖生产数据。 +- 根因:目录隔离和历史数据迁移是两个不同问题,不能用插件初始化顺手解决。 +- 修复或规避:增加 `core/ops_migration.py`,只允许管理员显式执行 preview/run;SQLite 使用只读校验与 backup,成功后写迁移标记。 +- 预防规则:先预览和备份,再执行迁移;目标非空、格式损坏或已有标记时必须拒绝。 +- 验证证据:`tests/test_ops_migration.py` 5 项 unittest 通过,`compileall` 通过。 diff --git a/main.py b/main.py index 89529b9..86adddc 100644 --- a/main.py +++ b/main.py @@ -291,6 +291,12 @@ class MemoryCompanionPlugin(Star): yield event.plain_result(await self.commands.portable_data(action, path)) @permission_type(PermissionType.ADMIN) + @mcomp.command("ops_migration", priority=10) + async def cmd_mcomp_ops_migration( + self, event: AstrMessageEvent, action: str = "preview", source: str = "" + ) -> AsyncGenerator[MessageEventResult, None]: + yield event.plain_result(await self.commands.ops_migration(action, source)) + @mcomp.command("sleep", priority=10) async def cmd_mcomp_sleep( self, event: AstrMessageEvent, action: str = "status" diff --git a/tests/test_ops_migration.py b/tests/test_ops_migration.py new file mode 100644 index 0000000..9882b64 --- /dev/null +++ b/tests/test_ops_migration.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import json +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT.parent / 'astrbot_plugin_private_companion-main')) + +import ops_migration as companion_migration +from core import ops_migration as memory_migration + + +def make_companion_db(path: Path) -> None: + connection = sqlite3.connect(path) + connection.execute( + "CREATE TABLE store_sections (section_name TEXT PRIMARY KEY, payload_json TEXT NOT NULL, updated_at REAL NOT NULL, checksum TEXT, schema_version INTEGER)" + ) + connection.execute( + "INSERT INTO store_sections VALUES ('daily_plan', '{\"today\":true}', 1, '', 1)" + ) + connection.commit() + connection.close() + + +def make_memory_db(path: Path) -> None: + connection = sqlite3.connect(path) + for table in ("memories", "identities", "timeline"): + connection.execute(f"CREATE TABLE {table} (id TEXT PRIMARY KEY)") + connection.execute("INSERT INTO memories VALUES ('memory-1')") + connection.commit() + connection.close() + + +class OpsMigrationTests(unittest.TestCase): + def test_companion_json_preview_and_execute_preserves_fields(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "old" + target = root / "new" + source.mkdir() + payload = { + "users": {"u1": {"preference": "tea"}}, + "daily_plan": {"date": "2026-07-24"}, + "important_dates": [{"name": "birthday"}], + "bookshelf_items": [{"title": "private"}], + "creative_projects": [{"title": "novel"}], + "news_integration": {"enabled": True}, + } + (source / "companions.json").write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + report = companion_migration.preview(source, target) + self.assertTrue(report["ok"]) + self.assertIn("json:daily_plan", report["retained_fields"]) + self.assertIn("news_integration", report["flagged_fields"]) + result = companion_migration.execute(source, target) + self.assertTrue(result["ok"]) + self.assertEqual(payload, json.loads((target / "companions.json").read_text(encoding="utf-8"))) + self.assertTrue((target / companion_migration.MARKER_NAME).is_file()) + self.assertFalse(companion_migration.execute(source, target)["ok"]) + + def test_companion_sqlite_preview_and_execute_keeps_json_mirror(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "old" + target = root / "new" + source.mkdir() + make_companion_db(source / "companions.db") + (source / "companions.json").write_text('{"creative_projects": []}', encoding="utf-8") + result = companion_migration.execute(source, target) + self.assertTrue(result["ok"]) + self.assertTrue((target / "companions.db").is_file()) + self.assertTrue((target / "companions.json").is_file()) + check = sqlite3.connect(target / "companions.db") + self.assertEqual(1, check.execute("SELECT COUNT(*) FROM store_sections").fetchone()[0]) + check.close() + + def test_memory_migration_preserves_database_and_state(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "old" + target = root / "new" + source.mkdir() + make_memory_db(source / memory_migration.DATABASE_NAME) + (source / "memory_companion_sleep_state.json").write_text('{"sleeping":false}', encoding="utf-8") + report = memory_migration.preview(source, target) + self.assertTrue(report["ok"]) + self.assertIn("memories", report["retained_tables"]) + result = memory_migration.execute(source, target) + self.assertTrue(result["ok"]) + check = sqlite3.connect(target / memory_migration.DATABASE_NAME) + self.assertEqual(1, check.execute("SELECT COUNT(*) FROM memories").fetchone()[0]) + check.close() + self.assertEqual('{"sleeping":false}', (target / "memory_companion_sleep_state.json").read_text(encoding="utf-8")) + + def test_non_empty_target_and_invalid_source_are_rejected_without_changes(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "old" + target = root / "new" + source.mkdir() + target.mkdir() + (target / "keep.txt").write_text("keep", encoding="utf-8") + (source / "companions.json").write_text("[]", encoding="utf-8") + report = companion_migration.preview(source, target) + self.assertFalse(report["ok"]) + self.assertIn("非空", report["errors"][0]) + self.assertEqual("keep", (target / "keep.txt").read_text(encoding="utf-8")) + + def test_memory_invalid_database_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "old" + source.mkdir() + (source / memory_migration.DATABASE_NAME).write_bytes(b"not sqlite") + report = memory_migration.preview(source, root / "new") + self.assertFalse(report["ok"]) + self.assertTrue(report["errors"]) + + +if __name__ == "__main__": + unittest.main() + +