feat: add explicit memory data migration

这个提交存在于:
YokiiroBW 2026-07-24 21:11:02 +08:00
父节点 04854669fd
当前提交 809a1ed3bb
共有 5 个文件被更改,包括 390 次插入 和 1 次删除

查看文件

@ -1,8 +1,10 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from .models import clean_text
from . import ops_migration
from .service import MemoryCompanionService
@ -308,6 +310,17 @@ class MemoryCompanionCommandHandler:
)
return "用法:/mcomp preset status|apply light|standard|companion"
async def ops_migration(self, action: str = "preview", source: str = "") -> str:
action = clean_text(action, 40).lower() or "preview"
source_path = Path(source).expanduser() if source.strip() else self.service.data_dir.parent / "astrbot_plugin_memory_companion"
target_path = self.service.data_dir
if action in {"preview", "check", "dry"}:
report = ops_migration.preview(source_path, target_path)
return ops_migration.format_report(report)
if action in {"run", "execute", "import"}:
report = ops_migration.execute(source_path, target_path)
return ops_migration.format_report(report)
return "用法:/mcomp ops_migration preview|run [旧数据目录]"
async def portable_data(self, action: str = "help", path: str = "") -> str:
action = clean_text(action, 40).lower() or "help"
path = str(path or "").strip()
@ -435,7 +448,8 @@ class MemoryCompanionCommandHandler:
"/mcomp data export|preview|import [jsonl_path]\n"
"/mcomp sleep status|run\n"
"/mcomp delete <memory_id>\n"
"/mcomp import_livingmemory preview|run|detail [db_path]"
"/mcomp import_livingmemory preview|run|detail [db_path]\n"
"/mcomp ops_migration preview|run [旧数据目录]"
)
def _format_livingmemory_preview(self, report: dict[str, Any], *, verbose: bool = False) -> str:

232
core/ops_migration.py 普通文件
查看文件

@ -0,0 +1,232 @@
from __future__ import annotations
import hashlib
import json
import os
import shutil
import sqlite3
import tempfile
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
TARGET_PLUGIN_NAME = "astrbot_plugin_memory_companion_ops"
MIGRATION_VERSION = 1
MARKER_NAME = "ops_migration_v1.json"
DATABASE_NAME = "memory_companion.db"
STATE_FILES = (
"memory_companion_sleep_state.json",
"memory_companion_token_usage.json",
"memory_companion_relationship_phase.json",
)
OPTIONAL_DIRS = ("historical_chat_imports", "exports")
REQUIRED_TABLES = ("memories", "identities", "timeline")
@dataclass(frozen=True)
class MigrationFile:
source: str
target: str
kind: str
bytes: int
sha256: str
@dataclass
class MigrationReport:
ok: bool
mode: str
source_dir: str
target_dir: str
source_plugin: str = "astrbot_plugin_memory_companion"
target_plugin: str = TARGET_PLUGIN_NAME
source_files: list[MigrationFile] = field(default_factory=list)
retained_tables: list[str] = field(default_factory=list)
retained_state_files: list[str] = field(default_factory=list)
skipped_paths: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
marker: str = ""
def to_dict(self) -> dict[str, Any]:
result = asdict(self)
result["source_files"] = [asdict(item) for item in self.source_files]
return result
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _file(path: Path, target_name: str, kind: str) -> MigrationFile:
return MigrationFile(str(path), target_name, kind, path.stat().st_size, _sha256(path))
def _validate_json(path: Path) -> None:
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise ValueError("状态文件顶层必须是对象")
def _validate_sqlite(path: Path) -> tuple[list[str], dict[str, int]]:
connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True, timeout=3)
try:
result = connection.execute("PRAGMA integrity_check").fetchone()
if not result or str(result[0]).lower() != "ok":
raise ValueError(f"SQLite integrity_check 失败: {result[0] if result else 'unknown'}")
tables = [row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")]
missing = [name for name in REQUIRED_TABLES if name not in tables]
if missing:
raise ValueError(f"缺少必要表:{', '.join(missing)}")
counts = {name: int(connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]) for name in tables}
return tables, counts
finally:
connection.close()
def _empty_target(target: Path) -> tuple[bool, str]:
if not target.exists():
return True, ""
if not target.is_dir():
return False, "目标路径不是目录"
entries = list(target.iterdir())
if not entries:
return True, ""
if (target / MARKER_NAME).is_file():
return False, "目标目录已有迁移标记,拒绝重复迁移"
return False, "目标目录非空,拒绝覆盖"
def preview(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
source = Path(source_dir).expanduser().resolve()
target = Path(target_dir).expanduser().resolve()
report = MigrationReport(ok=False, mode="preview", source_dir=str(source), target_dir=str(target))
report.marker = str(target / MARKER_NAME)
if not source.is_dir():
report.errors.append("源数据目录不存在")
return report.to_dict()
empty, reason = _empty_target(target)
if not empty:
report.errors.append(reason)
return report.to_dict()
database = source / DATABASE_NAME
if not database.is_file():
report.errors.append(f"未找到 {DATABASE_NAME}")
return report.to_dict()
try:
tables, counts = _validate_sqlite(database)
except Exception as exc:
report.errors.append(f"记忆数据库校验失败:{exc}")
return report.to_dict()
report.source_files.append(_file(database, DATABASE_NAME, "sqlite"))
report.retained_tables = sorted(tables)
report.warnings.append(f"SQLite 表记录数:{counts}")
for name in STATE_FILES:
path = source / name
if not path.is_file():
continue
try:
_validate_json(path)
except Exception as exc:
report.warnings.append(f"状态文件 {name} 无法读取,将跳过:{exc}")
report.skipped_paths.append(name)
continue
report.source_files.append(_file(path, name, "state"))
report.retained_state_files.append(name)
for name in OPTIONAL_DIRS:
path = source / name
if path.is_dir():
report.warnings.append(f"保留附属目录:{name}")
report.skipped_paths.append(f"directory:{name}")
report.ok = not report.errors
return report.to_dict()
def _backup_sqlite(source: Path, target: Path) -> None:
source_conn = sqlite3.connect(f"file:{source.resolve()}?mode=ro", uri=True, timeout=3)
try:
target_conn = sqlite3.connect(str(target), timeout=3)
try:
source_conn.backup(target_conn)
target_conn.commit()
finally:
target_conn.close()
finally:
source_conn.close()
def _report_from_dict(payload: dict[str, Any]) -> MigrationReport:
fields = {key: value for key, value in payload.items() if key in MigrationReport.__dataclass_fields__}
fields["source_files"] = [MigrationFile(**item) for item in fields.get("source_files", [])]
return MigrationReport(**fields)
def execute(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
report = _report_from_dict(preview(source_dir, target_dir))
report.mode = "execute"
if not report.ok:
return report.to_dict()
source = Path(report.source_dir)
target = Path(report.target_dir)
target.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix=".ops-migration-", dir=str(target.parent)))
moved: list[Path] = []
try:
_backup_sqlite(source / DATABASE_NAME, staging / DATABASE_NAME)
for name in report.retained_state_files:
shutil.copy2(source / name, staging / name)
marker_payload = {
"migration_version": MIGRATION_VERSION,
"source_plugin": report.source_plugin,
"target_plugin": TARGET_PLUGIN_NAME,
"source_dir": str(source),
"target_dir": str(target),
"created_at": datetime.now(timezone.utc).isoformat(),
"files": [asdict(item) for item in report.source_files],
}
(staging / MARKER_NAME).write_text(
json.dumps(marker_payload, ensure_ascii=False, indent=2), encoding="utf-8"
)
for item in staging.iterdir():
destination = target / item.name
if destination.exists():
raise FileExistsError(f"目标文件已存在:{destination}")
os.replace(item, destination)
moved.append(destination)
report.marker = str(target / MARKER_NAME)
report.warnings.append("迁移完成;源目录保持不变,删除旧目录前请先做 VM 验收")
return report.to_dict()
except Exception as exc:
for destination in reversed(moved):
try:
destination.unlink()
except OSError:
pass
report.ok = False
report.errors.append(f"迁移失败,源目录未修改:{exc}")
return report.to_dict()
finally:
shutil.rmtree(staging, ignore_errors=True)
def format_report(report: dict[str, Any]) -> str:
lines = [
f"迁移状态:{'可执行' if report.get('ok') else '拒绝/失败'}",
f"模式:{report.get('mode')}",
f"来源:{report.get('source_dir')}",
f"目标:{report.get('target_dir')}",
f"保留表:{len(report.get('retained_tables', []))} 个",
f"保留状态文件:{len(report.get('retained_state_files', []))} 个",
]
for error in report.get("errors", []):
lines.append("错误:" + str(error))
for warning in report.get("warnings", []):
lines.append("提示:" + str(warning))
return "\n".join(lines)

查看文件

@ -122,3 +122,13 @@
- 修复或规避:建立 companion-main 基线、upstream-sync 和 ops-main;Git 使用仓库级 safe.directory;测试使用 Codex bundled Python;benchmark 改用 astrbot_plugin_memory_companion_ops。此前关于“全部测试因旧包名无法收集”的记录在当前基线已被本条取代,当前测试直接引用 core 包。
- 预防规则:后续版本改名必须同步检查 main、metadata、页面路由、Bridge、历史导入、运维诊断、benchmark 和测试;测试结果必须来自当前分支实际执行。
- 验证证据:companion-main 基线提交为 2f24824,当前分支为 ops-main;compileall 通过;pytest 结果 128 passed in 13.51s。
## 2026-07-24 — Q1 迁移必须与插件启动解耦
- 阶段:Q1 数据迁移与启动安全
- 类型:功能行为 / 数据安全
- 现象:运维版数据目录已与普通版分离,但没有迁移器时,旧记忆不会自动出现;启动时直接搬迁又可能覆盖生产数据。
- 根因:目录隔离和历史数据迁移是两个不同问题,不能用插件初始化顺手解决。
- 修复或规避:增加 `core/ops_migration.py`,只允许管理员显式执行 preview/run;SQLite 使用只读校验与 backup,成功后写迁移标记。
- 预防规则:先预览和备份,再执行迁移;目标非空、格式损坏或已有标记时必须拒绝。
- 验证证据:`tests/test_ops_migration.py` 5 项 unittest 通过,`compileall` 通过。

查看文件

@ -291,6 +291,12 @@ class MemoryCompanionPlugin(Star):
yield event.plain_result(await self.commands.portable_data(action, path))
@permission_type(PermissionType.ADMIN)
@mcomp.command("ops_migration", priority=10)
async def cmd_mcomp_ops_migration(
self, event: AstrMessageEvent, action: str = "preview", source: str = ""
) -> AsyncGenerator[MessageEventResult, None]:
yield event.plain_result(await self.commands.ops_migration(action, source))
@mcomp.command("sleep", priority=10)
async def cmd_mcomp_sleep(
self, event: AstrMessageEvent, action: str = "status"

127
tests/test_ops_migration.py 普通文件
查看文件

@ -0,0 +1,127 @@
from __future__ import annotations
import json
import sqlite3
import sys
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT.parent / 'astrbot_plugin_private_companion-main'))
import ops_migration as companion_migration
from core import ops_migration as memory_migration
def make_companion_db(path: Path) -> None:
connection = sqlite3.connect(path)
connection.execute(
"CREATE TABLE store_sections (section_name TEXT PRIMARY KEY, payload_json TEXT NOT NULL, updated_at REAL NOT NULL, checksum TEXT, schema_version INTEGER)"
)
connection.execute(
"INSERT INTO store_sections VALUES ('daily_plan', '{\"today\":true}', 1, '', 1)"
)
connection.commit()
connection.close()
def make_memory_db(path: Path) -> None:
connection = sqlite3.connect(path)
for table in ("memories", "identities", "timeline"):
connection.execute(f"CREATE TABLE {table} (id TEXT PRIMARY KEY)")
connection.execute("INSERT INTO memories VALUES ('memory-1')")
connection.commit()
connection.close()
class OpsMigrationTests(unittest.TestCase):
def test_companion_json_preview_and_execute_preserves_fields(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "old"
target = root / "new"
source.mkdir()
payload = {
"users": {"u1": {"preference": "tea"}},
"daily_plan": {"date": "2026-07-24"},
"important_dates": [{"name": "birthday"}],
"bookshelf_items": [{"title": "private"}],
"creative_projects": [{"title": "novel"}],
"news_integration": {"enabled": True},
}
(source / "companions.json").write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
report = companion_migration.preview(source, target)
self.assertTrue(report["ok"])
self.assertIn("json:daily_plan", report["retained_fields"])
self.assertIn("news_integration", report["flagged_fields"])
result = companion_migration.execute(source, target)
self.assertTrue(result["ok"])
self.assertEqual(payload, json.loads((target / "companions.json").read_text(encoding="utf-8")))
self.assertTrue((target / companion_migration.MARKER_NAME).is_file())
self.assertFalse(companion_migration.execute(source, target)["ok"])
def test_companion_sqlite_preview_and_execute_keeps_json_mirror(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "old"
target = root / "new"
source.mkdir()
make_companion_db(source / "companions.db")
(source / "companions.json").write_text('{"creative_projects": []}', encoding="utf-8")
result = companion_migration.execute(source, target)
self.assertTrue(result["ok"])
self.assertTrue((target / "companions.db").is_file())
self.assertTrue((target / "companions.json").is_file())
check = sqlite3.connect(target / "companions.db")
self.assertEqual(1, check.execute("SELECT COUNT(*) FROM store_sections").fetchone()[0])
check.close()
def test_memory_migration_preserves_database_and_state(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "old"
target = root / "new"
source.mkdir()
make_memory_db(source / memory_migration.DATABASE_NAME)
(source / "memory_companion_sleep_state.json").write_text('{"sleeping":false}', encoding="utf-8")
report = memory_migration.preview(source, target)
self.assertTrue(report["ok"])
self.assertIn("memories", report["retained_tables"])
result = memory_migration.execute(source, target)
self.assertTrue(result["ok"])
check = sqlite3.connect(target / memory_migration.DATABASE_NAME)
self.assertEqual(1, check.execute("SELECT COUNT(*) FROM memories").fetchone()[0])
check.close()
self.assertEqual('{"sleeping":false}', (target / "memory_companion_sleep_state.json").read_text(encoding="utf-8"))
def test_non_empty_target_and_invalid_source_are_rejected_without_changes(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "old"
target = root / "new"
source.mkdir()
target.mkdir()
(target / "keep.txt").write_text("keep", encoding="utf-8")
(source / "companions.json").write_text("[]", encoding="utf-8")
report = companion_migration.preview(source, target)
self.assertFalse(report["ok"])
self.assertIn("非空", report["errors"][0])
self.assertEqual("keep", (target / "keep.txt").read_text(encoding="utf-8"))
def test_memory_invalid_database_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "old"
source.mkdir()
(source / memory_migration.DATABASE_NAME).write_bytes(b"not sqlite")
report = memory_migration.preview(source, root / "new")
self.assertFalse(report["ok"])
self.assertTrue(report["errors"])
if __name__ == "__main__":
unittest.main()