feat: add explicit memory data migration
这个提交存在于:
父节点
04854669fd
当前提交
809a1ed3bb
共有 5 个文件被更改,包括 390 次插入 和 1 次删除
|
|
@ -1,8 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .models import clean_text
|
||||
from . import ops_migration
|
||||
from .service import MemoryCompanionService
|
||||
|
||||
|
||||
|
|
@ -308,6 +310,17 @@ class MemoryCompanionCommandHandler:
|
|||
)
|
||||
return "用法:/mcomp preset status|apply light|standard|companion"
|
||||
|
||||
async def ops_migration(self, action: str = "preview", source: str = "") -> str:
|
||||
action = clean_text(action, 40).lower() or "preview"
|
||||
source_path = Path(source).expanduser() if source.strip() else self.service.data_dir.parent / "astrbot_plugin_memory_companion"
|
||||
target_path = self.service.data_dir
|
||||
if action in {"preview", "check", "dry"}:
|
||||
report = ops_migration.preview(source_path, target_path)
|
||||
return ops_migration.format_report(report)
|
||||
if action in {"run", "execute", "import"}:
|
||||
report = ops_migration.execute(source_path, target_path)
|
||||
return ops_migration.format_report(report)
|
||||
return "用法:/mcomp ops_migration preview|run [旧数据目录]"
|
||||
async def portable_data(self, action: str = "help", path: str = "") -> str:
|
||||
action = clean_text(action, 40).lower() or "help"
|
||||
path = str(path or "").strip()
|
||||
|
|
@ -435,7 +448,8 @@ class MemoryCompanionCommandHandler:
|
|||
"/mcomp data export|preview|import [jsonl_path]\n"
|
||||
"/mcomp sleep status|run\n"
|
||||
"/mcomp delete <memory_id>\n"
|
||||
"/mcomp import_livingmemory preview|run|detail [db_path]"
|
||||
"/mcomp import_livingmemory preview|run|detail [db_path]\n"
|
||||
"/mcomp ops_migration preview|run [旧数据目录]"
|
||||
)
|
||||
|
||||
def _format_livingmemory_preview(self, report: dict[str, Any], *, verbose: bool = False) -> str:
|
||||
|
|
|
|||
232
core/ops_migration.py
普通文件
232
core/ops_migration.py
普通文件
|
|
@ -0,0 +1,232 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
TARGET_PLUGIN_NAME = "astrbot_plugin_memory_companion_ops"
|
||||
MIGRATION_VERSION = 1
|
||||
MARKER_NAME = "ops_migration_v1.json"
|
||||
DATABASE_NAME = "memory_companion.db"
|
||||
STATE_FILES = (
|
||||
"memory_companion_sleep_state.json",
|
||||
"memory_companion_token_usage.json",
|
||||
"memory_companion_relationship_phase.json",
|
||||
)
|
||||
OPTIONAL_DIRS = ("historical_chat_imports", "exports")
|
||||
REQUIRED_TABLES = ("memories", "identities", "timeline")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationFile:
|
||||
source: str
|
||||
target: str
|
||||
kind: str
|
||||
bytes: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationReport:
|
||||
ok: bool
|
||||
mode: str
|
||||
source_dir: str
|
||||
target_dir: str
|
||||
source_plugin: str = "astrbot_plugin_memory_companion"
|
||||
target_plugin: str = TARGET_PLUGIN_NAME
|
||||
source_files: list[MigrationFile] = field(default_factory=list)
|
||||
retained_tables: list[str] = field(default_factory=list)
|
||||
retained_state_files: list[str] = field(default_factory=list)
|
||||
skipped_paths: list[str] = field(default_factory=list)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
marker: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
result["source_files"] = [asdict(item) for item in self.source_files]
|
||||
return result
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _file(path: Path, target_name: str, kind: str) -> MigrationFile:
|
||||
return MigrationFile(str(path), target_name, kind, path.stat().st_size, _sha256(path))
|
||||
|
||||
|
||||
def _validate_json(path: Path) -> None:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("状态文件顶层必须是对象")
|
||||
|
||||
|
||||
def _validate_sqlite(path: Path) -> tuple[list[str], dict[str, int]]:
|
||||
connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True, timeout=3)
|
||||
try:
|
||||
result = connection.execute("PRAGMA integrity_check").fetchone()
|
||||
if not result or str(result[0]).lower() != "ok":
|
||||
raise ValueError(f"SQLite integrity_check 失败: {result[0] if result else 'unknown'}")
|
||||
tables = [row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||
missing = [name for name in REQUIRED_TABLES if name not in tables]
|
||||
if missing:
|
||||
raise ValueError(f"缺少必要表:{', '.join(missing)}")
|
||||
counts = {name: int(connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]) for name in tables}
|
||||
return tables, counts
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _empty_target(target: Path) -> tuple[bool, str]:
|
||||
if not target.exists():
|
||||
return True, ""
|
||||
if not target.is_dir():
|
||||
return False, "目标路径不是目录"
|
||||
entries = list(target.iterdir())
|
||||
if not entries:
|
||||
return True, ""
|
||||
if (target / MARKER_NAME).is_file():
|
||||
return False, "目标目录已有迁移标记,拒绝重复迁移"
|
||||
return False, "目标目录非空,拒绝覆盖"
|
||||
|
||||
|
||||
def preview(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
|
||||
source = Path(source_dir).expanduser().resolve()
|
||||
target = Path(target_dir).expanduser().resolve()
|
||||
report = MigrationReport(ok=False, mode="preview", source_dir=str(source), target_dir=str(target))
|
||||
report.marker = str(target / MARKER_NAME)
|
||||
if not source.is_dir():
|
||||
report.errors.append("源数据目录不存在")
|
||||
return report.to_dict()
|
||||
empty, reason = _empty_target(target)
|
||||
if not empty:
|
||||
report.errors.append(reason)
|
||||
return report.to_dict()
|
||||
database = source / DATABASE_NAME
|
||||
if not database.is_file():
|
||||
report.errors.append(f"未找到 {DATABASE_NAME}")
|
||||
return report.to_dict()
|
||||
try:
|
||||
tables, counts = _validate_sqlite(database)
|
||||
except Exception as exc:
|
||||
report.errors.append(f"记忆数据库校验失败:{exc}")
|
||||
return report.to_dict()
|
||||
report.source_files.append(_file(database, DATABASE_NAME, "sqlite"))
|
||||
report.retained_tables = sorted(tables)
|
||||
report.warnings.append(f"SQLite 表记录数:{counts}")
|
||||
for name in STATE_FILES:
|
||||
path = source / name
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
_validate_json(path)
|
||||
except Exception as exc:
|
||||
report.warnings.append(f"状态文件 {name} 无法读取,将跳过:{exc}")
|
||||
report.skipped_paths.append(name)
|
||||
continue
|
||||
report.source_files.append(_file(path, name, "state"))
|
||||
report.retained_state_files.append(name)
|
||||
for name in OPTIONAL_DIRS:
|
||||
path = source / name
|
||||
if path.is_dir():
|
||||
report.warnings.append(f"保留附属目录:{name}")
|
||||
report.skipped_paths.append(f"directory:{name}")
|
||||
report.ok = not report.errors
|
||||
return report.to_dict()
|
||||
|
||||
|
||||
def _backup_sqlite(source: Path, target: Path) -> None:
|
||||
source_conn = sqlite3.connect(f"file:{source.resolve()}?mode=ro", uri=True, timeout=3)
|
||||
try:
|
||||
target_conn = sqlite3.connect(str(target), timeout=3)
|
||||
try:
|
||||
source_conn.backup(target_conn)
|
||||
target_conn.commit()
|
||||
finally:
|
||||
target_conn.close()
|
||||
finally:
|
||||
source_conn.close()
|
||||
|
||||
|
||||
def _report_from_dict(payload: dict[str, Any]) -> MigrationReport:
|
||||
fields = {key: value for key, value in payload.items() if key in MigrationReport.__dataclass_fields__}
|
||||
fields["source_files"] = [MigrationFile(**item) for item in fields.get("source_files", [])]
|
||||
return MigrationReport(**fields)
|
||||
|
||||
|
||||
def execute(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
|
||||
report = _report_from_dict(preview(source_dir, target_dir))
|
||||
report.mode = "execute"
|
||||
if not report.ok:
|
||||
return report.to_dict()
|
||||
source = Path(report.source_dir)
|
||||
target = Path(report.target_dir)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".ops-migration-", dir=str(target.parent)))
|
||||
moved: list[Path] = []
|
||||
try:
|
||||
_backup_sqlite(source / DATABASE_NAME, staging / DATABASE_NAME)
|
||||
for name in report.retained_state_files:
|
||||
shutil.copy2(source / name, staging / name)
|
||||
marker_payload = {
|
||||
"migration_version": MIGRATION_VERSION,
|
||||
"source_plugin": report.source_plugin,
|
||||
"target_plugin": TARGET_PLUGIN_NAME,
|
||||
"source_dir": str(source),
|
||||
"target_dir": str(target),
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"files": [asdict(item) for item in report.source_files],
|
||||
}
|
||||
(staging / MARKER_NAME).write_text(
|
||||
json.dumps(marker_payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
for item in staging.iterdir():
|
||||
destination = target / item.name
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"目标文件已存在:{destination}")
|
||||
os.replace(item, destination)
|
||||
moved.append(destination)
|
||||
report.marker = str(target / MARKER_NAME)
|
||||
report.warnings.append("迁移完成;源目录保持不变,删除旧目录前请先做 VM 验收")
|
||||
return report.to_dict()
|
||||
except Exception as exc:
|
||||
for destination in reversed(moved):
|
||||
try:
|
||||
destination.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
report.ok = False
|
||||
report.errors.append(f"迁移失败,源目录未修改:{exc}")
|
||||
return report.to_dict()
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def format_report(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"迁移状态:{'可执行' if report.get('ok') else '拒绝/失败'}",
|
||||
f"模式:{report.get('mode')}",
|
||||
f"来源:{report.get('source_dir')}",
|
||||
f"目标:{report.get('target_dir')}",
|
||||
f"保留表:{len(report.get('retained_tables', []))} 个",
|
||||
f"保留状态文件:{len(report.get('retained_state_files', []))} 个",
|
||||
]
|
||||
for error in report.get("errors", []):
|
||||
lines.append("错误:" + str(error))
|
||||
for warning in report.get("warnings", []):
|
||||
lines.append("提示:" + str(warning))
|
||||
return "\n".join(lines)
|
||||
|
||||
10
docs/错题本.md
10
docs/错题本.md
|
|
@ -122,3 +122,13 @@
|
|||
- 修复或规避:建立 companion-main 基线、upstream-sync 和 ops-main;Git 使用仓库级 safe.directory;测试使用 Codex bundled Python;benchmark 改用 astrbot_plugin_memory_companion_ops。此前关于“全部测试因旧包名无法收集”的记录在当前基线已被本条取代,当前测试直接引用 core 包。
|
||||
- 预防规则:后续版本改名必须同步检查 main、metadata、页面路由、Bridge、历史导入、运维诊断、benchmark 和测试;测试结果必须来自当前分支实际执行。
|
||||
- 验证证据:companion-main 基线提交为 2f24824,当前分支为 ops-main;compileall 通过;pytest 结果 128 passed in 13.51s。
|
||||
|
||||
## 2026-07-24 — Q1 迁移必须与插件启动解耦
|
||||
|
||||
- 阶段:Q1 数据迁移与启动安全
|
||||
- 类型:功能行为 / 数据安全
|
||||
- 现象:运维版数据目录已与普通版分离,但没有迁移器时,旧记忆不会自动出现;启动时直接搬迁又可能覆盖生产数据。
|
||||
- 根因:目录隔离和历史数据迁移是两个不同问题,不能用插件初始化顺手解决。
|
||||
- 修复或规避:增加 `core/ops_migration.py`,只允许管理员显式执行 preview/run;SQLite 使用只读校验与 backup,成功后写迁移标记。
|
||||
- 预防规则:先预览和备份,再执行迁移;目标非空、格式损坏或已有标记时必须拒绝。
|
||||
- 验证证据:`tests/test_ops_migration.py` 5 项 unittest 通过,`compileall` 通过。
|
||||
|
|
|
|||
6
main.py
6
main.py
|
|
@ -291,6 +291,12 @@ class MemoryCompanionPlugin(Star):
|
|||
yield event.plain_result(await self.commands.portable_data(action, path))
|
||||
|
||||
@permission_type(PermissionType.ADMIN)
|
||||
@mcomp.command("ops_migration", priority=10)
|
||||
async def cmd_mcomp_ops_migration(
|
||||
self, event: AstrMessageEvent, action: str = "preview", source: str = ""
|
||||
) -> AsyncGenerator[MessageEventResult, None]:
|
||||
yield event.plain_result(await self.commands.ops_migration(action, source))
|
||||
|
||||
@mcomp.command("sleep", priority=10)
|
||||
async def cmd_mcomp_sleep(
|
||||
self, event: AstrMessageEvent, action: str = "status"
|
||||
|
|
|
|||
127
tests/test_ops_migration.py
普通文件
127
tests/test_ops_migration.py
普通文件
|
|
@ -0,0 +1,127 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT.parent / 'astrbot_plugin_private_companion-main'))
|
||||
|
||||
import ops_migration as companion_migration
|
||||
from core import ops_migration as memory_migration
|
||||
|
||||
|
||||
def make_companion_db(path: Path) -> None:
|
||||
connection = sqlite3.connect(path)
|
||||
connection.execute(
|
||||
"CREATE TABLE store_sections (section_name TEXT PRIMARY KEY, payload_json TEXT NOT NULL, updated_at REAL NOT NULL, checksum TEXT, schema_version INTEGER)"
|
||||
)
|
||||
connection.execute(
|
||||
"INSERT INTO store_sections VALUES ('daily_plan', '{\"today\":true}', 1, '', 1)"
|
||||
)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
def make_memory_db(path: Path) -> None:
|
||||
connection = sqlite3.connect(path)
|
||||
for table in ("memories", "identities", "timeline"):
|
||||
connection.execute(f"CREATE TABLE {table} (id TEXT PRIMARY KEY)")
|
||||
connection.execute("INSERT INTO memories VALUES ('memory-1')")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
||||
|
||||
class OpsMigrationTests(unittest.TestCase):
|
||||
def test_companion_json_preview_and_execute_preserves_fields(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "old"
|
||||
target = root / "new"
|
||||
source.mkdir()
|
||||
payload = {
|
||||
"users": {"u1": {"preference": "tea"}},
|
||||
"daily_plan": {"date": "2026-07-24"},
|
||||
"important_dates": [{"name": "birthday"}],
|
||||
"bookshelf_items": [{"title": "private"}],
|
||||
"creative_projects": [{"title": "novel"}],
|
||||
"news_integration": {"enabled": True},
|
||||
}
|
||||
(source / "companions.json").write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||||
report = companion_migration.preview(source, target)
|
||||
self.assertTrue(report["ok"])
|
||||
self.assertIn("json:daily_plan", report["retained_fields"])
|
||||
self.assertIn("news_integration", report["flagged_fields"])
|
||||
result = companion_migration.execute(source, target)
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertEqual(payload, json.loads((target / "companions.json").read_text(encoding="utf-8")))
|
||||
self.assertTrue((target / companion_migration.MARKER_NAME).is_file())
|
||||
self.assertFalse(companion_migration.execute(source, target)["ok"])
|
||||
|
||||
def test_companion_sqlite_preview_and_execute_keeps_json_mirror(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "old"
|
||||
target = root / "new"
|
||||
source.mkdir()
|
||||
make_companion_db(source / "companions.db")
|
||||
(source / "companions.json").write_text('{"creative_projects": []}', encoding="utf-8")
|
||||
result = companion_migration.execute(source, target)
|
||||
self.assertTrue(result["ok"])
|
||||
self.assertTrue((target / "companions.db").is_file())
|
||||
self.assertTrue((target / "companions.json").is_file())
|
||||
check = sqlite3.connect(target / "companions.db")
|
||||
self.assertEqual(1, check.execute("SELECT COUNT(*) FROM store_sections").fetchone()[0])
|
||||
check.close()
|
||||
|
||||
def test_memory_migration_preserves_database_and_state(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "old"
|
||||
target = root / "new"
|
||||
source.mkdir()
|
||||
make_memory_db(source / memory_migration.DATABASE_NAME)
|
||||
(source / "memory_companion_sleep_state.json").write_text('{"sleeping":false}', encoding="utf-8")
|
||||
report = memory_migration.preview(source, target)
|
||||
self.assertTrue(report["ok"])
|
||||
self.assertIn("memories", report["retained_tables"])
|
||||
result = memory_migration.execute(source, target)
|
||||
self.assertTrue(result["ok"])
|
||||
check = sqlite3.connect(target / memory_migration.DATABASE_NAME)
|
||||
self.assertEqual(1, check.execute("SELECT COUNT(*) FROM memories").fetchone()[0])
|
||||
check.close()
|
||||
self.assertEqual('{"sleeping":false}', (target / "memory_companion_sleep_state.json").read_text(encoding="utf-8"))
|
||||
|
||||
def test_non_empty_target_and_invalid_source_are_rejected_without_changes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "old"
|
||||
target = root / "new"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
(target / "keep.txt").write_text("keep", encoding="utf-8")
|
||||
(source / "companions.json").write_text("[]", encoding="utf-8")
|
||||
report = companion_migration.preview(source, target)
|
||||
self.assertFalse(report["ok"])
|
||||
self.assertIn("非空", report["errors"][0])
|
||||
self.assertEqual("keep", (target / "keep.txt").read_text(encoding="utf-8"))
|
||||
|
||||
def test_memory_invalid_database_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
source = root / "old"
|
||||
source.mkdir()
|
||||
(source / memory_migration.DATABASE_NAME).write_bytes(b"not sqlite")
|
||||
report = memory_migration.preview(source, root / "new")
|
||||
self.assertFalse(report["ok"])
|
||||
self.assertTrue(report["errors"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
正在加载…
添加表格
在新工单中引用