feat: add safe companion data migration

这个提交存在于:
YokiiroBW 2026-07-24 21:10:46 +08:00
父节点 f496570b30
当前提交 f7c29f0e54
共有 3 个文件被更改,包括 271 次插入 和 0 次删除

查看文件

@ -91,3 +91,22 @@
- 修复或规避:本地建立 companion-main 基线、upstream-sync 和 ops-main;所有 Git 命令使用仓库级 safe.directory 参数,不修改全局配置;远端仓库创建与推送留待取得明确写权限。
- 预防规则:未经成功创建和查询验证,不得把 metadata 中的远端地址宣称为已存在;推送前必须再次检查远端仓库与默认分支。
- 验证证据:companion-main 基线提交为 6eaf1b7,当前分支为 ops-main;运维身份改为 astrbot_plugin_private_companion_ops;全仓 compileall 通过。
\n\n## 2026-07-23 — 运维陪伴插件开发队列规划
- 阶段:运维版产品规划
- 类型:功能行为 / API契约 / 性能
- 现象:运维版同时包含人格、日程、外部事实、任务跟踪、主动提醒和部分阅读/创作能力,直接按文件顺序开发容易把人格和 Controller、生活日程和机器计划混在一起。
- 根因:现有 PrivateCompanion 是大型 Mixin 组合,外部运维服务尚未形成统一 capability、事件和事实引用合同。
- 修复或规避:制定 Q0-Q12 队列,先做数据迁移、运行时兼容层和运维事件模型,再做双层日程、只读事实、任务联动、主动提醒和娱乐删减;私密阅读与长期创作明确保留。
- 预防规则:先只读后写入,先事实后表达;人格只委派,不直接拥有 Shell、SSH Key、Docker Socket 或 NAS 写权限;每阶段必须有独立提交、测试和回滚说明。
- 验证证据:已核对 `ops-main` 当前提交 `f496570`、相关源码 Mixin 和既有运维融合评估,队列已写入 `运维陪伴插件开发队列.md`。
## 2026-07-24 — Q1 迁移器不能依赖启动自动搬迁
- 阶段:Q1 数据迁移与启动安全
- 类型:功能行为 / 数据安全
- 现象:普通版和运维版插件身份已经分开,但旧数据不会自动进入运维目录;如果直接在启动时搬迁,可能覆盖正在使用的目标数据。
- 根因:插件名隔离只解决了新写入路径,没有定义一次性的跨目录迁移契约;启动阶段也不适合执行不可逆的数据搬运。
- 修复或规避:增加独立迁移器,支持 preview/run、格式校验、目标非空拒绝、SQLite backup、临时目录、迁移标记和失败清理;启动过程不自动执行。
- 预防规则:所有生产数据迁移必须先预览、再备份、后执行;目标目录必须为空;成功后保留迁移标记和源目录。
- 验证证据:`tests/test_ops_migration.py` 5 项 unittest 通过;人格与记忆插件 compileall 通过。

252
ops_migration.py 普通文件
查看文件

@ -0,0 +1,252 @@
from __future__ import annotations
import hashlib
import json
import os
import shutil
import sqlite3
import tempfile
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
SOURCE_PLUGIN_NAMES = (
"astrbot_plugin_private_companion",
"astrbot_plugin_private_companion-main",
)
TARGET_PLUGIN_NAME = "astrbot_plugin_private_companion_ops"
MIGRATION_VERSION = 1
MARKER_NAME = "ops_migration_v1.json"
SUPPORTED_JSON = "companions.json"
SUPPORTED_SQLITE = "companions.db"
@dataclass(frozen=True)
class MigrationFile:
source: str
target: str
kind: str
bytes: int
sha256: str
@dataclass
class MigrationReport:
ok: bool
mode: str
source_dir: str
target_dir: str
source_plugin: str = SOURCE_PLUGIN_NAMES[0]
target_plugin: str = TARGET_PLUGIN_NAME
source_format: str = ""
source_files: list[MigrationFile] = field(default_factory=list)
retained_fields: list[str] = field(default_factory=list)
flagged_fields: list[str] = field(default_factory=list)
skipped_fields: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
marker: str = ""
def to_dict(self) -> dict[str, Any]:
result = asdict(self)
result["source_files"] = [asdict(item) for item in self.source_files]
return result
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _file(path: Path, target_name: str, kind: str) -> MigrationFile:
return MigrationFile(str(path), target_name, kind, path.stat().st_size, _sha256(path))
def _read_json_store(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise ValueError("companions.json 顶层必须是对象")
return payload
def _validate_sqlite(path: Path) -> tuple[list[str], dict[str, int]]:
connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True, timeout=3)
try:
result = connection.execute("PRAGMA integrity_check").fetchone()
if not result or str(result[0]).lower() != "ok":
raise ValueError(f"SQLite integrity_check 失败: {result[0] if result else 'unknown'}")
tables = [row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")]
if "store_sections" not in tables:
raise ValueError("companions.db 缺少 store_sections 表")
counts = {name: int(connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]) for name in tables}
return tables, counts
finally:
connection.close()
def _candidate_files(source: Path) -> tuple[Path | None, Path | None]:
json_path = source / SUPPORTED_JSON
sqlite_path = source / SUPPORTED_SQLITE
return (json_path if json_path.is_file() else None, sqlite_path if sqlite_path.is_file() else None)
def _empty_target(target: Path) -> tuple[bool, str]:
if not target.exists():
return True, ""
if not target.is_dir():
return False, "目标路径不是目录"
entries = list(target.iterdir())
if not entries:
return True, ""
if (target / MARKER_NAME).is_file():
return False, "目标目录已有迁移标记,拒绝重复迁移"
return False, "目标目录非空,拒绝覆盖"
def preview(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
source = Path(source_dir).expanduser().resolve()
target = Path(target_dir).expanduser().resolve()
report = MigrationReport(ok=False, mode="preview", source_dir=str(source), target_dir=str(target))
report.marker = str(target / MARKER_NAME)
if not source.is_dir():
report.errors.append("源数据目录不存在")
return report.to_dict()
empty, reason = _empty_target(target)
if not empty:
report.errors.append(reason)
return report.to_dict()
json_path, sqlite_path = _candidate_files(source)
if not json_path and not sqlite_path:
report.errors.append("未找到 companions.json 或 companions.db")
return report.to_dict()
if sqlite_path:
try:
tables, counts = _validate_sqlite(sqlite_path)
report.source_format = "sqlite"
report.source_files.append(_file(sqlite_path, SUPPORTED_SQLITE, "sqlite"))
report.retained_fields.extend(f"sqlite.table:{name}" for name in tables)
report.warnings.append(f"SQLite 表记录数:{counts}")
except Exception as exc:
report.errors.append(f"SQLite 校验失败:{exc}")
return report.to_dict()
if json_path:
try:
payload = _read_json_store(json_path)
report.source_files.append(_file(json_path, SUPPORTED_JSON, "json-mirror"))
report.retained_fields.extend(f"json:{key}" for key in sorted(payload))
except Exception as exc:
report.warnings.append(f"JSON 镜像无法读取,将只迁移 SQLite:{exc}")
else:
try:
payload = _read_json_store(json_path)
report.source_format = "json"
report.source_files.append(_file(json_path, SUPPORTED_JSON, "json"))
report.retained_fields.extend(f"json:{key}" for key in sorted(payload))
except Exception as exc:
report.errors.append(f"JSON 校验失败:{exc}")
return report.to_dict()
flagged = {"news_integration", "web_exploration", "qzone_integration", "bilibili_integration"}
report.flagged_fields = sorted(
field_name.split(":", 1)[1]
for field_name in report.retained_fields
if field_name.startswith("json:") and field_name.split(":", 1)[1] in flagged
)
report.ok = not report.errors
return report.to_dict()
def _backup_sqlite(source: Path, target: Path) -> None:
source_conn = sqlite3.connect(f"file:{source.resolve()}?mode=ro", uri=True, timeout=3)
try:
target_conn = sqlite3.connect(str(target), timeout=3)
try:
source_conn.backup(target_conn)
target_conn.commit()
finally:
target_conn.close()
finally:
source_conn.close()
def _report_from_dict(payload: dict[str, Any]) -> MigrationReport:
fields = {key: value for key, value in payload.items() if key in MigrationReport.__dataclass_fields__}
fields["source_files"] = [MigrationFile(**item) for item in fields.get("source_files", [])]
return MigrationReport(**fields)
def execute(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
report = _report_from_dict(preview(source_dir, target_dir))
report.mode = "execute"
if not report.ok:
return report.to_dict()
source = Path(report.source_dir)
target = Path(report.target_dir)
target.mkdir(parents=True, exist_ok=True)
staging = Path(tempfile.mkdtemp(prefix=".ops-migration-", dir=str(target.parent)))
moved: list[Path] = []
try:
json_path, sqlite_path = _candidate_files(source)
if sqlite_path:
_backup_sqlite(sqlite_path, staging / SUPPORTED_SQLITE)
if json_path and any(item.target == SUPPORTED_JSON and item.kind in {"json", "json-mirror"} for item in report.source_files):
shutil.copy2(json_path, staging / SUPPORTED_JSON)
marker_payload = {
"migration_version": MIGRATION_VERSION,
"source_plugin": SOURCE_PLUGIN_NAMES[0],
"target_plugin": TARGET_PLUGIN_NAME,
"source_dir": str(source),
"target_dir": str(target),
"source_format": report.source_format,
"created_at": datetime.now(timezone.utc).isoformat(),
"files": [asdict(item) for item in report.source_files],
}
(staging / MARKER_NAME).write_text(
json.dumps(marker_payload, ensure_ascii=False, indent=2), encoding="utf-8"
)
for item in staging.iterdir():
destination = target / item.name
if destination.exists():
raise FileExistsError(f"目标文件已存在:{destination}")
os.replace(item, destination)
moved.append(destination)
report.marker = str(target / MARKER_NAME)
report.warnings.append("迁移完成;源目录保持不变,删除旧目录前请先做 VM 验收")
return report.to_dict()
except Exception as exc:
for destination in reversed(moved):
try:
destination.unlink()
except OSError:
pass
report.ok = False
report.errors.append(f"迁移失败,源目录未修改:{exc}")
return report.to_dict()
finally:
shutil.rmtree(staging, ignore_errors=True)
def format_report(report: dict[str, Any]) -> str:
lines = [
f"迁移状态:{'可执行' if report.get('ok') else '拒绝/失败'}",
f"模式:{report.get('mode')}",
f"来源:{report.get('source_dir')}",
f"目标:{report.get('target_dir')}",
f"格式:{report.get('source_format') or '-'}",
]
if report.get("retained_fields"):
lines.append(f"保留字段/表:{len(report['retained_fields'])} 项")
if report.get("flagged_fields"):
lines.append("待后续清理字段:" + ", ".join(report["flagged_fields"]))
for error in report.get("errors", []):
lines.append("错误:" + str(error))
for warning in report.get("warnings", []):
lines.append("提示:" + str(warning))
return "\n".join(lines)

0
tests/__init__.py 普通文件
查看文件