feat: add safe companion data migration
这个提交存在于:
父节点
f496570b30
当前提交
f7c29f0e54
共有 3 个文件被更改,包括 271 次插入 和 0 次删除
19
docs/错题本.md
19
docs/错题本.md
|
|
@ -91,3 +91,22 @@
|
|||
- 修复或规避:本地建立 companion-main 基线、upstream-sync 和 ops-main;所有 Git 命令使用仓库级 safe.directory 参数,不修改全局配置;远端仓库创建与推送留待取得明确写权限。
|
||||
- 预防规则:未经成功创建和查询验证,不得把 metadata 中的远端地址宣称为已存在;推送前必须再次检查远端仓库与默认分支。
|
||||
- 验证证据:companion-main 基线提交为 6eaf1b7,当前分支为 ops-main;运维身份改为 astrbot_plugin_private_companion_ops;全仓 compileall 通过。
|
||||
\n\n## 2026-07-23 — 运维陪伴插件开发队列规划
|
||||
|
||||
- 阶段:运维版产品规划
|
||||
- 类型:功能行为 / API契约 / 性能
|
||||
- 现象:运维版同时包含人格、日程、外部事实、任务跟踪、主动提醒和部分阅读/创作能力,直接按文件顺序开发容易把人格和 Controller、生活日程和机器计划混在一起。
|
||||
- 根因:现有 PrivateCompanion 是大型 Mixin 组合,外部运维服务尚未形成统一 capability、事件和事实引用合同。
|
||||
- 修复或规避:制定 Q0-Q12 队列,先做数据迁移、运行时兼容层和运维事件模型,再做双层日程、只读事实、任务联动、主动提醒和娱乐删减;私密阅读与长期创作明确保留。
|
||||
- 预防规则:先只读后写入,先事实后表达;人格只委派,不直接拥有 Shell、SSH Key、Docker Socket 或 NAS 写权限;每阶段必须有独立提交、测试和回滚说明。
|
||||
- 验证证据:已核对 `ops-main` 当前提交 `f496570`、相关源码 Mixin 和既有运维融合评估,队列已写入 `运维陪伴插件开发队列.md`。
|
||||
|
||||
## 2026-07-24 — Q1 迁移器不能依赖启动自动搬迁
|
||||
|
||||
- 阶段:Q1 数据迁移与启动安全
|
||||
- 类型:功能行为 / 数据安全
|
||||
- 现象:普通版和运维版插件身份已经分开,但旧数据不会自动进入运维目录;如果直接在启动时搬迁,可能覆盖正在使用的目标数据。
|
||||
- 根因:插件名隔离只解决了新写入路径,没有定义一次性的跨目录迁移契约;启动阶段也不适合执行不可逆的数据搬运。
|
||||
- 修复或规避:增加独立迁移器,支持 preview/run、格式校验、目标非空拒绝、SQLite backup、临时目录、迁移标记和失败清理;启动过程不自动执行。
|
||||
- 预防规则:所有生产数据迁移必须先预览、再备份、后执行;目标目录必须为空;成功后保留迁移标记和源目录。
|
||||
- 验证证据:`tests/test_ops_migration.py` 5 项 unittest 通过;人格与记忆插件 compileall 通过。
|
||||
|
|
|
|||
252
ops_migration.py
普通文件
252
ops_migration.py
普通文件
|
|
@ -0,0 +1,252 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SOURCE_PLUGIN_NAMES = (
|
||||
"astrbot_plugin_private_companion",
|
||||
"astrbot_plugin_private_companion-main",
|
||||
)
|
||||
TARGET_PLUGIN_NAME = "astrbot_plugin_private_companion_ops"
|
||||
MIGRATION_VERSION = 1
|
||||
MARKER_NAME = "ops_migration_v1.json"
|
||||
SUPPORTED_JSON = "companions.json"
|
||||
SUPPORTED_SQLITE = "companions.db"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationFile:
|
||||
source: str
|
||||
target: str
|
||||
kind: str
|
||||
bytes: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationReport:
|
||||
ok: bool
|
||||
mode: str
|
||||
source_dir: str
|
||||
target_dir: str
|
||||
source_plugin: str = SOURCE_PLUGIN_NAMES[0]
|
||||
target_plugin: str = TARGET_PLUGIN_NAME
|
||||
source_format: str = ""
|
||||
source_files: list[MigrationFile] = field(default_factory=list)
|
||||
retained_fields: list[str] = field(default_factory=list)
|
||||
flagged_fields: list[str] = field(default_factory=list)
|
||||
skipped_fields: list[str] = field(default_factory=list)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
marker: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = asdict(self)
|
||||
result["source_files"] = [asdict(item) for item in self.source_files]
|
||||
return result
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _file(path: Path, target_name: str, kind: str) -> MigrationFile:
|
||||
return MigrationFile(str(path), target_name, kind, path.stat().st_size, _sha256(path))
|
||||
|
||||
|
||||
def _read_json_store(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("companions.json 顶层必须是对象")
|
||||
return payload
|
||||
|
||||
|
||||
def _validate_sqlite(path: Path) -> tuple[list[str], dict[str, int]]:
|
||||
connection = sqlite3.connect(f"file:{path.resolve()}?mode=ro", uri=True, timeout=3)
|
||||
try:
|
||||
result = connection.execute("PRAGMA integrity_check").fetchone()
|
||||
if not result or str(result[0]).lower() != "ok":
|
||||
raise ValueError(f"SQLite integrity_check 失败: {result[0] if result else 'unknown'}")
|
||||
tables = [row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")]
|
||||
if "store_sections" not in tables:
|
||||
raise ValueError("companions.db 缺少 store_sections 表")
|
||||
counts = {name: int(connection.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]) for name in tables}
|
||||
return tables, counts
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _candidate_files(source: Path) -> tuple[Path | None, Path | None]:
|
||||
json_path = source / SUPPORTED_JSON
|
||||
sqlite_path = source / SUPPORTED_SQLITE
|
||||
return (json_path if json_path.is_file() else None, sqlite_path if sqlite_path.is_file() else None)
|
||||
|
||||
|
||||
def _empty_target(target: Path) -> tuple[bool, str]:
|
||||
if not target.exists():
|
||||
return True, ""
|
||||
if not target.is_dir():
|
||||
return False, "目标路径不是目录"
|
||||
entries = list(target.iterdir())
|
||||
if not entries:
|
||||
return True, ""
|
||||
if (target / MARKER_NAME).is_file():
|
||||
return False, "目标目录已有迁移标记,拒绝重复迁移"
|
||||
return False, "目标目录非空,拒绝覆盖"
|
||||
|
||||
|
||||
def preview(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
|
||||
source = Path(source_dir).expanduser().resolve()
|
||||
target = Path(target_dir).expanduser().resolve()
|
||||
report = MigrationReport(ok=False, mode="preview", source_dir=str(source), target_dir=str(target))
|
||||
report.marker = str(target / MARKER_NAME)
|
||||
if not source.is_dir():
|
||||
report.errors.append("源数据目录不存在")
|
||||
return report.to_dict()
|
||||
empty, reason = _empty_target(target)
|
||||
if not empty:
|
||||
report.errors.append(reason)
|
||||
return report.to_dict()
|
||||
json_path, sqlite_path = _candidate_files(source)
|
||||
if not json_path and not sqlite_path:
|
||||
report.errors.append("未找到 companions.json 或 companions.db")
|
||||
return report.to_dict()
|
||||
if sqlite_path:
|
||||
try:
|
||||
tables, counts = _validate_sqlite(sqlite_path)
|
||||
report.source_format = "sqlite"
|
||||
report.source_files.append(_file(sqlite_path, SUPPORTED_SQLITE, "sqlite"))
|
||||
report.retained_fields.extend(f"sqlite.table:{name}" for name in tables)
|
||||
report.warnings.append(f"SQLite 表记录数:{counts}")
|
||||
except Exception as exc:
|
||||
report.errors.append(f"SQLite 校验失败:{exc}")
|
||||
return report.to_dict()
|
||||
if json_path:
|
||||
try:
|
||||
payload = _read_json_store(json_path)
|
||||
report.source_files.append(_file(json_path, SUPPORTED_JSON, "json-mirror"))
|
||||
report.retained_fields.extend(f"json:{key}" for key in sorted(payload))
|
||||
except Exception as exc:
|
||||
report.warnings.append(f"JSON 镜像无法读取,将只迁移 SQLite:{exc}")
|
||||
else:
|
||||
try:
|
||||
payload = _read_json_store(json_path)
|
||||
report.source_format = "json"
|
||||
report.source_files.append(_file(json_path, SUPPORTED_JSON, "json"))
|
||||
report.retained_fields.extend(f"json:{key}" for key in sorted(payload))
|
||||
except Exception as exc:
|
||||
report.errors.append(f"JSON 校验失败:{exc}")
|
||||
return report.to_dict()
|
||||
flagged = {"news_integration", "web_exploration", "qzone_integration", "bilibili_integration"}
|
||||
report.flagged_fields = sorted(
|
||||
field_name.split(":", 1)[1]
|
||||
for field_name in report.retained_fields
|
||||
if field_name.startswith("json:") and field_name.split(":", 1)[1] in flagged
|
||||
)
|
||||
report.ok = not report.errors
|
||||
return report.to_dict()
|
||||
|
||||
|
||||
def _backup_sqlite(source: Path, target: Path) -> None:
|
||||
source_conn = sqlite3.connect(f"file:{source.resolve()}?mode=ro", uri=True, timeout=3)
|
||||
try:
|
||||
target_conn = sqlite3.connect(str(target), timeout=3)
|
||||
try:
|
||||
source_conn.backup(target_conn)
|
||||
target_conn.commit()
|
||||
finally:
|
||||
target_conn.close()
|
||||
finally:
|
||||
source_conn.close()
|
||||
|
||||
|
||||
def _report_from_dict(payload: dict[str, Any]) -> MigrationReport:
|
||||
fields = {key: value for key, value in payload.items() if key in MigrationReport.__dataclass_fields__}
|
||||
fields["source_files"] = [MigrationFile(**item) for item in fields.get("source_files", [])]
|
||||
return MigrationReport(**fields)
|
||||
|
||||
|
||||
def execute(source_dir: str | Path, target_dir: str | Path) -> dict[str, Any]:
|
||||
report = _report_from_dict(preview(source_dir, target_dir))
|
||||
report.mode = "execute"
|
||||
if not report.ok:
|
||||
return report.to_dict()
|
||||
source = Path(report.source_dir)
|
||||
target = Path(report.target_dir)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".ops-migration-", dir=str(target.parent)))
|
||||
moved: list[Path] = []
|
||||
try:
|
||||
json_path, sqlite_path = _candidate_files(source)
|
||||
if sqlite_path:
|
||||
_backup_sqlite(sqlite_path, staging / SUPPORTED_SQLITE)
|
||||
if json_path and any(item.target == SUPPORTED_JSON and item.kind in {"json", "json-mirror"} for item in report.source_files):
|
||||
shutil.copy2(json_path, staging / SUPPORTED_JSON)
|
||||
marker_payload = {
|
||||
"migration_version": MIGRATION_VERSION,
|
||||
"source_plugin": SOURCE_PLUGIN_NAMES[0],
|
||||
"target_plugin": TARGET_PLUGIN_NAME,
|
||||
"source_dir": str(source),
|
||||
"target_dir": str(target),
|
||||
"source_format": report.source_format,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
"files": [asdict(item) for item in report.source_files],
|
||||
}
|
||||
(staging / MARKER_NAME).write_text(
|
||||
json.dumps(marker_payload, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
for item in staging.iterdir():
|
||||
destination = target / item.name
|
||||
if destination.exists():
|
||||
raise FileExistsError(f"目标文件已存在:{destination}")
|
||||
os.replace(item, destination)
|
||||
moved.append(destination)
|
||||
report.marker = str(target / MARKER_NAME)
|
||||
report.warnings.append("迁移完成;源目录保持不变,删除旧目录前请先做 VM 验收")
|
||||
return report.to_dict()
|
||||
except Exception as exc:
|
||||
for destination in reversed(moved):
|
||||
try:
|
||||
destination.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
report.ok = False
|
||||
report.errors.append(f"迁移失败,源目录未修改:{exc}")
|
||||
return report.to_dict()
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def format_report(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
f"迁移状态:{'可执行' if report.get('ok') else '拒绝/失败'}",
|
||||
f"模式:{report.get('mode')}",
|
||||
f"来源:{report.get('source_dir')}",
|
||||
f"目标:{report.get('target_dir')}",
|
||||
f"格式:{report.get('source_format') or '-'}",
|
||||
]
|
||||
if report.get("retained_fields"):
|
||||
lines.append(f"保留字段/表:{len(report['retained_fields'])} 项")
|
||||
if report.get("flagged_fields"):
|
||||
lines.append("待后续清理字段:" + ", ".join(report["flagged_fields"]))
|
||||
for error in report.get("errors", []):
|
||||
lines.append("错误:" + str(error))
|
||||
for warning in report.get("warnings", []):
|
||||
lines.append("提示:" + str(warning))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
|
||||
0
tests/__init__.py
普通文件
0
tests/__init__.py
普通文件
正在加载…
添加表格
在新工单中引用