astrbot_plugin_imagestudio/backend/services/data_backup_service.py

341 行
18 KiB
Python

"""Server-side ImageStudio data backup and guarded restore."""
from __future__ import annotations
import hashlib
import json
import shutil
import sqlite3
import tempfile
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from uuid import uuid4
from backend.dto.common import ApiError, ErrorCode
from core.config_service import ConfigService
from core.runtime_paths import resolve_data_path
from database.db_helper import load_config_payload, load_database_settings
MAX_BACKUP_FILES = 50_000
MAX_RESTORE_BYTES = 10 * 1024 * 1024 * 1024
PROMPT_BACKUP_PLACEHOLDER = "[EXCLUDED_FROM_BACKUP]"
class DataBackupService:
def __init__(self, config_service: ConfigService | None = None):
self.config_service = config_service or ConfigService()
def list_backups(self) -> list[dict[str, Any]]:
root = self._backup_root()
if not root.exists():
return []
return [self._summary(path) for path in sorted(root.glob("*.zip"), key=lambda item: item.stat().st_mtime, reverse=True)]
2026-07-12 02:54:45 +08:00
def create_backup(
self,
*,
include_media: bool = True,
include_prompt_records: bool = True,
include_moderation_records: bool = True,
) -> dict[str, Any]:
backup_id = f"backup_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}_{uuid4().hex[:8]}"
root = self._backup_root()
root.mkdir(parents=True, exist_ok=True)
destination = root / f"{backup_id}.zip"
config = load_config_payload(self.config_service.config_path)
entries: list[dict[str, Any]] = []
with tempfile.TemporaryDirectory(prefix="imagestudio-backup-") as temp_dir:
temp = Path(temp_dir)
config_copy = temp / "config.yaml"
if self.config_service.config_path.exists():
shutil.copy2(self.config_service.config_path, config_copy)
entries.append(_entry(config_copy, "config/config.yaml"))
database_settings = load_database_settings(self.config_service.config_path)
warnings: list[str] = []
if database_settings.selected_backend == "sqlite":
source_db = resolve_data_path(database_settings.sqlite.path)
if source_db.exists():
db_copy = temp / "imagestudio.db"
source = sqlite3.connect(source_db)
target = sqlite3.connect(db_copy)
try:
source.backup(target)
finally:
target.close(); source.close()
2026-07-12 02:54:45 +08:00
if not include_prompt_records or not include_moderation_records:
_sanitize_sqlite_backup(
db_copy,
include_prompt_records=include_prompt_records,
include_moderation_records=include_moderation_records,
)
entries.append(_entry(db_copy, "database/imagestudio.db"))
else:
warnings.append("PostgreSQL data is not embedded; use pg_dump with the connection recorded in the redacted config.")
storage = config.get("storage") if isinstance(config.get("storage"), dict) else {}
media_roots = {
key: resolve_data_path(storage.get(f"{key}_path") or f"data/{key}")
for key in ("images", "thumbnails", "materials", "advanced_materials", "styles")
}
if include_media:
for key, media_root in media_roots.items():
if not media_root.exists():
continue
for path in media_root.rglob("*"):
if not path.is_file():
continue
if len(entries) >= MAX_BACKUP_FILES:
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {"limit": MAX_BACKUP_FILES})
entries.append(_entry(path, f"media/{key}/{path.relative_to(media_root).as_posix()}"))
manifest = {
"format": "imagestudio-full-backup",
"version": 3,
2026-07-12 02:54:45 +08:00
"material_schema_version": "1.0",
"backup_id": backup_id,
"created_at": datetime.now(timezone.utc).isoformat(),
"database_backend": database_settings.selected_backend,
"include_media": include_media,
2026-07-12 02:54:45 +08:00
"include_prompt_records": include_prompt_records,
"include_moderation_records": include_moderation_records,
"entries": entries,
"warnings": warnings,
}
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
for item in entries:
source_path = config_copy if item["path"] == "config/config.yaml" else db_copy if item["path"] == "database/imagestudio.db" else _source_for_media_entry(item["path"], media_roots)
archive.write(source_path, item["path"])
archive.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
return self._summary(destination, manifest=manifest)
def inspect_backup(self, backup_id: str) -> dict[str, Any]:
path = self._backup_path(backup_id)
with zipfile.ZipFile(path) as archive:
manifest = _read_manifest(archive)
_validate_archive(archive, manifest)
return {**self._summary(path, manifest=manifest), "manifest": manifest, "valid": True}
def restore_backup(self, backup_id: str, *, confirm: bool = False) -> dict[str, Any]:
if not confirm:
return {**self.inspect_backup(backup_id), "restored": False, "requires_confirmation": True}
path = self._backup_path(backup_id)
with zipfile.ZipFile(path) as archive:
manifest = _read_manifest(archive)
_validate_archive(archive, manifest)
2026-07-12 02:54:45 +08:00
safety_backup = self.create_backup(include_media=False, include_prompt_records=True, include_moderation_records=True)
with tempfile.TemporaryDirectory(prefix="imagestudio-restore-") as temp_dir:
temp = Path(temp_dir)
archive.extractall(temp)
config_source = temp / "config" / "config.yaml"
if config_source.exists():
self.config_service.config_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(config_source, self.config_service.config_path)
database_settings = load_database_settings(self.config_service.config_path)
db_source = temp / "database" / "imagestudio.db"
if db_source.exists() and database_settings.selected_backend == "sqlite":
db_target = resolve_data_path(database_settings.sqlite.path)
db_target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(db_source, db_target)
config = load_config_payload(self.config_service.config_path)
storage = config.get("storage") if isinstance(config.get("storage"), dict) else {}
for key in ("images", "thumbnails", "materials", "advanced_materials", "styles"):
source_root = temp / "media" / key
if not source_root.exists():
continue
target_root = resolve_data_path(storage.get(f"{key}_path") or f"data/{key}")
target_root.mkdir(parents=True, exist_ok=True)
shutil.copytree(source_root, target_root, dirs_exist_ok=True)
return {"restored": True, "backup_id": backup_id, "safety_backup_id": safety_backup["backup_id"]}
def _backup_root(self) -> Path:
config = self.config_service.load()
return resolve_data_path(config.storage.archives_path) / "backups"
def _backup_path(self, backup_id: str) -> Path:
normalized = str(backup_id or "").strip()
if not normalized or any(char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for char in normalized):
raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup_id", {"backup_id": backup_id})
path = self._backup_root() / f"{normalized}.zip"
if not path.exists():
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "backup not found", {"backup_id": normalized})
return path
def _summary(self, path: Path, *, manifest: dict[str, Any] | None = None) -> dict[str, Any]:
loaded = manifest
if loaded is None:
try:
with zipfile.ZipFile(path) as archive:
loaded = _read_manifest(archive)
except Exception:
loaded = {}
stat = path.stat()
2026-07-12 02:54:45 +08:00
return {"backup_id": loaded.get("backup_id") or path.stem, "created_at": loaded.get("created_at") or datetime.fromtimestamp(stat.st_mtime,timezone.utc).isoformat(), "size_bytes": stat.st_size, "file_count": len(loaded.get("entries") or []), "database_backend": loaded.get("database_backend"), "include_media": bool(loaded.get("include_media")), "include_prompt_records": loaded.get("include_prompt_records", True), "include_moderation_records": loaded.get("include_moderation_records", True), "material_schema_version": loaded.get("material_schema_version"), "warnings": loaded.get("warnings") or []}
def _entry(path: Path, archive_path: str) -> dict[str, Any]:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return {"path": archive_path, "size": path.stat().st_size, "sha256": digest.hexdigest()}
def _source_for_media_entry(archive_path: str, roots: dict[str, Path]) -> Path:
_, key, relative = archive_path.split("/", 2)
return roots[key] / Path(relative)
def _read_manifest(archive: zipfile.ZipFile) -> dict[str, Any]:
try:
value = json.loads(archive.read("manifest.json"))
except (KeyError, json.JSONDecodeError) as exc:
raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup manifest", {}) from exc
if not isinstance(value, dict) or value.get("format") != "imagestudio-full-backup":
raise ApiError(ErrorCode.VALIDATION_FAILED, "unsupported backup format", {})
return value
def _validate_archive(archive: zipfile.ZipFile, manifest: dict[str, Any]) -> None:
entries = manifest.get("entries") if isinstance(manifest.get("entries"), list) else []
if len(entries) > MAX_BACKUP_FILES:
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {})
names = set(archive.namelist())
declared_names = {str(item.get("path") or "") for item in entries if isinstance(item, dict)}
if names - declared_names != {"manifest.json"}:
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup contains undeclared entries", {})
total = 0
for item in entries:
name = str(item.get("path") or "") if isinstance(item, dict) else ""
parts = Path(name).parts
if not name or name not in names or Path(name).is_absolute() or ".." in parts:
raise ApiError(ErrorCode.VALIDATION_FAILED, "unsafe backup entry", {"path": name})
info = archive.getinfo(name)
total += info.file_size
if total > MAX_RESTORE_BYTES:
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup exceeds restore size limit", {})
digest = hashlib.sha256(archive.read(name)).hexdigest()
if digest != item.get("sha256"):
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup checksum mismatch", {"path": name})
2026-07-12 02:54:45 +08:00
def _sanitize_sqlite_backup(
path: Path,
*,
include_prompt_records: bool,
include_moderation_records: bool,
) -> None:
conn = sqlite3.connect(path)
conn.row_factory = sqlite3.Row
try:
tables = {str(row["name"]) for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
if not include_prompt_records:
if "materials" in tables:
conn.execute("UPDATE materials SET prompt = NULL, negative_prompt = NULL")
if "advanced_material_imports" in tables:
conn.execute(
"UPDATE advanced_material_imports "
"SET recognition_prompt = NULL, recognition_negative_prompt = NULL"
)
if "advanced_material_validation_attempts" in tables:
conn.execute(
"UPDATE advanced_material_validation_attempts "
"SET prompt = ?, negative_prompt = NULL, adjustment_text = NULL, decision_notes = NULL",
(PROMPT_BACKUP_PLACEHOLDER,),
)
if "advanced_materials" in tables:
conn.execute(
"UPDATE advanced_materials SET prompt = ?, negative_prompt = NULL",
(PROMPT_BACKUP_PLACEHOLDER,),
)
if "advanced_style_sources" in tables:
conn.execute(
"UPDATE advanced_style_sources "
"SET recognition_prompt = NULL, recognition_negative_prompt = NULL"
)
if "advanced_style_consensus_runs" in tables:
conn.execute(
"UPDATE advanced_style_consensus_runs SET prompt = NULL, negative_prompt = NULL"
)
if "advanced_style_validation_cases" in tables:
conn.execute(
"UPDATE advanced_style_validation_cases "
"SET content_prompt = ?, final_prompt = ?, negative_prompt = NULL",
(PROMPT_BACKUP_PLACEHOLDER, PROMPT_BACKUP_PLACEHOLDER),
)
2026-07-12 02:54:45 +08:00
if "generation_images" in tables:
conn.execute("UPDATE generation_images SET positive_prompt = NULL, negative_prompt = NULL")
json_columns = {
"materials": ("material_id", ("metadata_json",)),
"advanced_material_imports": ("import_id", ("recognition_result_json", "error_json", "metadata_json")),
"advanced_material_validation_attempts": ("attempt_id", ("usage_json", "error_json", "metadata_json")),
"advanced_materials": ("material_id", ("capsule_json", "metadata_json")),
"advanced_style_sources": (
"source_id",
("recognition_tags_json", "recognition_result_json", "error_json", "metadata_json"),
),
"advanced_style_consensus_runs": (
"consensus_id",
("source_ids_json", "tags_json", "result_json", "error_json", "metadata_json"),
),
"advanced_style_validation_cases": (
"case_id",
("usage_json", "style_review_json", "error_json", "metadata_json"),
),
2026-07-12 02:54:45 +08:00
"generation_images": ("image_id", ("metadata_json",)),
"generation_tasks": ("task_id", ("request_json", "result_json", "metadata_json")),
"audit_logs": ("audit_id", ("before_snapshot", "after_snapshot", "metadata_json")),
}
for table, (primary_key, columns) in json_columns.items():
if table not in tables:
continue
existing_columns = {str(row["name"]) for row in conn.execute(f"PRAGMA table_info({table})")}
selected_columns = [column for column in columns if column in existing_columns]
if not selected_columns:
continue
for row in conn.execute(f"SELECT {primary_key}, {', '.join(selected_columns)} FROM {table}").fetchall():
updates: dict[str, str | None] = {}
for column in selected_columns:
raw = row[column]
if not raw:
continue
try:
parsed = json.loads(raw)
except (TypeError, json.JSONDecodeError):
continue
cleaned = _strip_sensitive_records(
parsed,
include_prompt_records=include_prompt_records,
include_moderation_records=include_moderation_records,
)
updates[column] = json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
if updates:
assignments = ", ".join(f"{column} = ?" for column in updates)
conn.execute(f"UPDATE {table} SET {assignments} WHERE {primary_key} = ?", (*updates.values(), row[primary_key]))
conn.commit()
finally:
conn.close()
def _strip_sensitive_records(
value: Any,
*,
include_prompt_records: bool,
include_moderation_records: bool,
) -> Any:
prompt_keys = {"prompt", "positive_prompt", "negative_prompt", "expanded_prompt", "expanded_negative_prompt", "original_text", "translated_text", "normalized_text", "source_payload"}
moderation_keys = {"prompt_governance", "moderation", "moderation_recheck", "image_moderation", "route_fallback", "generation_route_decision"}
if isinstance(value, dict):
result: dict[str, Any] = {}
for key, item in value.items():
lowered = str(key).lower()
if not include_prompt_records and lowered in prompt_keys:
result[key] = PROMPT_BACKUP_PLACEHOLDER
2026-07-12 02:54:45 +08:00
continue
if not include_moderation_records and lowered in moderation_keys:
continue
result[key] = _strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records)
return result
if isinstance(value, list):
return [_strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records) for item in value]
return value