2026-07-11 22:27:35 +08:00
|
|
|
"""Server-side ImageStudio data backup and guarded restore."""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import hashlib
|
|
|
|
|
import json
|
|
|
|
|
import shutil
|
|
|
|
|
import sqlite3
|
|
|
|
|
import tempfile
|
|
|
|
|
import zipfile
|
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Any
|
|
|
|
|
from uuid import uuid4
|
|
|
|
|
|
|
|
|
|
from backend.dto.common import ApiError, ErrorCode
|
|
|
|
|
from core.config_service import ConfigService
|
|
|
|
|
from core.runtime_paths import resolve_data_path
|
|
|
|
|
from database.db_helper import load_config_payload, load_database_settings
|
|
|
|
|
|
|
|
|
|
MAX_BACKUP_FILES = 50_000
|
|
|
|
|
MAX_RESTORE_BYTES = 10 * 1024 * 1024 * 1024
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DataBackupService:
|
|
|
|
|
def __init__(self, config_service: ConfigService | None = None):
|
|
|
|
|
self.config_service = config_service or ConfigService()
|
|
|
|
|
|
|
|
|
|
def list_backups(self) -> list[dict[str, Any]]:
|
|
|
|
|
root = self._backup_root()
|
|
|
|
|
if not root.exists():
|
|
|
|
|
return []
|
|
|
|
|
return [self._summary(path) for path in sorted(root.glob("*.zip"), key=lambda item: item.stat().st_mtime, reverse=True)]
|
|
|
|
|
|
2026-07-12 02:54:45 +08:00
|
|
|
def create_backup(
|
|
|
|
|
self,
|
|
|
|
|
*,
|
|
|
|
|
include_media: bool = True,
|
|
|
|
|
include_prompt_records: bool = True,
|
|
|
|
|
include_moderation_records: bool = True,
|
|
|
|
|
) -> dict[str, Any]:
|
2026-07-11 22:27:35 +08:00
|
|
|
backup_id = f"backup_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}_{uuid4().hex[:8]}"
|
|
|
|
|
root = self._backup_root()
|
|
|
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
destination = root / f"{backup_id}.zip"
|
|
|
|
|
config = load_config_payload(self.config_service.config_path)
|
|
|
|
|
entries: list[dict[str, Any]] = []
|
|
|
|
|
with tempfile.TemporaryDirectory(prefix="imagestudio-backup-") as temp_dir:
|
|
|
|
|
temp = Path(temp_dir)
|
|
|
|
|
config_copy = temp / "config.yaml"
|
|
|
|
|
if self.config_service.config_path.exists():
|
|
|
|
|
shutil.copy2(self.config_service.config_path, config_copy)
|
|
|
|
|
entries.append(_entry(config_copy, "config/config.yaml"))
|
|
|
|
|
database_settings = load_database_settings(self.config_service.config_path)
|
|
|
|
|
warnings: list[str] = []
|
|
|
|
|
if database_settings.selected_backend == "sqlite":
|
|
|
|
|
source_db = resolve_data_path(database_settings.sqlite.path)
|
|
|
|
|
if source_db.exists():
|
|
|
|
|
db_copy = temp / "imagestudio.db"
|
|
|
|
|
source = sqlite3.connect(source_db)
|
|
|
|
|
target = sqlite3.connect(db_copy)
|
|
|
|
|
try:
|
|
|
|
|
source.backup(target)
|
|
|
|
|
finally:
|
|
|
|
|
target.close(); source.close()
|
2026-07-12 02:54:45 +08:00
|
|
|
if not include_prompt_records or not include_moderation_records:
|
|
|
|
|
_sanitize_sqlite_backup(
|
|
|
|
|
db_copy,
|
|
|
|
|
include_prompt_records=include_prompt_records,
|
|
|
|
|
include_moderation_records=include_moderation_records,
|
|
|
|
|
)
|
2026-07-11 22:27:35 +08:00
|
|
|
entries.append(_entry(db_copy, "database/imagestudio.db"))
|
|
|
|
|
else:
|
|
|
|
|
warnings.append("PostgreSQL data is not embedded; use pg_dump with the connection recorded in the redacted config.")
|
|
|
|
|
storage = config.get("storage") if isinstance(config.get("storage"), dict) else {}
|
|
|
|
|
media_roots = {
|
|
|
|
|
key: resolve_data_path(storage.get(f"{key}_path") or f"data/{key}")
|
|
|
|
|
for key in ("images", "thumbnails", "materials", "styles")
|
|
|
|
|
}
|
|
|
|
|
if include_media:
|
|
|
|
|
for key, media_root in media_roots.items():
|
|
|
|
|
if not media_root.exists():
|
|
|
|
|
continue
|
|
|
|
|
for path in media_root.rglob("*"):
|
|
|
|
|
if not path.is_file():
|
|
|
|
|
continue
|
|
|
|
|
if len(entries) >= MAX_BACKUP_FILES:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {"limit": MAX_BACKUP_FILES})
|
|
|
|
|
entries.append(_entry(path, f"media/{key}/{path.relative_to(media_root).as_posix()}"))
|
|
|
|
|
manifest = {
|
|
|
|
|
"format": "imagestudio-full-backup",
|
2026-07-12 02:54:45 +08:00
|
|
|
"version": 2,
|
|
|
|
|
"material_schema_version": "1.0",
|
2026-07-11 22:27:35 +08:00
|
|
|
"backup_id": backup_id,
|
|
|
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
|
|
|
"database_backend": database_settings.selected_backend,
|
|
|
|
|
"include_media": include_media,
|
2026-07-12 02:54:45 +08:00
|
|
|
"include_prompt_records": include_prompt_records,
|
|
|
|
|
"include_moderation_records": include_moderation_records,
|
2026-07-11 22:27:35 +08:00
|
|
|
"entries": entries,
|
|
|
|
|
"warnings": warnings,
|
|
|
|
|
}
|
|
|
|
|
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
|
|
|
|
|
for item in entries:
|
|
|
|
|
source_path = config_copy if item["path"] == "config/config.yaml" else db_copy if item["path"] == "database/imagestudio.db" else _source_for_media_entry(item["path"], media_roots)
|
|
|
|
|
archive.write(source_path, item["path"])
|
|
|
|
|
archive.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
|
|
|
|
|
return self._summary(destination, manifest=manifest)
|
|
|
|
|
|
|
|
|
|
def inspect_backup(self, backup_id: str) -> dict[str, Any]:
|
|
|
|
|
path = self._backup_path(backup_id)
|
|
|
|
|
with zipfile.ZipFile(path) as archive:
|
|
|
|
|
manifest = _read_manifest(archive)
|
|
|
|
|
_validate_archive(archive, manifest)
|
|
|
|
|
return {**self._summary(path, manifest=manifest), "manifest": manifest, "valid": True}
|
|
|
|
|
|
|
|
|
|
def restore_backup(self, backup_id: str, *, confirm: bool = False) -> dict[str, Any]:
|
|
|
|
|
if not confirm:
|
|
|
|
|
return {**self.inspect_backup(backup_id), "restored": False, "requires_confirmation": True}
|
|
|
|
|
path = self._backup_path(backup_id)
|
|
|
|
|
with zipfile.ZipFile(path) as archive:
|
|
|
|
|
manifest = _read_manifest(archive)
|
|
|
|
|
_validate_archive(archive, manifest)
|
2026-07-12 02:54:45 +08:00
|
|
|
safety_backup = self.create_backup(include_media=False, include_prompt_records=True, include_moderation_records=True)
|
2026-07-11 22:27:35 +08:00
|
|
|
with tempfile.TemporaryDirectory(prefix="imagestudio-restore-") as temp_dir:
|
|
|
|
|
temp = Path(temp_dir)
|
|
|
|
|
archive.extractall(temp)
|
|
|
|
|
config_source = temp / "config" / "config.yaml"
|
|
|
|
|
if config_source.exists():
|
|
|
|
|
self.config_service.config_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
shutil.copy2(config_source, self.config_service.config_path)
|
|
|
|
|
database_settings = load_database_settings(self.config_service.config_path)
|
|
|
|
|
db_source = temp / "database" / "imagestudio.db"
|
|
|
|
|
if db_source.exists() and database_settings.selected_backend == "sqlite":
|
|
|
|
|
db_target = resolve_data_path(database_settings.sqlite.path)
|
|
|
|
|
db_target.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
shutil.copy2(db_source, db_target)
|
|
|
|
|
config = load_config_payload(self.config_service.config_path)
|
|
|
|
|
storage = config.get("storage") if isinstance(config.get("storage"), dict) else {}
|
|
|
|
|
for key in ("images", "thumbnails", "materials", "styles"):
|
|
|
|
|
source_root = temp / "media" / key
|
|
|
|
|
if not source_root.exists():
|
|
|
|
|
continue
|
|
|
|
|
target_root = resolve_data_path(storage.get(f"{key}_path") or f"data/{key}")
|
|
|
|
|
target_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
shutil.copytree(source_root, target_root, dirs_exist_ok=True)
|
|
|
|
|
return {"restored": True, "backup_id": backup_id, "safety_backup_id": safety_backup["backup_id"]}
|
|
|
|
|
|
|
|
|
|
def _backup_root(self) -> Path:
|
|
|
|
|
config = self.config_service.load()
|
|
|
|
|
return resolve_data_path(config.storage.archives_path) / "backups"
|
|
|
|
|
|
|
|
|
|
def _backup_path(self, backup_id: str) -> Path:
|
|
|
|
|
normalized = str(backup_id or "").strip()
|
|
|
|
|
if not normalized or any(char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for char in normalized):
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup_id", {"backup_id": backup_id})
|
|
|
|
|
path = self._backup_root() / f"{normalized}.zip"
|
|
|
|
|
if not path.exists():
|
|
|
|
|
raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "backup not found", {"backup_id": normalized})
|
|
|
|
|
return path
|
|
|
|
|
|
|
|
|
|
def _summary(self, path: Path, *, manifest: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
|
|
|
loaded = manifest
|
|
|
|
|
if loaded is None:
|
|
|
|
|
try:
|
|
|
|
|
with zipfile.ZipFile(path) as archive:
|
|
|
|
|
loaded = _read_manifest(archive)
|
|
|
|
|
except Exception:
|
|
|
|
|
loaded = {}
|
|
|
|
|
stat = path.stat()
|
2026-07-12 02:54:45 +08:00
|
|
|
return {"backup_id": loaded.get("backup_id") or path.stem, "created_at": loaded.get("created_at") or datetime.fromtimestamp(stat.st_mtime,timezone.utc).isoformat(), "size_bytes": stat.st_size, "file_count": len(loaded.get("entries") or []), "database_backend": loaded.get("database_backend"), "include_media": bool(loaded.get("include_media")), "include_prompt_records": loaded.get("include_prompt_records", True), "include_moderation_records": loaded.get("include_moderation_records", True), "material_schema_version": loaded.get("material_schema_version"), "warnings": loaded.get("warnings") or []}
|
2026-07-11 22:27:35 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _entry(path: Path, archive_path: str) -> dict[str, Any]:
|
|
|
|
|
digest = hashlib.sha256()
|
|
|
|
|
with path.open("rb") as handle:
|
|
|
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
|
|
|
digest.update(chunk)
|
|
|
|
|
return {"path": archive_path, "size": path.stat().st_size, "sha256": digest.hexdigest()}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _source_for_media_entry(archive_path: str, roots: dict[str, Path]) -> Path:
|
|
|
|
|
_, key, relative = archive_path.split("/", 2)
|
|
|
|
|
return roots[key] / Path(relative)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _read_manifest(archive: zipfile.ZipFile) -> dict[str, Any]:
|
|
|
|
|
try:
|
|
|
|
|
value = json.loads(archive.read("manifest.json"))
|
|
|
|
|
except (KeyError, json.JSONDecodeError) as exc:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup manifest", {}) from exc
|
|
|
|
|
if not isinstance(value, dict) or value.get("format") != "imagestudio-full-backup":
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "unsupported backup format", {})
|
|
|
|
|
return value
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_archive(archive: zipfile.ZipFile, manifest: dict[str, Any]) -> None:
|
|
|
|
|
entries = manifest.get("entries") if isinstance(manifest.get("entries"), list) else []
|
|
|
|
|
if len(entries) > MAX_BACKUP_FILES:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {})
|
|
|
|
|
names = set(archive.namelist())
|
|
|
|
|
declared_names = {str(item.get("path") or "") for item in entries if isinstance(item, dict)}
|
|
|
|
|
if names - declared_names != {"manifest.json"}:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup contains undeclared entries", {})
|
|
|
|
|
total = 0
|
|
|
|
|
for item in entries:
|
|
|
|
|
name = str(item.get("path") or "") if isinstance(item, dict) else ""
|
|
|
|
|
parts = Path(name).parts
|
|
|
|
|
if not name or name not in names or Path(name).is_absolute() or ".." in parts:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "unsafe backup entry", {"path": name})
|
|
|
|
|
info = archive.getinfo(name)
|
|
|
|
|
total += info.file_size
|
|
|
|
|
if total > MAX_RESTORE_BYTES:
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup exceeds restore size limit", {})
|
|
|
|
|
digest = hashlib.sha256(archive.read(name)).hexdigest()
|
|
|
|
|
if digest != item.get("sha256"):
|
|
|
|
|
raise ApiError(ErrorCode.VALIDATION_FAILED, "backup checksum mismatch", {"path": name})
|
2026-07-12 02:54:45 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def _sanitize_sqlite_backup(
|
|
|
|
|
path: Path,
|
|
|
|
|
*,
|
|
|
|
|
include_prompt_records: bool,
|
|
|
|
|
include_moderation_records: bool,
|
|
|
|
|
) -> None:
|
|
|
|
|
conn = sqlite3.connect(path)
|
|
|
|
|
conn.row_factory = sqlite3.Row
|
|
|
|
|
try:
|
|
|
|
|
tables = {str(row["name"]) for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
|
|
|
|
|
if not include_prompt_records:
|
|
|
|
|
if "materials" in tables:
|
|
|
|
|
conn.execute("UPDATE materials SET prompt = NULL, negative_prompt = NULL")
|
|
|
|
|
if "generation_images" in tables:
|
|
|
|
|
conn.execute("UPDATE generation_images SET positive_prompt = NULL, negative_prompt = NULL")
|
|
|
|
|
json_columns = {
|
|
|
|
|
"materials": ("material_id", ("metadata_json",)),
|
|
|
|
|
"generation_images": ("image_id", ("metadata_json",)),
|
|
|
|
|
"generation_tasks": ("task_id", ("request_json", "result_json", "metadata_json")),
|
|
|
|
|
"audit_logs": ("audit_id", ("before_snapshot", "after_snapshot", "metadata_json")),
|
|
|
|
|
}
|
|
|
|
|
for table, (primary_key, columns) in json_columns.items():
|
|
|
|
|
if table not in tables:
|
|
|
|
|
continue
|
|
|
|
|
existing_columns = {str(row["name"]) for row in conn.execute(f"PRAGMA table_info({table})")}
|
|
|
|
|
selected_columns = [column for column in columns if column in existing_columns]
|
|
|
|
|
if not selected_columns:
|
|
|
|
|
continue
|
|
|
|
|
for row in conn.execute(f"SELECT {primary_key}, {', '.join(selected_columns)} FROM {table}").fetchall():
|
|
|
|
|
updates: dict[str, str | None] = {}
|
|
|
|
|
for column in selected_columns:
|
|
|
|
|
raw = row[column]
|
|
|
|
|
if not raw:
|
|
|
|
|
continue
|
|
|
|
|
try:
|
|
|
|
|
parsed = json.loads(raw)
|
|
|
|
|
except (TypeError, json.JSONDecodeError):
|
|
|
|
|
continue
|
|
|
|
|
cleaned = _strip_sensitive_records(
|
|
|
|
|
parsed,
|
|
|
|
|
include_prompt_records=include_prompt_records,
|
|
|
|
|
include_moderation_records=include_moderation_records,
|
|
|
|
|
)
|
|
|
|
|
updates[column] = json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
|
if updates:
|
|
|
|
|
assignments = ", ".join(f"{column} = ?" for column in updates)
|
|
|
|
|
conn.execute(f"UPDATE {table} SET {assignments} WHERE {primary_key} = ?", (*updates.values(), row[primary_key]))
|
|
|
|
|
conn.commit()
|
|
|
|
|
finally:
|
|
|
|
|
conn.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _strip_sensitive_records(
|
|
|
|
|
value: Any,
|
|
|
|
|
*,
|
|
|
|
|
include_prompt_records: bool,
|
|
|
|
|
include_moderation_records: bool,
|
|
|
|
|
) -> Any:
|
|
|
|
|
prompt_keys = {"prompt", "positive_prompt", "negative_prompt", "expanded_prompt", "expanded_negative_prompt", "original_text", "translated_text", "normalized_text", "source_payload"}
|
|
|
|
|
moderation_keys = {"prompt_governance", "moderation", "moderation_recheck", "image_moderation", "route_fallback", "generation_route_decision"}
|
|
|
|
|
if isinstance(value, dict):
|
|
|
|
|
result: dict[str, Any] = {}
|
|
|
|
|
for key, item in value.items():
|
|
|
|
|
lowered = str(key).lower()
|
|
|
|
|
if not include_prompt_records and lowered in prompt_keys:
|
|
|
|
|
result[key] = "[EXCLUDED_FROM_BACKUP]"
|
|
|
|
|
continue
|
|
|
|
|
if not include_moderation_records and lowered in moderation_keys:
|
|
|
|
|
continue
|
|
|
|
|
result[key] = _strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records)
|
|
|
|
|
return result
|
|
|
|
|
if isinstance(value, list):
|
|
|
|
|
return [_strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records) for item in value]
|
|
|
|
|
return value
|