"""Server-side ImageStudio data backup and guarded restore.""" from __future__ import annotations import hashlib import json import shutil import sqlite3 import tempfile import zipfile from datetime import datetime, timezone from pathlib import Path from typing import Any from uuid import uuid4 from backend.dto.common import ApiError, ErrorCode from core.config_service import ConfigService from core.runtime_paths import resolve_data_path from database.db_helper import load_config_payload, load_database_settings MAX_BACKUP_FILES = 50_000 MAX_RESTORE_BYTES = 10 * 1024 * 1024 * 1024 PROMPT_BACKUP_PLACEHOLDER = "[EXCLUDED_FROM_BACKUP]" class DataBackupService: def __init__(self, config_service: ConfigService | None = None): self.config_service = config_service or ConfigService() def list_backups(self) -> list[dict[str, Any]]: root = self._backup_root() if not root.exists(): return [] return [self._summary(path) for path in sorted(root.glob("*.zip"), key=lambda item: item.stat().st_mtime, reverse=True)] def create_backup( self, *, include_media: bool = True, include_prompt_records: bool = True, include_moderation_records: bool = True, ) -> dict[str, Any]: backup_id = f"backup_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}_{uuid4().hex[:8]}" root = self._backup_root() root.mkdir(parents=True, exist_ok=True) destination = root / f"{backup_id}.zip" config = load_config_payload(self.config_service.config_path) entries: list[dict[str, Any]] = [] with tempfile.TemporaryDirectory(prefix="imagestudio-backup-") as temp_dir: temp = Path(temp_dir) config_copy = temp / "config.yaml" if self.config_service.config_path.exists(): shutil.copy2(self.config_service.config_path, config_copy) entries.append(_entry(config_copy, "config/config.yaml")) database_settings = load_database_settings(self.config_service.config_path) warnings: list[str] = [] if database_settings.selected_backend == "sqlite": source_db = resolve_data_path(database_settings.sqlite.path) if source_db.exists(): db_copy = temp / "imagestudio.db" source = sqlite3.connect(source_db) target = sqlite3.connect(db_copy) try: source.backup(target) finally: target.close(); source.close() if not include_prompt_records or not include_moderation_records: _sanitize_sqlite_backup( db_copy, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records, ) entries.append(_entry(db_copy, "database/imagestudio.db")) else: warnings.append("PostgreSQL data is not embedded; use pg_dump with the connection recorded in the redacted config.") storage = config.get("storage") if isinstance(config.get("storage"), dict) else {} media_roots = { key: resolve_data_path(storage.get(f"{key}_path") or f"data/{key}") for key in ("images", "thumbnails", "materials", "advanced_materials", "styles") } if include_media: for key, media_root in media_roots.items(): if not media_root.exists(): continue for path in media_root.rglob("*"): if not path.is_file(): continue if len(entries) >= MAX_BACKUP_FILES: raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {"limit": MAX_BACKUP_FILES}) entries.append(_entry(path, f"media/{key}/{path.relative_to(media_root).as_posix()}")) manifest = { "format": "imagestudio-full-backup", "version": 3, "material_schema_version": "1.0", "backup_id": backup_id, "created_at": datetime.now(timezone.utc).isoformat(), "database_backend": database_settings.selected_backend, "include_media": include_media, "include_prompt_records": include_prompt_records, "include_moderation_records": include_moderation_records, "entries": entries, "warnings": warnings, } with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive: for item in entries: source_path = config_copy if item["path"] == "config/config.yaml" else db_copy if item["path"] == "database/imagestudio.db" else _source_for_media_entry(item["path"], media_roots) archive.write(source_path, item["path"]) archive.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2)) return self._summary(destination, manifest=manifest) def inspect_backup(self, backup_id: str) -> dict[str, Any]: path = self._backup_path(backup_id) with zipfile.ZipFile(path) as archive: manifest = _read_manifest(archive) _validate_archive(archive, manifest) return {**self._summary(path, manifest=manifest), "manifest": manifest, "valid": True} def restore_backup(self, backup_id: str, *, confirm: bool = False) -> dict[str, Any]: if not confirm: return {**self.inspect_backup(backup_id), "restored": False, "requires_confirmation": True} path = self._backup_path(backup_id) with zipfile.ZipFile(path) as archive: manifest = _read_manifest(archive) _validate_archive(archive, manifest) safety_backup = self.create_backup(include_media=False, include_prompt_records=True, include_moderation_records=True) with tempfile.TemporaryDirectory(prefix="imagestudio-restore-") as temp_dir: temp = Path(temp_dir) archive.extractall(temp) config_source = temp / "config" / "config.yaml" if config_source.exists(): self.config_service.config_path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(config_source, self.config_service.config_path) database_settings = load_database_settings(self.config_service.config_path) db_source = temp / "database" / "imagestudio.db" if db_source.exists() and database_settings.selected_backend == "sqlite": db_target = resolve_data_path(database_settings.sqlite.path) db_target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(db_source, db_target) config = load_config_payload(self.config_service.config_path) storage = config.get("storage") if isinstance(config.get("storage"), dict) else {} for key in ("images", "thumbnails", "materials", "advanced_materials", "styles"): source_root = temp / "media" / key if not source_root.exists(): continue target_root = resolve_data_path(storage.get(f"{key}_path") or f"data/{key}") target_root.mkdir(parents=True, exist_ok=True) shutil.copytree(source_root, target_root, dirs_exist_ok=True) return {"restored": True, "backup_id": backup_id, "safety_backup_id": safety_backup["backup_id"]} def _backup_root(self) -> Path: config = self.config_service.load() return resolve_data_path(config.storage.archives_path) / "backups" def _backup_path(self, backup_id: str) -> Path: normalized = str(backup_id or "").strip() if not normalized or any(char not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for char in normalized): raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup_id", {"backup_id": backup_id}) path = self._backup_root() / f"{normalized}.zip" if not path.exists(): raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "backup not found", {"backup_id": normalized}) return path def _summary(self, path: Path, *, manifest: dict[str, Any] | None = None) -> dict[str, Any]: loaded = manifest if loaded is None: try: with zipfile.ZipFile(path) as archive: loaded = _read_manifest(archive) except Exception: loaded = {} stat = path.stat() return {"backup_id": loaded.get("backup_id") or path.stem, "created_at": loaded.get("created_at") or datetime.fromtimestamp(stat.st_mtime,timezone.utc).isoformat(), "size_bytes": stat.st_size, "file_count": len(loaded.get("entries") or []), "database_backend": loaded.get("database_backend"), "include_media": bool(loaded.get("include_media")), "include_prompt_records": loaded.get("include_prompt_records", True), "include_moderation_records": loaded.get("include_moderation_records", True), "material_schema_version": loaded.get("material_schema_version"), "warnings": loaded.get("warnings") or []} def _entry(path: Path, archive_path: str) -> dict[str, Any]: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return {"path": archive_path, "size": path.stat().st_size, "sha256": digest.hexdigest()} def _source_for_media_entry(archive_path: str, roots: dict[str, Path]) -> Path: _, key, relative = archive_path.split("/", 2) return roots[key] / Path(relative) def _read_manifest(archive: zipfile.ZipFile) -> dict[str, Any]: try: value = json.loads(archive.read("manifest.json")) except (KeyError, json.JSONDecodeError) as exc: raise ApiError(ErrorCode.VALIDATION_FAILED, "invalid backup manifest", {}) from exc if not isinstance(value, dict) or value.get("format") != "imagestudio-full-backup": raise ApiError(ErrorCode.VALIDATION_FAILED, "unsupported backup format", {}) return value def _validate_archive(archive: zipfile.ZipFile, manifest: dict[str, Any]) -> None: entries = manifest.get("entries") if isinstance(manifest.get("entries"), list) else [] if len(entries) > MAX_BACKUP_FILES: raise ApiError(ErrorCode.VALIDATION_FAILED, "backup file count exceeds safety limit", {}) names = set(archive.namelist()) declared_names = {str(item.get("path") or "") for item in entries if isinstance(item, dict)} if names - declared_names != {"manifest.json"}: raise ApiError(ErrorCode.VALIDATION_FAILED, "backup contains undeclared entries", {}) total = 0 for item in entries: name = str(item.get("path") or "") if isinstance(item, dict) else "" parts = Path(name).parts if not name or name not in names or Path(name).is_absolute() or ".." in parts: raise ApiError(ErrorCode.VALIDATION_FAILED, "unsafe backup entry", {"path": name}) info = archive.getinfo(name) total += info.file_size if total > MAX_RESTORE_BYTES: raise ApiError(ErrorCode.VALIDATION_FAILED, "backup exceeds restore size limit", {}) digest = hashlib.sha256(archive.read(name)).hexdigest() if digest != item.get("sha256"): raise ApiError(ErrorCode.VALIDATION_FAILED, "backup checksum mismatch", {"path": name}) def _sanitize_sqlite_backup( path: Path, *, include_prompt_records: bool, include_moderation_records: bool, ) -> None: conn = sqlite3.connect(path) conn.row_factory = sqlite3.Row try: tables = {str(row["name"]) for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} if not include_prompt_records: if "materials" in tables: conn.execute("UPDATE materials SET prompt = NULL, negative_prompt = NULL") if "advanced_material_imports" in tables: conn.execute( "UPDATE advanced_material_imports " "SET recognition_prompt = NULL, recognition_negative_prompt = NULL" ) if "advanced_material_validation_attempts" in tables: conn.execute( "UPDATE advanced_material_validation_attempts " "SET prompt = ?, negative_prompt = NULL, adjustment_text = NULL, decision_notes = NULL", (PROMPT_BACKUP_PLACEHOLDER,), ) if "advanced_materials" in tables: conn.execute( "UPDATE advanced_materials SET prompt = ?, negative_prompt = NULL", (PROMPT_BACKUP_PLACEHOLDER,), ) if "advanced_style_sources" in tables: conn.execute( "UPDATE advanced_style_sources " "SET recognition_prompt = NULL, recognition_negative_prompt = NULL" ) if "advanced_style_consensus_runs" in tables: conn.execute( "UPDATE advanced_style_consensus_runs SET prompt = NULL, negative_prompt = NULL" ) if "advanced_style_validation_cases" in tables: conn.execute( "UPDATE advanced_style_validation_cases " "SET content_prompt = ?, final_prompt = ?, negative_prompt = NULL", (PROMPT_BACKUP_PLACEHOLDER, PROMPT_BACKUP_PLACEHOLDER), ) if "generation_images" in tables: conn.execute("UPDATE generation_images SET positive_prompt = NULL, negative_prompt = NULL") json_columns = { "materials": ("material_id", ("metadata_json",)), "advanced_material_imports": ("import_id", ("recognition_result_json", "error_json", "metadata_json")), "advanced_material_validation_attempts": ("attempt_id", ("usage_json", "error_json", "metadata_json")), "advanced_materials": ("material_id", ("capsule_json", "metadata_json")), "advanced_style_sources": ( "source_id", ("recognition_tags_json", "recognition_result_json", "error_json", "metadata_json"), ), "advanced_style_consensus_runs": ( "consensus_id", ("source_ids_json", "tags_json", "result_json", "error_json", "metadata_json"), ), "advanced_style_validation_cases": ( "case_id", ("usage_json", "style_review_json", "error_json", "metadata_json"), ), "generation_images": ("image_id", ("metadata_json",)), "generation_tasks": ("task_id", ("request_json", "result_json", "metadata_json")), "audit_logs": ("audit_id", ("before_snapshot", "after_snapshot", "metadata_json")), } for table, (primary_key, columns) in json_columns.items(): if table not in tables: continue existing_columns = {str(row["name"]) for row in conn.execute(f"PRAGMA table_info({table})")} selected_columns = [column for column in columns if column in existing_columns] if not selected_columns: continue for row in conn.execute(f"SELECT {primary_key}, {', '.join(selected_columns)} FROM {table}").fetchall(): updates: dict[str, str | None] = {} for column in selected_columns: raw = row[column] if not raw: continue try: parsed = json.loads(raw) except (TypeError, json.JSONDecodeError): continue cleaned = _strip_sensitive_records( parsed, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records, ) updates[column] = json.dumps(cleaned, ensure_ascii=False, separators=(",", ":")) if updates: assignments = ", ".join(f"{column} = ?" for column in updates) conn.execute(f"UPDATE {table} SET {assignments} WHERE {primary_key} = ?", (*updates.values(), row[primary_key])) conn.commit() finally: conn.close() def _strip_sensitive_records( value: Any, *, include_prompt_records: bool, include_moderation_records: bool, ) -> Any: prompt_keys = {"prompt", "positive_prompt", "negative_prompt", "expanded_prompt", "expanded_negative_prompt", "original_text", "translated_text", "normalized_text", "source_payload"} moderation_keys = {"prompt_governance", "moderation", "moderation_recheck", "image_moderation", "route_fallback", "generation_route_decision"} if isinstance(value, dict): result: dict[str, Any] = {} for key, item in value.items(): lowered = str(key).lower() if not include_prompt_records and lowered in prompt_keys: result[key] = PROMPT_BACKUP_PLACEHOLDER continue if not include_moderation_records and lowered in moderation_keys: continue result[key] = _strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records) return result if isinstance(value, list): return [_strip_sensitive_records(item, include_prompt_records=include_prompt_records, include_moderation_records=include_moderation_records) for item in value] return value