From 3799c77f06cbebaa28bf6bffaa47aefec6ae7065 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Sat, 4 Jul 2026 23:08:05 +0800 Subject: [PATCH] feat: add backend foundation and database selection --- DEVELOPMENT.md | 33 +++- README.md | 20 +- backend/dto/common.py | 83 +++++++- config.yaml | 16 ++ core/audit_service.py | 117 +++++++++++- core/identity.py | 66 ++++++- core/security.py | 33 +++- database/db_helper.py | 254 ++++++++++++++++++++++++- database/migrations/0001_init_core.sql | 17 ++ tests/test_s0_infrastructure.py | 103 ++++++++++ 10 files changed, 728 insertions(+), 14 deletions(-) create mode 100644 database/migrations/0001_init_core.sql create mode 100644 tests/test_s0_infrastructure.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f6d4953..3772e61 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -8,6 +8,7 @@ Prepared: - AstrBot plugin root files and v3 configuration placeholders. - v3 top-level module layout: `adapters`, `core`, `engines`, `backend`, `database`, `assets_builtin`, `frontend`, `pages`, `web`, and `data`. +- Database helper foundation with SQLite default and optional PostgreSQL selection from `config.yaml`. - Frontend source skeleton for Plugin Page UI and Standalone Web UI. - Runtime data directories with `.gitkeep` files only. - Generated data, local virtual environments, node modules, and build artifacts are ignored. @@ -15,9 +16,7 @@ Prepared: Not implemented yet: - AstrBot adapter registration. -- Common response DTO and error handling. -- Request context, permission checks, and audit logger. -- Database migrations, repositories, and services. +- Repositories and business services. - P0 API routes. - Real UI screens beyond scaffold placeholders. @@ -69,3 +68,31 @@ media/sign ``` Do not start a separate FastAPI, Flask, or Vite production port for the plugin. Dev servers may only be used for local frontend development before build output is copied into AstrBot static paths. + +## Database Backend + +Default behavior: + +```text +database.backend = auto +postgresql.enabled = false +selected backend = sqlite +``` + +When PostgreSQL is configured: + +```yaml +database: + backend: auto + postgresql: + enabled: true + host: 127.0.0.1 + port: 5432 + database: imagestudio + user: imagestudio + password: "" +``` + +With `backend: auto`, enabling a complete PostgreSQL config makes PostgreSQL the selected backend. Use `backend: sqlite` to force SQLite even when PostgreSQL fields exist, or `backend: postgresql` to fail fast when PG config/driver is missing. + +PostgreSQL support is optional and only imports `psycopg` when the PG backend is selected. diff --git a/README.md b/README.md index b38d7ca..da16557 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,23 @@ character_profiles ## Current Stage -This repository is aligned to `S0-1`: v3 directory and configuration baseline. +This repository has completed `S0-1` and started the S0 backend foundation. + +Implemented foundation: + +- v3 directory and configuration baseline. +- Common API response helpers matching the v3 contract. +- Request context and permission guard helpers. +- Database helper with SQLite default and optional PostgreSQL selection. +- Initial `audit_logs` migration and audit recording service. Next tasks: -1. `S0-2` common response DTO. -2. `S0-3` request id and audit logger foundation. -3. `S1-1` `character_profiles` table, repository, and service. -4. `S1-2` `profile_workflows` and `workflow_nodes`. -5. `S1-3` `generation_images`. +1. Finish route-level usage of S0 helpers. +2. `S1-1` `character_profiles` table, repository, and service. +3. `S1-2` `profile_workflows` and `workflow_nodes`. +4. `S1-3` `generation_images`. +5. `S2-1` `ui/bootstrap`. ## Repository Layout diff --git a/backend/dto/common.py b/backend/dto/common.py index cb8dcdb..0224b8e 100644 --- a/backend/dto/common.py +++ b/backend/dto/common.py @@ -1 +1,82 @@ -"""Common API DTO placeholder.""" +"""Common API response helpers for the v3 contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + + +class ErrorCode(StrEnum): + UNAUTHORIZED = "UNAUTHORIZED" + FORBIDDEN = "FORBIDDEN" + VALIDATION_FAILED = "VALIDATION_FAILED" + RESOURCE_NOT_FOUND = "RESOURCE_NOT_FOUND" + CONFIG_REVISION_CONFLICT = "CONFIG_REVISION_CONFLICT" + TASK_NOT_RETRYABLE = "TASK_NOT_RETRYABLE" + TASK_NOT_CANCELABLE = "TASK_NOT_CANCELABLE" + MODEL_CAPABILITY_UNCONFIRMED = "MODEL_CAPABILITY_UNCONFIRMED" + MODEL_CAPABILITY_UNSUPPORTED = "MODEL_CAPABILITY_UNSUPPORTED" + STORAGE_WRITE_FAILED = "STORAGE_WRITE_FAILED" + MEDIA_NOT_FOUND = "MEDIA_NOT_FOUND" + UPLOAD_TOO_LARGE = "UPLOAD_TOO_LARGE" + UNSUPPORTED_FILE_TYPE = "UNSUPPORTED_FILE_TYPE" + INTEGRATION_TOKEN_INVALID = "INTEGRATION_TOKEN_INVALID" + UNKNOWN_ERROR = "UNKNOWN_ERROR" + + +@dataclass(frozen=True) +class ApiError(Exception): + error_code: ErrorCode + message: str + details: dict[str, Any] | None = None + + def to_response(self, request_id: str | None = None) -> dict[str, Any]: + return error_response( + error_code=self.error_code, + message=self.message, + details=self.details, + request_id=request_id, + ) + + +def success_response( + data: Any, + *, + request_id: str | None = None, + warnings: list[str] | None = None, +) -> dict[str, Any]: + return { + "ok": True, + "data": data, + "warnings": warnings or [], + "request_id": request_id, + } + + +def error_response( + *, + error_code: ErrorCode | str, + message: str, + details: dict[str, Any] | None = None, + request_id: str | None = None, +) -> dict[str, Any]: + code = error_code.value if isinstance(error_code, ErrorCode) else error_code + return { + "ok": False, + "error_code": code, + "message": message, + "details": details or {}, + "request_id": request_id, + } + + +def response_from_exception(exc: Exception, request_id: str | None = None) -> dict[str, Any]: + if isinstance(exc, ApiError): + return exc.to_response(request_id=request_id) + return error_response( + error_code=ErrorCode.UNKNOWN_ERROR, + message="服务器内部错误", + details={"exception": exc.__class__.__name__}, + request_id=request_id, + ) diff --git a/config.yaml b/config.yaml index 4abc3a3..a0cdffb 100644 --- a/config.yaml +++ b/config.yaml @@ -20,6 +20,22 @@ storage: archives_path: data/archives temp_path: data/temp +database: + # auto: use SQLite by default, switch to PostgreSQL when postgresql.enabled is true + # sqlite: force local SQLite + # postgresql: force PostgreSQL and fail fast if the driver/config is missing + backend: auto + sqlite: + path: data/db/imagestudio.db + postgresql: + enabled: false + dsn: "" + host: "" + port: 5432 + database: "" + user: "" + password: "" + security: integration_token_set: false allowed_integration_plugins: diff --git a/core/audit_service.py b/core/audit_service.py index 2bf1db5..7bec0ba 100644 --- a/core/audit_service.py +++ b/core/audit_service.py @@ -1 +1,116 @@ -"""Audit logging service placeholder.""" +"""Audit logging service foundation.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +from core.identity import RequestContext +from database.db_helper import Database, DatabaseSettings + + +def new_audit_id() -> str: + return f"audit_{uuid4().hex}" + + +@dataclass(frozen=True) +class AuditRecord: + audit_id: str + request_id: str | None + actor_type: str + actor_id: str | None + action: str + target_type: str + target_id: str | None + before_snapshot: str | None + after_snapshot: str | None + metadata_json: str | None + created_at: str + + +class AuditService: + def __init__(self, database: Database | None = None): + self.database = database or Database() + + def record( + self, + *, + ctx: RequestContext, + action: str, + target_type: str, + target_id: str | None = None, + before_snapshot: Any | None = None, + after_snapshot: Any | None = None, + metadata: dict[str, Any] | None = None, + ) -> AuditRecord: + record = AuditRecord( + audit_id=new_audit_id(), + request_id=ctx.request_id, + actor_type=ctx.actor_type, + actor_id=ctx.actor_id, + action=action, + target_type=target_type, + target_id=target_id, + before_snapshot=_json_or_none(before_snapshot), + after_snapshot=_json_or_none(after_snapshot), + metadata_json=_json_or_none(metadata or {"request_id": ctx.request_id}), + created_at=datetime.now(UTC).isoformat(), + ) + self._insert(record) + return record + + def _insert(self, record: AuditRecord) -> None: + placeholder = self.database._placeholder + sql = f""" + INSERT INTO audit_logs ( + audit_id, + request_id, + actor_type, + actor_id, + action, + target_type, + target_id, + before_snapshot, + after_snapshot, + metadata_json, + created_at + ) VALUES ( + {placeholder}, {placeholder}, {placeholder}, {placeholder}, + {placeholder}, {placeholder}, {placeholder}, {placeholder}, + {placeholder}, {placeholder}, {placeholder} + ) + """ + conn = self.database.connect() + try: + conn.execute( + sql, + ( + record.audit_id, + record.request_id, + record.actor_type, + record.actor_id, + record.action, + record.target_type, + record.target_id, + record.before_snapshot, + record.after_snapshot, + record.metadata_json, + record.created_at, + ), + ) + conn.commit() + finally: + conn.close() + + +def create_audit_service(settings: DatabaseSettings | None = None) -> AuditService: + return AuditService(Database(settings)) + + +def _json_or_none(value: Any | None) -> str | None: + if value is None: + return None + return json.dumps(value, ensure_ascii=False, sort_keys=True) diff --git a/core/identity.py b/core/identity.py index c79eb67..1372e9a 100644 --- a/core/identity.py +++ b/core/identity.py @@ -1 +1,65 @@ -"""Identity and request actor helpers placeholder.""" +"""Identity and request actor helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from uuid import uuid4 + + +def new_request_id() -> str: + return f"req_{uuid4().hex}" + + +@dataclass(frozen=True) +class RequestContext: + request_id: str = field(default_factory=new_request_id) + username: str | None = None + user_id: str | None = None + client_ip: str | None = None + source: str = "system" + platform: str = "webui" + session_key: str | None = None + is_authenticated: bool = False + permissions: frozenset[str] = field(default_factory=frozenset) + created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) + + @property + def actor_id(self) -> str | None: + return self.user_id or self.username + + @property + def actor_type(self) -> str: + return "user" if self.actor_id else "system" + + def has_permission(self, permission: str) -> bool: + return permission in self.permissions or "*" in self.permissions + + @classmethod + def system(cls, source: str = "system") -> "RequestContext": + return cls( + username="system", + user_id="system", + source=source, + is_authenticated=True, + permissions=frozenset({"*"}), + ) + + @classmethod + def from_astrbot( + cls, + *, + username: str | None, + user_id: str | None = None, + client_ip: str | None = None, + source: str = "standalone", + permissions: set[str] | frozenset[str] | None = None, + ) -> "RequestContext": + return cls( + username=username, + user_id=user_id, + client_ip=client_ip, + source=source, + is_authenticated=bool(username or user_id), + permissions=frozenset(permissions or set()), + ) diff --git a/core/security.py b/core/security.py index 97e3754..5a536a5 100644 --- a/core/security.py +++ b/core/security.py @@ -1 +1,32 @@ -"""Security and permission helpers placeholder.""" +"""Security and permission helpers.""" + +from __future__ import annotations + +from backend.dto.common import ApiError, ErrorCode +from core.identity import RequestContext + + +def require_authenticated(ctx: RequestContext) -> None: + if not ctx.is_authenticated: + raise ApiError( + ErrorCode.UNAUTHORIZED, + "ImageStudio 需要 AstrBot WebUI 登录态", + ) + + +def require_permission(ctx: RequestContext, permission: str) -> None: + require_authenticated(ctx) + if not ctx.has_permission(permission): + raise ApiError( + ErrorCode.FORBIDDEN, + "当前用户没有执行该操作的权限", + {"permission": permission}, + ) + + +def require_write(ctx: RequestContext) -> None: + require_permission(ctx, "can_write") + + +def require_config_manager(ctx: RequestContext) -> None: + require_permission(ctx, "can_manage_config") diff --git a/database/db_helper.py b/database/db_helper.py index 01538de..077f305 100644 --- a/database/db_helper.py +++ b/database/db_helper.py @@ -1 +1,253 @@ -"""Database connection and migration helper placeholder.""" +"""Database connection and migration helpers. + +SQLite is the zero-config default. PostgreSQL becomes the selected backend when +configuration explicitly enables it, or when `backend: postgresql` is set. +""" + +from __future__ import annotations + +import os +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Mapping + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class SqliteSettings: + path: str = "data/db/imagestudio.db" + + +@dataclass(frozen=True) +class PostgresSettings: + enabled: bool = False + dsn: str = "" + host: str = "" + port: int = 5432 + database: str = "" + user: str = "" + password: str = "" + + @property + def is_configured(self) -> bool: + return self.enabled and bool(self.dsn or (self.host and self.database and self.user)) + + +@dataclass(frozen=True) +class DatabaseSettings: + backend: str = "auto" + sqlite: SqliteSettings = SqliteSettings() + postgresql: PostgresSettings = PostgresSettings() + + @property + def selected_backend(self) -> str: + backend = self.backend.lower() + if backend in {"postgres", "postgresql", "pg"}: + return "postgresql" + if backend == "sqlite": + return "sqlite" + if self.postgresql.is_configured: + return "postgresql" + return "sqlite" + + @classmethod + def from_mapping(cls, payload: Mapping[str, Any]) -> "DatabaseSettings": + database = payload.get("database", payload) + if not isinstance(database, Mapping): + database = {} + + sqlite_payload = database.get("sqlite", {}) + postgres_payload = database.get("postgresql", {}) + if not isinstance(sqlite_payload, Mapping): + sqlite_payload = {} + if not isinstance(postgres_payload, Mapping): + postgres_payload = {} + + return cls( + backend=str(database.get("backend", "auto")), + sqlite=SqliteSettings(path=str(sqlite_payload.get("path", "data/db/imagestudio.db"))), + postgresql=PostgresSettings( + enabled=_as_bool(postgres_payload.get("enabled", False)), + dsn=str(postgres_payload.get("dsn", "") or ""), + host=str(postgres_payload.get("host", "") or ""), + port=int(postgres_payload.get("port", 5432) or 5432), + database=str(postgres_payload.get("database", "") or ""), + user=str(postgres_payload.get("user", "") or ""), + password=str(postgres_payload.get("password", "") or ""), + ), + ) + + +class Database: + def __init__(self, settings: DatabaseSettings | None = None): + self.settings = settings or DatabaseSettings() + + @property + def backend(self) -> str: + return self.settings.selected_backend + + def connect(self): + if self.backend == "sqlite": + db_path = self._resolve_sqlite_path() + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + return conn + return self._connect_postgresql() + + def initialize(self, migrations_dir: str | Path | None = None) -> None: + conn = self.connect() + try: + self._ensure_schema_migrations(conn) + for migration_path in self._iter_migrations(migrations_dir): + version = migration_path.stem + if self._migration_applied(conn, version): + continue + self._execute_script(conn, migration_path.read_text(encoding="utf-8")) + self._mark_migration_applied(conn, version) + if self.backend == "postgresql": + conn.commit() + else: + conn.commit() + finally: + conn.close() + + def _resolve_sqlite_path(self) -> Path: + db_path = Path(self.settings.sqlite.path) + if not db_path.is_absolute(): + db_path = REPO_ROOT / db_path + return db_path + + def _connect_postgresql(self): + try: + import psycopg + except ImportError as exc: + raise RuntimeError( + "PostgreSQL backend requires the optional 'psycopg' package." + ) from exc + + pg = self.settings.postgresql + if pg.dsn: + return psycopg.connect(pg.dsn) + return psycopg.connect( + host=pg.host, + port=pg.port, + dbname=pg.database, + user=pg.user, + password=pg.password, + ) + + def _ensure_schema_migrations(self, conn) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + + def _migration_applied(self, conn, version: str) -> bool: + placeholder = self._placeholder + row = conn.execute( + f"SELECT version FROM schema_migrations WHERE version = {placeholder}", + (version,), + ).fetchone() + return row is not None + + def _mark_migration_applied(self, conn, version: str) -> None: + placeholder = self._placeholder + conn.execute( + f"INSERT INTO schema_migrations(version) VALUES ({placeholder})", + (version,), + ) + + def _execute_script(self, conn, script: str) -> None: + if self.backend == "sqlite": + conn.executescript(script) + return + statements = [statement.strip() for statement in script.split(";") if statement.strip()] + for statement in statements: + conn.execute(statement) + + @property + def _placeholder(self) -> str: + return "%s" if self.backend == "postgresql" else "?" + + def _iter_migrations(self, migrations_dir: str | Path | None) -> Iterable[Path]: + directory = Path(migrations_dir) if migrations_dir else REPO_ROOT / "database" / "migrations" + return sorted(path for path in directory.glob("*.sql") if path.is_file()) + + +def load_database_settings(config_path: str | Path = "config.yaml") -> DatabaseSettings: + path = Path(config_path) + if not path.is_absolute(): + path = REPO_ROOT / path + if not path.exists(): + return DatabaseSettings() + payload = _load_yaml_subset(path.read_text(encoding="utf-8")) + return DatabaseSettings.from_mapping(payload) + + +def _as_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _load_yaml_subset(text: str) -> dict[str, Any]: + """Parse the simple YAML shape used by config.yaml without extra deps.""" + root: dict[str, Any] = {} + stack: list[tuple[int, dict[str, Any]]] = [(-1, root)] + last_key_by_indent: dict[int, str] = {} + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = len(line) - len(line.lstrip(" ")) + stripped = line.strip() + if stripped.startswith("- "): + parent = stack[-1][1] + list_key = last_key_by_indent.get(indent - 2) + if list_key: + parent.setdefault(list_key, []).append(_parse_scalar(stripped[2:])) + continue + if ":" not in stripped: + continue + key, value = stripped.split(":", 1) + key = key.strip() + value = value.strip() + while stack and indent <= stack[-1][0]: + stack.pop() + parent = stack[-1][1] + if value == "": + child: dict[str, Any] = {} + parent[key] = child + stack.append((indent, child)) + last_key_by_indent[indent] = key + else: + parent[key] = _parse_scalar(value) + last_key_by_indent[indent] = key + return root + + +def _parse_scalar(value: str) -> Any: + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + if lowered in {"null", "none"}: + return None + try: + return int(value) + except ValueError: + return os.path.expandvars(value) diff --git a/database/migrations/0001_init_core.sql b/database/migrations/0001_init_core.sql new file mode 100644 index 0000000..0ca7d28 --- /dev/null +++ b/database/migrations/0001_init_core.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS audit_logs ( + audit_id TEXT PRIMARY KEY, + request_id TEXT, + actor_type TEXT NOT NULL, + actor_id TEXT, + action TEXT NOT NULL, + target_type TEXT NOT NULL, + target_id TEXT, + before_snapshot TEXT, + after_snapshot TEXT, + metadata_json TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_audit_logs_request ON audit_logs(request_id); +CREATE INDEX IF NOT EXISTS idx_audit_logs_action ON audit_logs(action, created_at); +CREATE INDEX IF NOT EXISTS idx_audit_logs_target ON audit_logs(target_type, target_id); diff --git a/tests/test_s0_infrastructure.py b/tests/test_s0_infrastructure.py new file mode 100644 index 0000000..3fdb49f --- /dev/null +++ b/tests/test_s0_infrastructure.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from backend.dto.common import ErrorCode, error_response, success_response +from core.audit_service import AuditService +from core.identity import RequestContext +from database.db_helper import Database, DatabaseSettings, PostgresSettings, SqliteSettings + + +class CommonDtoTests(unittest.TestCase): + def test_success_response_matches_contract(self): + response = success_response({"ready": True}, request_id="req_test") + + self.assertEqual( + response, + { + "ok": True, + "data": {"ready": True}, + "warnings": [], + "request_id": "req_test", + }, + ) + + def test_error_response_matches_contract(self): + response = error_response( + error_code=ErrorCode.VALIDATION_FAILED, + message="缺少字段", + details={"field": "name"}, + request_id="req_test", + ) + + self.assertEqual(response["ok"], False) + self.assertEqual(response["error_code"], "VALIDATION_FAILED") + self.assertEqual(response["message"], "缺少字段") + self.assertEqual(response["details"], {"field": "name"}) + self.assertEqual(response["request_id"], "req_test") + + +class DatabaseSettingsTests(unittest.TestCase): + def test_sqlite_is_default_backend(self): + settings = DatabaseSettings() + + self.assertEqual(settings.selected_backend, "sqlite") + + def test_postgres_is_selected_when_configured(self): + settings = DatabaseSettings( + backend="auto", + postgresql=PostgresSettings( + enabled=True, + host="localhost", + database="imagestudio", + user="imagestudio", + ), + ) + + self.assertEqual(settings.selected_backend, "postgresql") + + def test_forced_sqlite_ignores_configured_postgres(self): + settings = DatabaseSettings( + backend="sqlite", + postgresql=PostgresSettings(enabled=True, dsn="postgresql://example"), + ) + + self.assertEqual(settings.selected_backend, "sqlite") + + +class AuditServiceTests(unittest.TestCase): + def test_record_writes_audit_log_to_sqlite(self): + with tempfile.TemporaryDirectory() as tmp: + db_path = Path(tmp) / "imagestudio.db" + settings = DatabaseSettings(sqlite=SqliteSettings(path=str(db_path))) + database = Database(settings) + database.initialize() + ctx = RequestContext.system() + + record = AuditService(database).record( + ctx=ctx, + action="unit_test", + target_type="test_target", + target_id="target_001", + after_snapshot={"ok": True}, + ) + + conn = database.connect() + try: + row = conn.execute( + "SELECT action, target_type, target_id FROM audit_logs WHERE audit_id = ?", + (record.audit_id,), + ).fetchone() + finally: + conn.close() + + self.assertIsNotNone(row) + self.assertEqual(row["action"], "unit_test") + self.assertEqual(row["target_type"], "test_target") + self.assertEqual(row["target_id"], "target_001") + + +if __name__ == "__main__": + unittest.main()