S98 孤儿媒体扫描
这个提交存在于:
父节点
35b4ab5425
当前提交
297f3beaef
共有 5 个文件被更改,包括 143 次插入 和 0 次删除
|
|
@ -93,6 +93,7 @@ ROUTES: tuple[RouteSpec, ...] = (
|
||||||
RouteSpec("POST", "storage/migrate", "backend.routes.storage_routes.migrate_storage", "config"),
|
RouteSpec("POST", "storage/migrate", "backend.routes.storage_routes.migrate_storage", "config"),
|
||||||
RouteSpec("GET", "media/sign", "backend.routes.media_routes.sign_media", "read"),
|
RouteSpec("GET", "media/sign", "backend.routes.media_routes.sign_media", "read"),
|
||||||
RouteSpec("GET", "media/index", "backend.routes.media_routes.index_media", "read"),
|
RouteSpec("GET", "media/index", "backend.routes.media_routes.index_media", "read"),
|
||||||
|
RouteSpec("GET", "media/orphans", "backend.routes.media_routes.scan_orphan_media", "read"),
|
||||||
RouteSpec("GET", "media/fetch", "backend.routes.media_routes.fetch_media", "public_token"),
|
RouteSpec("GET", "media/fetch", "backend.routes.media_routes.fetch_media", "public_token"),
|
||||||
RouteSpec("GET", "material-categories", "backend.routes.material_routes.list_material_categories", "read"),
|
RouteSpec("GET", "material-categories", "backend.routes.material_routes.list_material_categories", "read"),
|
||||||
RouteSpec("POST", "material-categories", "backend.routes.material_routes.create_material_category", "materials"),
|
RouteSpec("POST", "material-categories", "backend.routes.material_routes.create_material_category", "materials"),
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,10 @@ from backend.dto.common import ApiError, ErrorCode
|
||||||
from backend.routes.common import handle_request
|
from backend.routes.common import handle_request
|
||||||
from backend.services.media_index_service import MediaIndexService
|
from backend.services.media_index_service import MediaIndexService
|
||||||
from backend.services.media_service import MediaService
|
from backend.services.media_service import MediaService
|
||||||
|
from backend.services.orphan_media_service import OrphanMediaService
|
||||||
from core.identity import RequestContext
|
from core.identity import RequestContext
|
||||||
from core.security import require_authenticated
|
from core.security import require_authenticated
|
||||||
|
from database.db_helper import Database, load_database_settings
|
||||||
|
|
||||||
|
|
||||||
def sign_media(
|
def sign_media(
|
||||||
|
|
@ -63,6 +65,27 @@ def index_media(
|
||||||
return handle_request(ctx, action)
|
return handle_request(ctx, action)
|
||||||
|
|
||||||
|
|
||||||
|
def scan_orphan_media(
|
||||||
|
ctx: RequestContext,
|
||||||
|
query: dict[str, Any] | None = None,
|
||||||
|
service: OrphanMediaService | None = None,
|
||||||
|
) -> dict:
|
||||||
|
params = query or {}
|
||||||
|
orphan_service = service or OrphanMediaService(
|
||||||
|
Database(load_database_settings()),
|
||||||
|
MediaIndexService(MediaService()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def action() -> dict[str, Any]:
|
||||||
|
require_authenticated(ctx)
|
||||||
|
return orphan_service.scan(
|
||||||
|
root=_optional_str(params.get("root")),
|
||||||
|
limit=_int_or_default(params.get("limit"), 500),
|
||||||
|
)
|
||||||
|
|
||||||
|
return handle_request(ctx, action)
|
||||||
|
|
||||||
|
|
||||||
def _optional_str(value: Any) -> str | None:
|
def _optional_str(value: Any) -> str | None:
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
"""Find storage files that are not referenced by database records."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from backend.services.media_index_service import MediaIndexService
|
||||||
|
from database.db_helper import Database
|
||||||
|
|
||||||
|
|
||||||
|
class OrphanMediaService:
|
||||||
|
def __init__(self, database: Database, media_index_service: MediaIndexService):
|
||||||
|
self.database = database
|
||||||
|
self.media_index_service = media_index_service
|
||||||
|
|
||||||
|
def scan(self, *, root: str | None = None, limit: int = 500) -> dict[str, Any]:
|
||||||
|
references = self._referenced_uris()
|
||||||
|
index = self.media_index_service.index_files(root=root, limit=limit)
|
||||||
|
orphans = [item for item in index["items"] if item["uri"] not in references]
|
||||||
|
return {
|
||||||
|
"root": root,
|
||||||
|
"total_indexed": len(index["items"]),
|
||||||
|
"total_files": index["total"],
|
||||||
|
"index_truncated": index["truncated"],
|
||||||
|
"referenced_uri_count": len(references),
|
||||||
|
"orphan_count": len(orphans),
|
||||||
|
"orphans": orphans,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _referenced_uris(self) -> set[str]:
|
||||||
|
refs: set[str] = set()
|
||||||
|
conn = self.database.connect()
|
||||||
|
try:
|
||||||
|
for table, columns in (
|
||||||
|
("generation_images", ("image_uri", "thumbnail_uri")),
|
||||||
|
("materials", ("image_uri", "thumbnail_uri")),
|
||||||
|
("character_profiles", ("avatar_image_uri",)),
|
||||||
|
):
|
||||||
|
for row in conn.execute(f"SELECT {', '.join(columns)} FROM {table}").fetchall():
|
||||||
|
for column in columns:
|
||||||
|
uri = str(row[column] or "").strip()
|
||||||
|
if uri.startswith("storage://"):
|
||||||
|
refs.add(uri)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return refs
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from backend.routes.media_routes import scan_orphan_media
|
||||||
|
from backend.services.media_index_service import MediaIndexService
|
||||||
|
from backend.services.media_service import MediaService
|
||||||
|
from backend.services.orphan_media_service import OrphanMediaService
|
||||||
|
from core.config_service import ConfigService
|
||||||
|
from core.identity import RequestContext
|
||||||
|
from database.db_helper import Database, DatabaseSettings, SqliteSettings
|
||||||
|
from database.repositories.image_repository import ImageRepository
|
||||||
|
|
||||||
|
|
||||||
|
class OrphanMediaServiceTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
self.config_path = self.root / "config.yaml"
|
||||||
|
self.config_path.write_text(
|
||||||
|
f"""
|
||||||
|
storage:
|
||||||
|
images_path: "{(self.root / "images").as_posix()}"
|
||||||
|
thumbnails_path: "{(self.root / "thumbnails").as_posix()}"
|
||||||
|
materials_path: "{(self.root / "materials").as_posix()}"
|
||||||
|
styles_path: "{(self.root / "styles").as_posix()}"
|
||||||
|
archives_path: "{(self.root / "archives").as_posix()}"
|
||||||
|
temp_path: "{(self.root / "temp").as_posix()}"
|
||||||
|
""",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
for folder in ("images", "thumbnails", "materials", "styles", "archives", "temp"):
|
||||||
|
(self.root / folder).mkdir(parents=True, exist_ok=True)
|
||||||
|
(self.root / "images" / "used.png").write_bytes(b"used")
|
||||||
|
(self.root / "images" / "orphan.png").write_bytes(b"orphan")
|
||||||
|
self.database = Database(DatabaseSettings(sqlite=SqliteSettings(path=str(self.root / "imagestudio.db"))))
|
||||||
|
self.database.initialize()
|
||||||
|
ImageRepository(self.database).create(
|
||||||
|
{
|
||||||
|
"image_id": "img_used",
|
||||||
|
"image_uri": "storage://images/used.png",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
media_service = MediaService(ConfigService(self.config_path))
|
||||||
|
self.service = OrphanMediaService(self.database, MediaIndexService(media_service))
|
||||||
|
self.ctx = RequestContext.from_astrbot(username="admin", user_id="user_001")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_scan_reports_unreferenced_storage_files_without_local_paths(self):
|
||||||
|
result = self.service.scan(root="images")
|
||||||
|
|
||||||
|
self.assertEqual(result["total_files"], 2)
|
||||||
|
self.assertEqual(result["orphan_count"], 1)
|
||||||
|
self.assertEqual(result["orphans"][0]["uri"], "storage://images/orphan.png")
|
||||||
|
self.assertNotIn(str(self.root), str(result))
|
||||||
|
|
||||||
|
def test_scan_orphan_media_route_requires_authentication(self):
|
||||||
|
response = scan_orphan_media(self.ctx, {"root": "images"}, self.service)
|
||||||
|
anonymous = scan_orphan_media(RequestContext(), {"root": "images"}, self.service)
|
||||||
|
|
||||||
|
self.assertTrue(response["ok"])
|
||||||
|
self.assertEqual(response["data"]["orphan_count"], 1)
|
||||||
|
self.assertFalse(anonymous["ok"])
|
||||||
|
self.assertEqual(anonymous["error_code"], "UNAUTHORIZED")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -58,6 +58,7 @@ class RouteRegistryTests(unittest.TestCase):
|
||||||
self.assertIn(("POST", "settings/rollback"), keys)
|
self.assertIn(("POST", "settings/rollback"), keys)
|
||||||
self.assertIn(("POST", "storage/migrate"), keys)
|
self.assertIn(("POST", "storage/migrate"), keys)
|
||||||
self.assertIn(("GET", "media/index"), keys)
|
self.assertIn(("GET", "media/index"), keys)
|
||||||
|
self.assertIn(("GET", "media/orphans"), keys)
|
||||||
self.assertIn(("GET", "events/stream"), keys)
|
self.assertIn(("GET", "events/stream"), keys)
|
||||||
self.assertIn(("GET", "costs/usage"), keys)
|
self.assertIn(("GET", "costs/usage"), keys)
|
||||||
self.assertIn(("POST", "costs/guard"), keys)
|
self.assertIn(("POST", "costs/guard"), keys)
|
||||||
|
|
|
||||||
正在加载…
添加表格
在新工单中引用