From d3393ebd095e738c1234f951da90adde764f636b Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Sun, 5 Jul 2026 15:28:28 +0800 Subject: [PATCH] feat: expand log filters and snapshot detail --- backend/route_registry.py | 1 + backend/routes/integration_routes.py | 4 +- backend/routes/log_routes.py | 39 ++++- backend/routes/settings_routes.py | 14 ++ backend/services/integration_service.py | 47 +++++- backend/services/log_service.py | 141 ++++++++++++++---- backend/services/settings_service.py | 49 ++++-- .../repositories/integration_repository.py | 22 ++- tests/test_integration_service.py | 18 +++ tests/test_log_routes.py | 26 ++++ tests/test_route_registry.py | 1 + tests/test_s2_routes.py | 6 + 12 files changed, 313 insertions(+), 55 deletions(-) diff --git a/backend/route_registry.py b/backend/route_registry.py index 8f2623e..fd6f69d 100644 --- a/backend/route_registry.py +++ b/backend/route_registry.py @@ -74,6 +74,7 @@ ROUTES: tuple[RouteSpec, ...] = ( RouteSpec("GET", "settings/export", "backend.routes.settings_routes.export_settings", "config"), RouteSpec("POST", "settings/import", "backend.routes.settings_routes.import_settings", "config"), RouteSpec("GET", "settings/snapshots", "backend.routes.settings_routes.list_settings_snapshots", "config"), + RouteSpec("GET", "settings/snapshots/{snapshot_id}", "backend.routes.settings_routes.get_settings_snapshot", "config"), RouteSpec("POST", "settings/rollback", "backend.routes.settings_routes.rollback_settings", "config"), RouteSpec("GET", "storage/status", "backend.routes.storage_routes.get_storage_status", "read"), RouteSpec("GET", "media/sign", "backend.routes.media_routes.sign_media", "read"), diff --git a/backend/routes/integration_routes.py b/backend/routes/integration_routes.py index 3809070..c1b7284 100644 --- a/backend/routes/integration_routes.py +++ b/backend/routes/integration_routes.py @@ -42,7 +42,9 @@ def list_integration_logs( def action() -> list[dict[str, Any]]: require_authenticated(ctx) integration_name = str(params.get("integration_name") or "").strip() or None - return integration_service.list_integration_logs(integration_name) + action_name = str(params.get("action") or "").strip() or None + status = str(params.get("status") or "").strip() or None + return integration_service.list_integration_logs(integration_name, action=action_name, status=status) return handle_request(ctx, action) diff --git a/backend/routes/log_routes.py b/backend/routes/log_routes.py index 3f6d763..77ae3a8 100644 --- a/backend/routes/log_routes.py +++ b/backend/routes/log_routes.py @@ -12,27 +12,58 @@ from database.db_helper import Database, load_database_settings def get_generation_logs(ctx: RequestContext, query: dict[str, Any] | None = None, service: LogService | None = None) -> dict: - return _read_logs(ctx, query, service, lambda log_service, limit: log_service.generation_logs(limit)) + params = query or {} + + def action(log_service: LogService, limit: int) -> list[dict[str, Any]]: + return log_service.generation_logs( + limit, + status=_optional_str(params.get("status")), + workflow_id=_optional_str(params.get("workflow_id")), + profile_id=_optional_str(params.get("profile_id")), + node_id=_optional_str(params.get("node_id")), + ) + + return _read_logs(ctx, query, service, action) def get_workflow_logs(ctx: RequestContext, query: dict[str, Any] | None = None, service: LogService | None = None) -> dict: params = query or {} def action(log_service: LogService, limit: int) -> list[dict[str, Any]]: - return log_service.workflow_logs(_optional_str(params.get("workflow_id")), limit) + return log_service.workflow_logs( + _optional_str(params.get("workflow_id")), + limit, + action=_optional_str(params.get("action")), + node_id=_optional_str(params.get("node_id")), + branch_name=_optional_str(params.get("branch_name")), + ) return _read_logs(ctx, query, service, action) def get_error_logs(ctx: RequestContext, query: dict[str, Any] | None = None, service: LogService | None = None) -> dict: - return _read_logs(ctx, query, service, lambda log_service, limit: log_service.error_logs(limit)) + params = query or {} + + def action(log_service: LogService, limit: int) -> list[dict[str, Any]]: + return log_service.error_logs( + limit, + source=_optional_str(params.get("source")), + status=_optional_str(params.get("status")), + ) + + return _read_logs(ctx, query, service, action) def get_integration_logs(ctx: RequestContext, query: dict[str, Any] | None = None, service: LogService | None = None) -> dict: params = query or {} def action(log_service: LogService, limit: int) -> list[dict[str, Any]]: - return log_service.integration_logs(_optional_str(params.get("integration_name")), limit) + return log_service.integration_logs( + _optional_str(params.get("integration_name")), + limit, + action=_optional_str(params.get("action")), + status=_optional_str(params.get("status")), + ) return _read_logs(ctx, query, service, action) diff --git a/backend/routes/settings_routes.py b/backend/routes/settings_routes.py index e3180b9..e7e3a3d 100644 --- a/backend/routes/settings_routes.py +++ b/backend/routes/settings_routes.py @@ -135,6 +135,20 @@ def list_settings_snapshots( return handle_request(ctx, action) +def get_settings_snapshot( + ctx: RequestContext, + snapshot_id: str, + service: SettingsService | None = None, +) -> dict: + settings_service = service or _create_settings_service() + + def action() -> dict[str, Any]: + require_config_manager(ctx) + return settings_service.get_snapshot(snapshot_id) + + return handle_request(ctx, action) + + def rollback_settings( ctx: RequestContext, payload: dict[str, Any], diff --git a/backend/services/integration_service.py b/backend/services/integration_service.py index 2ce7ae1..ab7af32 100644 --- a/backend/services/integration_service.py +++ b/backend/services/integration_service.py @@ -26,16 +26,24 @@ class IntegrationService: def daily_outfit(self, payload: dict[str, Any], ctx: RequestContext) -> dict[str, Any]: source = str(payload.get("request_source") or PRIVATE_COMPANION) if source != PRIVATE_COMPANION: - raise ApiError( + error = ApiError( ErrorCode.INTEGRATION_TOKEN_INVALID, "unsupported integration source", {"request_source": source}, ) + self._record_failure(source, payload, error) + raise error profile_id = str(payload.get("profile_id") or "").strip() if not profile_id: - raise ApiError(ErrorCode.VALIDATION_FAILED, "profile_id is required", {"field": "profile_id"}) + error = ApiError(ErrorCode.VALIDATION_FAILED, "profile_id is required", {"field": "profile_id"}) + self._record_failure(source, payload, error) + raise error categories = payload.get("categories") or _categories_from_keywords(payload.get("keywords") or []) - match = self.wardrobe_service.match(profile_id, {"categories": categories}, ctx) + try: + match = self.wardrobe_service.match(profile_id, {"categories": categories}, ctx) + except Exception as exc: + self._record_failure(source, payload, exc) + raise fallback_prompt = _fallback_prompt(payload) final_prompt = match["outfit_prompt"] or fallback_prompt response = { @@ -57,8 +65,31 @@ class IntegrationService: ) return response - def list_integration_logs(self, integration_name: str | None = None) -> list[dict[str, Any]]: - return self.integration_repository.list(integration_name=integration_name) + def list_integration_logs( + self, + integration_name: str | None = None, + *, + action: str | None = None, + status: str | None = None, + ) -> list[dict[str, Any]]: + return self.integration_repository.list( + integration_name=integration_name, + action=action, + status=status, + ) + + def _record_failure(self, source: str, payload: dict[str, Any], exc: Exception) -> None: + self.integration_repository.record( + { + "integration_log_id": new_integration_log_id(), + "integration_name": source, + "action": "daily_outfit", + "status": "failed", + "request": payload, + "response": {"error_code": _error_code(exc)}, + "error_message": str(getattr(exc, "message", None) or exc), + } + ) def new_integration_log_id() -> str: @@ -83,3 +114,9 @@ def _fallback_prompt(payload: dict[str, Any]) -> str: keywords = payload.get("keywords") or [] parts.extend(str(keyword) for keyword in keywords) return ", ".join(parts) + + +def _error_code(exc: Exception) -> str: + code = getattr(exc, "error_code", None) + value = getattr(code, "value", None) + return str(value or code or exc.__class__.__name__) diff --git a/backend/services/log_service.py b/backend/services/log_service.py index 125554b..37d627e 100644 --- a/backend/services/log_service.py +++ b/backend/services/log_service.py @@ -12,18 +12,39 @@ class LogService: def __init__(self, database: Database): self.database = database - def generation_logs(self, limit: int = 50) -> list[dict[str, Any]]: + def generation_logs( + self, + limit: int = 50, + *, + status: str | None = None, + workflow_id: str | None = None, + profile_id: str | None = None, + node_id: str | None = None, + ) -> list[dict[str, Any]]: placeholder = self.database._placeholder + where: list[str] = [] + params: list[Any] = [] + for column, value in ( + ("status", status), + ("workflow_id", workflow_id), + ("profile_id", profile_id), + ("node_id", node_id), + ): + if value: + where.append(f"{column} = {placeholder}") + params.append(value) + where_sql = f"WHERE {' AND '.join(where)}" if where else "" conn = self.database.connect() try: rows = conn.execute( f""" SELECT task_id, profile_id, workflow_id, node_id, status, progress, error_message, created_at, updated_at FROM generation_tasks + {where_sql} ORDER BY created_at DESC LIMIT {placeholder} """, - (_limit(limit),), + tuple(params + [_limit(limit)]), ).fetchall() finally: conn.close() @@ -42,20 +63,38 @@ class LogService: for row in rows ] - def workflow_logs(self, workflow_id: str | None = None, limit: int = 50) -> list[dict[str, Any]]: + def workflow_logs( + self, + workflow_id: str | None = None, + limit: int = 50, + *, + action: str | None = None, + node_id: str | None = None, + branch_name: str | None = None, + ) -> list[dict[str, Any]]: placeholder = self.database._placeholder - where = "" + where: list[str] = [] params: list[Any] = [] if workflow_id: - where = f"WHERE workflow_id = {placeholder}" + where.append(f"workflow_id = {placeholder}") params.append(workflow_id) + if action: + where.append(f"action = {placeholder}") + params.append(action) + if node_id: + where.append(f"node_id = {placeholder}") + params.append(node_id) + if branch_name: + where.append(f"branch_name = {placeholder}") + params.append(branch_name) + where_sql = f"WHERE {' AND '.join(where)}" if where else "" conn = self.database.connect() try: rows = conn.execute( f""" SELECT history_id, profile_id, workflow_id, node_id, image_id, branch_name, action, title, created_at FROM workflow_history - {where} + {where_sql} ORDER BY created_at DESC LIMIT {placeholder} """, @@ -78,20 +117,35 @@ class LogService: for row in rows ] - def integration_logs(self, integration_name: str | None = None, limit: int = 50) -> list[dict[str, Any]]: + def integration_logs( + self, + integration_name: str | None = None, + limit: int = 50, + *, + action: str | None = None, + status: str | None = None, + ) -> list[dict[str, Any]]: placeholder = self.database._placeholder - where = "" + where: list[str] = [] params: list[Any] = [] if integration_name: - where = f"WHERE integration_name = {placeholder}" + where.append(f"integration_name = {placeholder}") params.append(integration_name) + if action: + where.append(f"action = {placeholder}") + params.append(action) + if status: + where.append(f"status = {placeholder}") + params.append(status) + where_sql = f"WHERE {' AND '.join(where)}" if where else "" conn = self.database.connect() try: rows = conn.execute( f""" - SELECT integration_log_id, integration_name, action, status, error_message, created_at + SELECT integration_log_id, integration_name, action, status, + request_json, response_json, error_message, created_at FROM integration_logs - {where} + {where_sql} ORDER BY created_at DESC LIMIT {placeholder} """, @@ -105,36 +159,59 @@ class LogService: "integration_name": row["integration_name"], "action": row["action"], "status": row["status"], + "request": _json_loads(row["request_json"]) or {}, + "response": _json_loads(row["response_json"]) or {}, "error_message": row["error_message"], "created_at": str(row["created_at"]), } for row in rows ] - def error_logs(self, limit: int = 50) -> list[dict[str, Any]]: + def error_logs( + self, + limit: int = 50, + *, + source: str | None = None, + status: str | None = None, + ) -> list[dict[str, Any]]: placeholder = self.database._placeholder + include_tasks = source in {None, "generation_task"} + include_integrations = source in {None, "integration"} + task_statuses = ("failed", "dead_letter") + task_status_filter = status if status in task_statuses else None + integration_status_filter = status if status else None conn = self.database.connect() try: - task_rows = conn.execute( - f""" - SELECT task_id AS id, status, error_message, created_at - FROM generation_tasks - WHERE status IN ('failed', 'dead_letter') - ORDER BY created_at DESC - LIMIT {placeholder} - """, - (_limit(limit),), - ).fetchall() - integration_rows = conn.execute( - f""" - SELECT integration_log_id AS id, status, error_message, created_at - FROM integration_logs - WHERE status != 'succeeded' - ORDER BY created_at DESC - LIMIT {placeholder} - """, - (_limit(limit),), - ).fetchall() + task_rows = [] + integration_rows = [] + if include_tasks: + task_status_sql = f"= {placeholder}" if task_status_filter else "IN ('failed', 'dead_letter')" + task_params: tuple[Any, ...] = (task_status_filter, _limit(limit)) if task_status_filter else (_limit(limit),) + task_rows = conn.execute( + f""" + SELECT task_id AS id, status, error_message, created_at + FROM generation_tasks + WHERE status {task_status_sql} + ORDER BY created_at DESC + LIMIT {placeholder} + """, + task_params, + ).fetchall() + if include_integrations: + integration_where = f"status = {placeholder}" if integration_status_filter else "status != 'succeeded'" + integration_params: tuple[Any, ...] = ( + (integration_status_filter, _limit(limit)) if integration_status_filter else (_limit(limit),) + ) + integration_rows = conn.execute( + f""" + SELECT integration_log_id AS id, status, error_message, created_at + FROM integration_logs + WHERE {integration_where} + ORDER BY created_at DESC + LIMIT {placeholder} + """, + integration_params, + ).fetchall() finally: conn.close() errors = [ diff --git a/backend/services/settings_service.py b/backend/services/settings_service.py index 20ba579..5c0449d 100644 --- a/backend/services/settings_service.py +++ b/backend/services/settings_service.py @@ -244,21 +244,35 @@ class SettingsService: ) return snapshots + def get_snapshot(self, snapshot_id: str) -> dict[str, Any]: + if not self.audit_service: + raise ApiError(ErrorCode.VALIDATION_FAILED, "config snapshots are unavailable", {}) + normalized_snapshot_id = str(snapshot_id or "").strip() + if not normalized_snapshot_id: + raise ApiError(ErrorCode.VALIDATION_FAILED, "snapshot_id is required", {"field": "snapshot_id"}) + row = self._load_snapshot_row(normalized_snapshot_id) + if row is None: + raise ApiError( + ErrorCode.RESOURCE_NOT_FOUND, + "config snapshot not found", + {"snapshot_id": normalized_snapshot_id}, + ) + config = json.loads(row["config_json"] or "{}") + return { + "snapshot_id": row["snapshot_id"], + "reason": row["reason"], + "actor_id": row["actor_id"], + "config": _mask_sensitive(config), + "created_at": row["created_at"], + } + def rollback(self, payload: Mapping[str, Any], ctx: RequestContext) -> dict[str, Any]: if not self.audit_service: raise ApiError(ErrorCode.VALIDATION_FAILED, "config snapshots are unavailable", {}) snapshot_id = str(payload.get("snapshot_id") or "").strip() if not snapshot_id: raise ApiError(ErrorCode.VALIDATION_FAILED, "snapshot_id is required", {"field": "snapshot_id"}) - placeholder = self.audit_service.database._placeholder - conn = self.audit_service.database.connect() - try: - row = conn.execute( - f"SELECT snapshot_id, config_json FROM config_snapshots WHERE snapshot_id = {placeholder}", - (snapshot_id,), - ).fetchone() - finally: - conn.close() + row = self._load_snapshot_row(snapshot_id) if row is None: raise ApiError(ErrorCode.RESOURCE_NOT_FOUND, "config snapshot not found", {"snapshot_id": snapshot_id}) config = json.loads(row["config_json"] or "{}") @@ -281,6 +295,23 @@ class SettingsService: ) return {"rolled_back": True, "snapshot_id": snapshot_id, "config": _mask_sensitive(after)} + def _load_snapshot_row(self, snapshot_id: str): + if not self.audit_service: + return None + placeholder = self.audit_service.database._placeholder + conn = self.audit_service.database.connect() + try: + return conn.execute( + f""" + SELECT snapshot_id, reason, actor_id, config_json, created_at + FROM config_snapshots + WHERE snapshot_id = {placeholder} + """, + (snapshot_id,), + ).fetchone() + finally: + conn.close() + def _load_raw_schema(self) -> dict[str, Any]: if not self.schema_path.exists(): return {} diff --git a/database/repositories/integration_repository.py b/database/repositories/integration_repository.py index 2c1a67a..396de9f 100644 --- a/database/repositories/integration_repository.py +++ b/database/repositories/integration_repository.py @@ -50,20 +50,34 @@ class IntegrationRepository: conn.close() return payload - def list(self, integration_name: str | None = None, limit: int = 50) -> list[dict[str, Any]]: + def list( + self, + integration_name: str | None = None, + limit: int = 50, + *, + action: str | None = None, + status: str | None = None, + ) -> list[dict[str, Any]]: placeholder = self.placeholder - where = "" + where: list[str] = [] params: list[Any] = [] if integration_name: - where = f"WHERE integration_name = {placeholder}" + where.append(f"integration_name = {placeholder}") params.append(integration_name) + if action: + where.append(f"action = {placeholder}") + params.append(action) + if status: + where.append(f"status = {placeholder}") + params.append(status) + where_sql = f"WHERE {' AND '.join(where)}" if where else "" conn = self.database.connect() try: rows = conn.execute( f""" SELECT integration_log_id, integration_name, action, status, request_json, response_json, error_message, created_at FROM integration_logs - {where} + {where_sql} ORDER BY created_at DESC LIMIT {placeholder} """, diff --git a/tests/test_integration_service.py b/tests/test_integration_service.py index 17ac14e..e56e129 100644 --- a/tests/test_integration_service.py +++ b/tests/test_integration_service.py @@ -120,6 +120,24 @@ class IntegrationServiceTests(unittest.TestCase): self.assertFalse(bad_source["ok"]) self.assertEqual(bad_source["error_code"], "INTEGRATION_TOKEN_INVALID") + failed_logs = list_integration_logs( + self.ctx, + {"integration_name": "unknown_plugin", "status": "failed"}, + self.integration_service, + ) + self.assertTrue(failed_logs["ok"]) + self.assertEqual(len(failed_logs["data"]), 1) + self.assertEqual(failed_logs["data"][0]["response"]["error_code"], "INTEGRATION_TOKEN_INVALID") + + def test_daily_outfit_validation_failure_records_failed_log(self): + with self.assertRaises(Exception): + self.integration_service.daily_outfit({"keywords": ["top"]}, self.ctx) + + logs = self.integration_service.list_integration_logs(status="failed") + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0]["action"], "daily_outfit") + self.assertEqual(logs[0]["response"]["error_code"], "VALIDATION_FAILED") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_log_routes.py b/tests/test_log_routes.py index 25f9897..e482b9b 100644 --- a/tests/test_log_routes.py +++ b/tests/test_log_routes.py @@ -98,18 +98,44 @@ class LogRoutesTests(unittest.TestCase): self.assertTrue(generation["ok"]) self.assertEqual(generation["data"][0]["status"], "failed") + filtered_generation = get_generation_logs(self.ctx, {"status": "failed", "limit": 5}, self.log_service) + self.assertTrue(filtered_generation["ok"]) + self.assertTrue(filtered_generation["data"]) + self.assertTrue(all(item["status"] == "failed" for item in filtered_generation["data"])) + workflow_logs = get_workflow_logs(self.ctx, {"workflow_id": workflow.workflow.workflow_id}, self.log_service) self.assertTrue(workflow_logs["ok"]) self.assertEqual(workflow_logs["data"][0]["action"], "adopt_image") + filtered_workflow_logs = get_workflow_logs( + self.ctx, + {"workflow_id": workflow.workflow.workflow_id, "action": "adopt_image"}, + self.log_service, + ) + self.assertTrue(filtered_workflow_logs["ok"]) + self.assertEqual(filtered_workflow_logs["data"][0]["node_id"], node_id) + integrations = get_integration_logs(self.ctx, {}, self.log_service) self.assertTrue(integrations["ok"]) self.assertEqual(integrations["data"][0]["action"], "daily_outfit") + self.assertIn("profile_id", integrations["data"][0]["request"]) + + filtered_integrations = get_integration_logs( + self.ctx, + {"status": "succeeded", "action": "daily_outfit"}, + self.log_service, + ) + self.assertTrue(filtered_integrations["ok"]) + self.assertTrue(all(item["status"] == "succeeded" for item in filtered_integrations["data"])) errors = get_error_logs(self.ctx, {}, self.log_service) self.assertTrue(errors["ok"]) self.assertEqual(errors["data"][0]["source"], "generation_task") + task_errors = get_error_logs(self.ctx, {"source": "generation_task", "status": "failed"}, self.log_service) + self.assertTrue(task_errors["ok"]) + self.assertTrue(all(item["source"] == "generation_task" for item in task_errors["data"])) + audit = get_audit_logs(self.ctx, {"limit": 10}, self.log_service) self.assertTrue(audit["ok"]) self.assertTrue(any(item["action"] == "create_profile" for item in audit["data"])) diff --git a/tests/test_route_registry.py b/tests/test_route_registry.py index 8cc32d0..40753f7 100644 --- a/tests/test_route_registry.py +++ b/tests/test_route_registry.py @@ -28,6 +28,7 @@ class RouteRegistryTests(unittest.TestCase): self.assertIn(("POST", "nodes/{node_id}/retry"), keys) self.assertIn(("POST", "nodes/{node_id}/complete"), keys) self.assertIn(("GET", "settings/snapshots"), keys) + self.assertIn(("GET", "settings/snapshots/{snapshot_id}"), keys) self.assertIn(("POST", "settings/rollback"), keys) self.assertEqual(len(routes), len(ROUTES)) diff --git a/tests/test_s2_routes.py b/tests/test_s2_routes.py index f0fe4a0..126a2ca 100644 --- a/tests/test_s2_routes.py +++ b/tests/test_s2_routes.py @@ -41,6 +41,7 @@ from backend.routes.settings_routes import ( diff_settings, export_settings, get_current_settings, + get_settings_snapshot, get_settings_schema, import_settings, list_settings_snapshots, @@ -534,6 +535,11 @@ database: old_snapshot = next( item for item in snapshots["data"] if item["config"].get("plugin", {}).get("display_name") == "Old Name" ) + snapshot_detail = get_settings_snapshot(self.ctx, old_snapshot["snapshot_id"], self.settings_service) + self.assertTrue(snapshot_detail["ok"]) + self.assertEqual(snapshot_detail["data"]["snapshot_id"], old_snapshot["snapshot_id"]) + self.assertNotIn("secret", str(snapshot_detail["data"])) + rollback = rollback_settings( self.ctx, {"snapshot_id": old_snapshot["snapshot_id"]},