256 行
11 KiB
Python
256 行
11 KiB
Python
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""Structured, privacy-limited DTOs for Bot personal archive handoff."""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
from copy import deepcopy
|
|||
|
|
from dataclasses import dataclass
|
|||
|
|
from datetime import datetime
|
|||
|
|
import re
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
FORBIDDEN_KEY_PARTS = {
|
|||
|
|
"prompt", "conversation", "chat_history", "transcript", "cookie", "token", "password",
|
|||
|
|
"passwd", "secret", "credential", "authorization", "api_key", "apikey", "access_key",
|
|||
|
|
"private_key", "base64", "raw_prompt", "raw_message", "binary", "messages", "message_chain", "attachment",
|
|||
|
|
# contexts 是 AstrBot 里 LLM 会话历史的字段名,属「完整会话」,必须挡在归档之外
|
|||
|
|
"contexts",
|
|||
|
|
# 只禁媒体「内容」,不禁媒体「标识」。原先有一条裸 "media",把 media_id / media_kind /
|
|||
|
|
# media_name 一并封死,等于让合同要求的「媒体只存索引」根本无法实现。
|
|||
|
|
"media_bytes", "raw_media", "media_binary", "media_data", "media_content", "media_blob",
|
|||
|
|
"image_bytes", "audio_bytes", "video_bytes",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
WINDOW_ALIASES = {
|
|||
|
|
"深夜": "late_night", "凌晨": "late_night", "早晨": "morning", "上午": "morning",
|
|||
|
|
"下午": "afternoon", "晚上": "evening", "傍晚": "evening",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
CERTAINTY_ALIASES = {"high": 0.9, "medium": 0.6, "low": 0.3, "高": 0.9, "中": 0.6, "低": 0.3}
|
|||
|
|
|
|||
|
|
# 与记忆侧 core/bot_personal.py:BOT_PERSONAL_TYPE_CONTRACTS 对齐:
|
|||
|
|
# (source_kind, 默认 evidence_level, 默认 status)。
|
|||
|
|
# 记忆侧一律用合同 source_kind 覆盖调用方,这里同步固定;R1 只覆盖了严格五类,
|
|||
|
|
# 其余七类落到 "observed" 兜底,导致本地记录把创作/梦境/日历等标成真实观察。
|
|||
|
|
TYPE_CONTRACTS = {
|
|||
|
|
"bot_schedule_plan": ("planned", "L0", "planned"),
|
|||
|
|
"bot_observed_activity": ("observed", "L2", "active"),
|
|||
|
|
"bot_schedule_reconciliation": ("reconciled", "L3", "completed"),
|
|||
|
|
"bot_window_snapshot": ("projection", "L2", "completed"),
|
|||
|
|
"bot_daily_diary": ("subjective", "L1", "completed"),
|
|||
|
|
"bot_creative_work": ("creative", "L1", "active"),
|
|||
|
|
"bot_media_memory": ("media", "L1", "active"),
|
|||
|
|
"bot_subjective_memory": ("subjective", "L1", "active"),
|
|||
|
|
"bot_shared_activity": ("shared", "L1", "active"),
|
|||
|
|
"bot_detail_fragment": ("detail", "L1", "active"),
|
|||
|
|
"bot_calendar_event": ("calendar", "L1", "active"),
|
|||
|
|
"bot_proactive_message": ("proactive", "L1", "active"),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
class PrivacyRejected(ValueError):
|
|||
|
|
"""Raised when an archive payload contains forbidden private material."""
|
|||
|
|
|
|||
|
|
def __init__(self, path: str, reason: str) -> None:
|
|||
|
|
self.path = path
|
|||
|
|
self.reason = reason
|
|||
|
|
super().__init__(f"privacy_rejected:{path}:{reason}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _text(value: Any, limit: int = 240) -> str:
|
|||
|
|
return str(value or "").strip()[:limit]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _certainty(value: Any, default: float = 0.6) -> float:
|
|||
|
|
if isinstance(value, str):
|
|||
|
|
value = CERTAINTY_ALIASES.get(value.strip().lower(), value)
|
|||
|
|
try:
|
|||
|
|
result = float(value)
|
|||
|
|
except (TypeError, ValueError):
|
|||
|
|
result = default
|
|||
|
|
return max(0.0, min(1.0, result))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _window(value: Any) -> str:
|
|||
|
|
text = _text(value, 24)
|
|||
|
|
return WINDOW_ALIASES.get(text, text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _parse_moment(value: Any) -> datetime | None:
|
|||
|
|
try:
|
|||
|
|
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|||
|
|
except (TypeError, ValueError):
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _derive_date(value: Any, occurred_at: str, now: datetime) -> str:
|
|||
|
|
"""归档日期:payload 显式 date/window_date 优先,缺失时从 occurred_at 派生。
|
|||
|
|
|
|||
|
|
派生刻意不做时区换算,取 occurred_at 自带偏移下的墙钟日期,与 _infer_window 同一时钟;
|
|||
|
|
若换算到宿主时区,跨零点时会出现 date 与 window 来自两个时钟的错位。
|
|||
|
|
occurred_at 语义不变,派生只补 date 字段本身。
|
|||
|
|
"""
|
|||
|
|
explicit = _text(value, 20)
|
|||
|
|
if explicit:
|
|||
|
|
return explicit.split("T")[0]
|
|||
|
|
parsed = _parse_moment(occurred_at) or now.astimezone()
|
|||
|
|
return parsed.date().isoformat()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _infer_window(value: Any, occurred_at: str, memory_type: str) -> str:
|
|||
|
|
explicit = _window(value)
|
|||
|
|
if explicit:
|
|||
|
|
return explicit
|
|||
|
|
if memory_type == "bot_schedule_plan":
|
|||
|
|
return "morning"
|
|||
|
|
parsed = _parse_moment(occurred_at)
|
|||
|
|
if parsed is None:
|
|||
|
|
return "morning"
|
|||
|
|
hour = parsed.hour
|
|||
|
|
if hour >= 22 or hour < 6:
|
|||
|
|
return "late_night"
|
|||
|
|
if hour < 12:
|
|||
|
|
return "morning"
|
|||
|
|
if hour < 18:
|
|||
|
|
return "afternoon"
|
|||
|
|
return "evening"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _looks_sensitive_value(value: str) -> str:
|
|||
|
|
text = value.strip()
|
|||
|
|
lowered = text.lower()
|
|||
|
|
if re.search(r"(?:password|passwd|token|secret|api[_-]?key|authorization)\s*[:=]", lowered):
|
|||
|
|
return "credential_pattern"
|
|||
|
|
if re.search(r"^bearer\s+", lowered):
|
|||
|
|
return "authorization_value"
|
|||
|
|
if "base64," in lowered or lowered.startswith("data:"):
|
|||
|
|
return "base64_or_data_uri"
|
|||
|
|
if "-----begin " in lowered:
|
|||
|
|
return "private_key_material"
|
|||
|
|
# 边界类含 : ; , 是因为路径常跟在分隔符后面(幂等键 `photo:D:\...`、`标题:C:/x` 都曾漏检),
|
|||
|
|
# 只有 ^|\s|= 会放过它们。`https://` 不受影响:`s:/` 前面是 `p`,不在边界类里。
|
|||
|
|
if re.search(r"(?:^|[\s=:;,])(?:[a-z]:[\\/]|\\\\|/home/|/root/|/tmp/|/var/|/volume\d+/)", text, re.IGNORECASE):
|
|||
|
|
return "absolute_path"
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate_bot_personal_key(value: Any, *, field: str = "idempotency_key") -> None:
|
|||
|
|
"""幂等键与 payload 走同一条隐私底线。
|
|||
|
|
|
|||
|
|
幂等键不在 payload 里,此前完全绕过校验,却会原样进入 envelope 的 idempotency_key、
|
|||
|
|
落进记忆库、并参与 record_id 计算——等于给合同禁止的数据留了一条未校验通道
|
|||
|
|
(曾经的 `photo:{绝对路径}:{prompt}` 就是这么来的)。
|
|||
|
|
"""
|
|||
|
|
reason = _looks_sensitive_value(str(value or ""))
|
|||
|
|
if reason:
|
|||
|
|
raise PrivacyRejected(field, reason)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def validate_bot_personal_payload(value: Any, *, path: str = "payload", depth: int = 0) -> None:
|
|||
|
|
if depth > 8:
|
|||
|
|
raise PrivacyRejected(path, "max_depth")
|
|||
|
|
if isinstance(value, dict):
|
|||
|
|
for key, item in value.items():
|
|||
|
|
name = str(key or "").strip()
|
|||
|
|
lowered = name.lower().replace("-", "_")
|
|||
|
|
if any(part in lowered for part in FORBIDDEN_KEY_PARTS):
|
|||
|
|
raise PrivacyRejected(f"{path}.{name}", "forbidden_key")
|
|||
|
|
validate_bot_personal_payload(item, path=f"{path}.{name}", depth=depth + 1)
|
|||
|
|
return
|
|||
|
|
if isinstance(value, (list, tuple)):
|
|||
|
|
for index, item in enumerate(value):
|
|||
|
|
validate_bot_personal_payload(item, path=f"{path}[{index}]", depth=depth + 1)
|
|||
|
|
return
|
|||
|
|
if isinstance(value, (bytes, bytearray, memoryview)):
|
|||
|
|
raise PrivacyRejected(path, "binary_media")
|
|||
|
|
if isinstance(value, str):
|
|||
|
|
reason = _looks_sensitive_value(value)
|
|||
|
|
if reason:
|
|||
|
|
raise PrivacyRejected(path, reason)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _safe_value(value: Any, *, depth: int = 0, max_depth: int = 8) -> Any:
|
|||
|
|
if depth > max_depth:
|
|||
|
|
return None
|
|||
|
|
if isinstance(value, dict):
|
|||
|
|
return {
|
|||
|
|
_text(key, 80): _safe_value(item, depth=depth + 1, max_depth=max_depth)
|
|||
|
|
for key, item in list(value.items())[:64]
|
|||
|
|
if _text(key, 80)
|
|||
|
|
}
|
|||
|
|
if isinstance(value, (list, tuple)):
|
|||
|
|
return [_safe_value(item, depth=depth + 1, max_depth=max_depth) for item in list(value)[:64]]
|
|||
|
|
if isinstance(value, (str, int, float, bool)) or value is None:
|
|||
|
|
return _text(value, 1200) if isinstance(value, str) else value
|
|||
|
|
return _text(value, 240)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True)
|
|||
|
|
class BotPersonalArchiveDTO:
|
|||
|
|
memory_type: str
|
|||
|
|
record_id: str
|
|||
|
|
memory_domain: str
|
|||
|
|
subject: str
|
|||
|
|
date: str
|
|||
|
|
window: str
|
|||
|
|
occurred_at: str
|
|||
|
|
created_at: str
|
|||
|
|
updated_at: str
|
|||
|
|
source_kind: str
|
|||
|
|
source_refs: list[str]
|
|||
|
|
certainty: float
|
|||
|
|
evidence_level: str
|
|||
|
|
status: str
|
|||
|
|
version: int
|
|||
|
|
idempotency_key: str
|
|||
|
|
payload_schema_version: str
|
|||
|
|
payload: dict[str, Any]
|
|||
|
|
|
|||
|
|
def envelope(self) -> dict[str, Any]:
|
|||
|
|
return {
|
|||
|
|
"record_id": self.record_id, "memory_domain": self.memory_domain, "memory_type": self.memory_type,
|
|||
|
|
"subject": self.subject, "date": self.date, "window": self.window, "occurred_at": self.occurred_at,
|
|||
|
|
"created_at": self.created_at, "updated_at": self.updated_at, "source_kind": self.source_kind,
|
|||
|
|
"source_refs": list(self.source_refs), "certainty": self.certainty, "evidence_level": self.evidence_level,
|
|||
|
|
"status": self.status, "version": self.version, "idempotency_key": self.idempotency_key,
|
|||
|
|
"payload_schema_version": self.payload_schema_version, "payload": deepcopy(self.payload),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_bot_personal_dto(
|
|||
|
|
*, memory_type: str, kind: str, namespace: str, payload: dict[str, Any], idempotency_key: str,
|
|||
|
|
occurred_at: str, now: datetime, version: int = 1,
|
|||
|
|
) -> BotPersonalArchiveDTO:
|
|||
|
|
validate_bot_personal_payload(payload)
|
|||
|
|
validate_bot_personal_key(idempotency_key)
|
|||
|
|
safe = _safe_value(payload) or {}
|
|||
|
|
date = _derive_date(safe.get("date") or safe.get("window_date"), occurred_at, now)
|
|||
|
|
window = _infer_window(safe.get("window"), occurred_at, memory_type)
|
|||
|
|
source_refs = []
|
|||
|
|
for item in safe.get("source_refs") or []:
|
|||
|
|
text = _text(item, 160)
|
|||
|
|
if text and text not in source_refs:
|
|||
|
|
source_refs.append(text)
|
|||
|
|
source_kind = _text(safe.get("source_kind"), 24)
|
|||
|
|
evidence_level = _text(safe.get("evidence_level"), 8).upper()
|
|||
|
|
certainty = _certainty(safe.get("certainty"), 0.6)
|
|||
|
|
status = _text(safe.get("status"), 32)
|
|||
|
|
contract = TYPE_CONTRACTS.get(memory_type)
|
|||
|
|
if contract:
|
|||
|
|
contract_kind, default_evidence, default_status = contract
|
|||
|
|
source_kind = contract_kind
|
|||
|
|
# 计划恒为 L0(合同强制),其余类型保留调用方证据等级
|
|||
|
|
evidence_level = "L0" if memory_type == "bot_schedule_plan" else (evidence_level or default_evidence)
|
|||
|
|
status = status or default_status
|
|||
|
|
if not source_refs:
|
|||
|
|
source_refs = [f"archive:{_text(idempotency_key, 240)}"]
|
|||
|
|
created_at = _text(safe.get("created_at"), 80) or now.astimezone().isoformat(timespec="seconds")
|
|||
|
|
updated_at = _text(safe.get("updated_at"), 80) or created_at
|
|||
|
|
return BotPersonalArchiveDTO(
|
|||
|
|
memory_type=_text(memory_type, 80), record_id=_text(safe.get("record_id"), 160) or f"local:{_text(idempotency_key, 240)}",
|
|||
|
|
memory_domain="bot_self_schedule", subject="bot_self", date=date, window=window,
|
|||
|
|
occurred_at=_text(occurred_at, 80) or now.astimezone().isoformat(timespec="seconds"), created_at=created_at, updated_at=updated_at,
|
|||
|
|
source_kind=source_kind or "unspecified", source_refs=source_refs, certainty=certainty,
|
|||
|
|
evidence_level=evidence_level or "L1", status=status or "active", version=max(1, int(version or 1)),
|
|||
|
|
idempotency_key=_text(idempotency_key, 240), payload_schema_version="1.0", payload=safe,
|
|||
|
|
)
|