feat(p5): add provenance isolation gates
这个提交存在于:
父节点
6db395beb9
当前提交
274c9b95ea
共有 12 个文件被更改,包括 3008 次插入 和 12 次删除
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import threading
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
|
|
@ -15,6 +16,10 @@ from .unified_person_contract import (
|
|||
)
|
||||
from .person_memory_contract import capabilities as person_memory_capabilities, consume_person_memory_envelope
|
||||
from .context_orchestration import contract_descriptor, contract_fingerprint
|
||||
from .p5_provenance import (
|
||||
contract_descriptor as p5_provenance_contract_descriptor,
|
||||
contract_fingerprint as p5_provenance_contract_fingerprint,
|
||||
)
|
||||
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
|
@ -43,6 +48,28 @@ class MemoryCompanionBridge:
|
|||
|
||||
def __init__(self, plugin: Any):
|
||||
self._plugin = plugin
|
||||
self._p5_attestation_verifier: Any = None
|
||||
self._p5_attestation_verifier_bound = False
|
||||
self._p5_attestation_verifier_lock = threading.RLock()
|
||||
|
||||
def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool:
|
||||
"""Pin the first in-process P5 verifier; replacement is forbidden."""
|
||||
|
||||
if not callable(consume_callable):
|
||||
return False
|
||||
with self._p5_attestation_verifier_lock:
|
||||
if self._p5_attestation_verifier_bound:
|
||||
return False
|
||||
target_binder = getattr(self._plugin, "bind_p5_attestation_verifier", None)
|
||||
if callable(target_binder):
|
||||
try:
|
||||
if target_binder(consume_callable) is not True:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
self._p5_attestation_verifier = consume_callable
|
||||
self._p5_attestation_verifier_bound = True
|
||||
return True
|
||||
|
||||
def _trusted_session_context(self, session_context: SessionContext | dict[str, Any] | None) -> SessionContext | dict[str, Any] | None:
|
||||
"""Normalize a legacy dict at the public bridge boundary.
|
||||
|
|
@ -164,6 +191,23 @@ class MemoryCompanionBridge:
|
|||
result["operations"] = list(methods); result["operation_availability"] = operation_availability
|
||||
return result
|
||||
|
||||
def probe_p5_provenance_migration_capabilities(self) -> dict[str, Any]:
|
||||
methods = {
|
||||
"preview": "preview_p5_provenance_migration",
|
||||
"apply": "apply_p5_provenance_migration",
|
||||
"rollback": "rollback_p5_provenance_migration",
|
||||
}
|
||||
availability = {name: callable(getattr(self._plugin, method, None)) for name, method in methods.items()}
|
||||
return {
|
||||
"available": all(availability.values()),
|
||||
"mode": "migration_only",
|
||||
"runtime_sink_behavior": "not_connected",
|
||||
"contract": p5_provenance_contract_descriptor(),
|
||||
"contract_fingerprint": p5_provenance_contract_fingerprint(),
|
||||
"operations": list(methods),
|
||||
"operation_availability": availability,
|
||||
}
|
||||
|
||||
def consume_person_memory_envelope(self, envelope: dict[str, Any] | None, *, companion_available: bool = True) -> dict[str, Any]:
|
||||
return consume_person_memory_envelope(envelope, companion_available=companion_available)
|
||||
|
||||
|
|
@ -182,6 +226,27 @@ class MemoryCompanionBridge:
|
|||
if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"}
|
||||
return await getter(migration_id)
|
||||
|
||||
async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]:
|
||||
getter = getattr(self._plugin, "preview_p5_provenance_migration", None)
|
||||
if not callable(getter):
|
||||
return {"mode": "preview", "ok": False, "readonly": True, "error_code": "bridge_method_unavailable", "count": 0}
|
||||
result = await getter(limit=limit)
|
||||
return result if isinstance(result, dict) else {"mode": "preview", "ok": False, "readonly": True, "error_code": "invalid_bridge_response", "count": 0}
|
||||
|
||||
async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
|
||||
getter = getattr(self._plugin, "apply_p5_provenance_migration", None)
|
||||
if not callable(getter):
|
||||
return {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "changed": 0}
|
||||
result = await getter(operation_id=operation_id, limit=limit)
|
||||
return result if isinstance(result, dict) else {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "changed": 0}
|
||||
|
||||
async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]:
|
||||
getter = getattr(self._plugin, "rollback_p5_provenance_migration", None)
|
||||
if not callable(getter):
|
||||
return {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "restored": 0, "conflicts": 0}
|
||||
result = await getter(operation_id)
|
||||
return result if isinstance(result, dict) else {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "restored": 0, "conflicts": 0}
|
||||
|
||||
async def correct_person_memory(self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord) -> dict[str, Any]:
|
||||
getter = getattr(self._plugin, "bridge_correct_person_memory", None)
|
||||
if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"}
|
||||
|
|
@ -702,6 +767,7 @@ class MemoryCompanionBridge:
|
|||
max_chars: int | None = None,
|
||||
companion_bot_mood: str = "",
|
||||
companion_bot_energy: float = 0.0,
|
||||
p5_b1_bridge_invocation: Any = None,
|
||||
) -> str:
|
||||
return await self._plugin.bridge_compose_injection(
|
||||
query,
|
||||
|
|
@ -710,6 +776,7 @@ class MemoryCompanionBridge:
|
|||
max_chars=max_chars,
|
||||
companion_bot_mood=companion_bot_mood,
|
||||
companion_bot_energy=companion_bot_energy,
|
||||
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
|
||||
)
|
||||
|
||||
def get_reply_delay(
|
||||
|
|
@ -739,6 +806,7 @@ class MemoryCompanionBridge:
|
|||
companion_bot_mood: str = "",
|
||||
companion_bot_energy: float = 0.0,
|
||||
retrieval_profile: str = "",
|
||||
p5_b1_bridge_invocation: Any = None,
|
||||
) -> str:
|
||||
return await self._plugin.bridge_compose_context(
|
||||
query=query,
|
||||
|
|
@ -748,6 +816,7 @@ class MemoryCompanionBridge:
|
|||
companion_bot_mood=companion_bot_mood,
|
||||
companion_bot_energy=companion_bot_energy,
|
||||
retrieval_profile=retrieval_profile,
|
||||
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
|
||||
)
|
||||
|
||||
async def create_archive_memory(
|
||||
|
|
|
|||
139
core/p5_b1_recall_gate.py
普通文件
139
core/p5_b1_recall_gate.py
普通文件
|
|
@ -0,0 +1,139 @@
|
|||
"""P5-B1 attestation gates for normal recall and Bridge serialization.
|
||||
|
||||
The handle carrier is deliberately untrusted and private to the current
|
||||
request/event runtime objects. This module never serializes it, stores it,
|
||||
or interprets a source trust label. Authenticity belongs solely to the
|
||||
one-shot verifier pinned by ``MemoryCompanionBridge``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
P5_B1_ATTESTATION_HANDLES_ATTRIBUTE = "private_companion_p5_b1_attestation_handles"
|
||||
MEMORY_RECALL_SINK = "memory_recall"
|
||||
BRIDGE_SERIALIZATION_SINK = "bridge_serialization"
|
||||
_BRIDGE_INVOCATION_ARGUMENTS_METHOD = "_p5_b1_bridge_attestation_arguments"
|
||||
|
||||
|
||||
def _carrier_handle(req: Any, event: Any) -> Any:
|
||||
"""Read a carrier value without copying it onto another object."""
|
||||
|
||||
for carrier in (req, event):
|
||||
handles = getattr(carrier, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, None)
|
||||
if isinstance(handles, Mapping) and MEMORY_RECALL_SINK in handles:
|
||||
return handles.get(MEMORY_RECALL_SINK)
|
||||
return None
|
||||
|
||||
|
||||
def _snapshot_field(snapshot: Any, name: str) -> Any:
|
||||
if isinstance(snapshot, Mapping):
|
||||
return snapshot.get(name)
|
||||
return getattr(snapshot, name, None)
|
||||
|
||||
|
||||
def evaluate_normal_recall_gate(
|
||||
verifier: Callable[..., Any] | None,
|
||||
*,
|
||||
req: Any,
|
||||
event: Any,
|
||||
p3_state: Any,
|
||||
) -> str:
|
||||
"""Return a static safe reason or ``""`` when normal recall may proceed.
|
||||
|
||||
The pinned verifier receives the original object identities. It may
|
||||
consume a Companion opaque handle only once; any error, missing carrier,
|
||||
unavailable P3 anchor, or non-allow result fails closed without retaining
|
||||
its error or snapshot payload.
|
||||
"""
|
||||
|
||||
if not callable(verifier):
|
||||
return "p5_b1_verifier_unavailable"
|
||||
if p3_state is None:
|
||||
return "p5_b1_p3_unavailable"
|
||||
handle = _carrier_handle(req, event)
|
||||
if handle is None:
|
||||
return "p5_b1_handle_missing"
|
||||
try:
|
||||
snapshot = verifier(handle, req, event, p3_state, MEMORY_RECALL_SINK)
|
||||
except Exception:
|
||||
return "p5_b1_verifier_rejected"
|
||||
if snapshot is None:
|
||||
return "p5_b1_verifier_rejected"
|
||||
if _snapshot_field(snapshot, "sink") != MEMORY_RECALL_SINK:
|
||||
return "p5_b1_sink_mismatch"
|
||||
if _snapshot_field(snapshot, "disposition") != "allow":
|
||||
return "p5_b1_disposition_denied"
|
||||
return ""
|
||||
|
||||
|
||||
def _bridge_invocation_arguments(invocation: Any) -> tuple[Any, Any, Any, Any] | None:
|
||||
"""Read live anchors from an opaque invocation without retaining them.
|
||||
|
||||
A mapping, serialized reconstruction, or arbitrary payload cannot be an
|
||||
invocation. Even an object that mimics the private method cannot create a
|
||||
valid authorization because the pinned verifier still receives and checks
|
||||
the opaque issuer handle plus every exact object identity.
|
||||
"""
|
||||
|
||||
if invocation is None or isinstance(invocation, Mapping):
|
||||
return None
|
||||
argument_getter = getattr(invocation, _BRIDGE_INVOCATION_ARGUMENTS_METHOD, None)
|
||||
if not callable(argument_getter):
|
||||
return None
|
||||
try:
|
||||
arguments = argument_getter()
|
||||
except Exception:
|
||||
return None
|
||||
if type(arguments) is not tuple or len(arguments) != 4:
|
||||
return None
|
||||
return arguments
|
||||
|
||||
|
||||
def evaluate_bridge_serialization_gate(
|
||||
verifier: Callable[..., Any] | None,
|
||||
*,
|
||||
invocation: Any,
|
||||
) -> str:
|
||||
"""Return a static reason unless an opaque Bridge invocation is allowed.
|
||||
|
||||
The call is deliberately separate from normal recall. The actual pinned
|
||||
verifier is always called by Memory with its fixed ``bridge_serialization``
|
||||
sink; the invocation never supplies a sink, a snapshot, a SessionContext,
|
||||
or a persistence payload.
|
||||
"""
|
||||
|
||||
if not callable(verifier):
|
||||
return "p5_b1_bridge_verifier_unavailable"
|
||||
arguments = _bridge_invocation_arguments(invocation)
|
||||
if arguments is None:
|
||||
return "p5_b1_bridge_invocation_invalid"
|
||||
handle, request, event, p3_state = arguments
|
||||
if p3_state is None:
|
||||
return "p5_b1_bridge_p3_unavailable"
|
||||
if request is None or event is None:
|
||||
return "p5_b1_bridge_carrier_missing"
|
||||
if handle is None:
|
||||
return "p5_b1_bridge_handle_missing"
|
||||
try:
|
||||
snapshot = verifier(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK)
|
||||
except Exception:
|
||||
return "p5_b1_bridge_verifier_rejected"
|
||||
if snapshot is None:
|
||||
return "p5_b1_bridge_verifier_rejected"
|
||||
if _snapshot_field(snapshot, "sink") != BRIDGE_SERIALIZATION_SINK:
|
||||
return "p5_b1_bridge_sink_mismatch"
|
||||
if _snapshot_field(snapshot, "disposition") != "allow":
|
||||
return "p5_b1_bridge_disposition_denied"
|
||||
return ""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BRIDGE_SERIALIZATION_SINK",
|
||||
"MEMORY_RECALL_SINK",
|
||||
"P5_B1_ATTESTATION_HANDLES_ATTRIBUTE",
|
||||
"evaluate_bridge_serialization_gate",
|
||||
"evaluate_normal_recall_gate",
|
||||
]
|
||||
799
core/p5_provenance.py
普通文件
799
core/p5_provenance.py
普通文件
|
|
@ -0,0 +1,799 @@
|
|||
"""P5-B metadata-only provenance and migration planning contract.
|
||||
|
||||
This module deliberately has no dependency on MemoryRecord, Store, Service,
|
||||
Bridge, or a database. It is the small, deterministic contract that a later
|
||||
integration may use to *plan* provenance backfill and guarded rollback. It
|
||||
does not classify content, read a body, perform I/O, or grant a live recall or
|
||||
tool permission.
|
||||
|
||||
Only :func:`observed_from_companion_snapshot` can create an ``observed``
|
||||
record. It requires the frozen P3 carrier reference and a Companion
|
||||
attestation reference. Historical records are represented explicitly as
|
||||
``legacy_unresolved`` with ``unknown`` / ``T4`` provenance instead of being
|
||||
silently upgraded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict, is_dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
PROVENANCE_CONTRACT_NAME = "ops.p5.provenance.v1"
|
||||
PROVENANCE_CONTRACT_VERSION = "1.0"
|
||||
|
||||
# P3 remains owned by PrivateCompanion. This consumer duplicates only its
|
||||
# already-frozen identity, never its state machine or source classifier.
|
||||
P3_CONTRACT_NAME = "ops.context_orchestration.v1"
|
||||
P3_CONTRACT_VERSION = "1.0"
|
||||
P3_CONTRACT_FINGERPRINT = (
|
||||
"3bb7a12af05bb4d9a47cef9f31e78752cec512b90b1d50ae49feef54470974f8"
|
||||
)
|
||||
COMPANION_ATTESTATION_ISSUER = "private_companion"
|
||||
ATTESTATION_SCHEMA_VERSION = "ops.p5.attestation.v1"
|
||||
|
||||
TRUST_LEVELS = ("T0", "T1", "T2", "T3", "T4")
|
||||
PROVENANCE_STATES = ("observed", "legacy_unresolved", "owner_recovered", "invalid")
|
||||
# Ordered tuples are part of the shared cross-repository descriptor. Trust
|
||||
# must remain explicit: source kind and firewall status never imply it.
|
||||
SOURCE_KINDS = (
|
||||
"policy_config",
|
||||
"verified_authorization",
|
||||
"current_user_intent",
|
||||
"forwarded_text",
|
||||
"quoted_text",
|
||||
"vision_summary",
|
||||
"tool_output",
|
||||
"web_extract",
|
||||
"memory_recall",
|
||||
"derived_summary",
|
||||
"legacy_memory",
|
||||
"unknown",
|
||||
)
|
||||
FIREWALL_STATUSES = (
|
||||
"allowed",
|
||||
"sanitized",
|
||||
"blocked",
|
||||
"rejected",
|
||||
"quarantined",
|
||||
"unavailable",
|
||||
"unknown",
|
||||
)
|
||||
DISPOSITIONS = ("allow", "shadow_quarantine", "deny_high_risk")
|
||||
HASH_FORMAT = "sha256_hex_lower"
|
||||
_SOURCE_KIND_SET = frozenset(SOURCE_KINDS)
|
||||
_ATTESTATION_SINKS = frozenset(
|
||||
{
|
||||
"prompt_forward_quote",
|
||||
"prompt_vision_summary",
|
||||
"memory_recall",
|
||||
"bridge_serialization",
|
||||
"tool_retrieval",
|
||||
"external_export",
|
||||
"cross_user_read",
|
||||
}
|
||||
)
|
||||
|
||||
RECORD_FIELDS = (
|
||||
"contract_name",
|
||||
"contract_version",
|
||||
"contract_fingerprint",
|
||||
"memory_id",
|
||||
"source_kind",
|
||||
"source_trust",
|
||||
"firewall_status",
|
||||
"source_event_ref_hash",
|
||||
"authority_attestation_ref_hash",
|
||||
"provenance_state",
|
||||
"migration_operation_ref",
|
||||
"recovery_operation_ref",
|
||||
"record_revision",
|
||||
)
|
||||
OPERATION_FIELDS = (
|
||||
"contract_name",
|
||||
"contract_version",
|
||||
"contract_fingerprint",
|
||||
"operation_kind",
|
||||
"operation_ref_hash",
|
||||
"memory_id",
|
||||
"expected_revision",
|
||||
"before_record_digest",
|
||||
"after_record_digest",
|
||||
"before_record",
|
||||
"after_record",
|
||||
)
|
||||
|
||||
_TOKEN_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
|
||||
_MEMORY_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$")
|
||||
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
_FORBIDDEN_RAW_FIELDS = frozenset(
|
||||
{
|
||||
"body",
|
||||
"content",
|
||||
"text",
|
||||
"message",
|
||||
"raw_id",
|
||||
"source_id",
|
||||
"source_event_id",
|
||||
"event_id",
|
||||
"attestation_id",
|
||||
"authority_attestation_id",
|
||||
"request_id",
|
||||
"session_id",
|
||||
}
|
||||
)
|
||||
_SNAPSHOT_FIELDS = frozenset(
|
||||
{
|
||||
"schema_version",
|
||||
"contract_name",
|
||||
"contract_version",
|
||||
"contract_fingerprint",
|
||||
"issuer",
|
||||
"issuer_epoch",
|
||||
"p3_contract_name",
|
||||
"p3_contract_version",
|
||||
"p3_contract_fingerprint",
|
||||
"source_kind",
|
||||
"source_trust",
|
||||
"trust",
|
||||
"firewall_status",
|
||||
"source_event_ref_hash",
|
||||
"source_hash",
|
||||
"authority_attestation_ref_hash",
|
||||
"attestation_ref_hash",
|
||||
# The following are bounded, non-authoritative diagnostic metadata
|
||||
# emitted by the companion registry. They never enter a P5 record.
|
||||
"disposition",
|
||||
"reason_codes",
|
||||
"request_hash",
|
||||
"session_hash",
|
||||
"derived_hash",
|
||||
"derived_from_ref_hash",
|
||||
"provenance_state",
|
||||
"sink",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _digest(value: Mapping[str, Any]) -> str:
|
||||
return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def contract_descriptor() -> dict[str, Any]:
|
||||
"""Return the frozen, JSON-serializable P5 provenance descriptor."""
|
||||
|
||||
return {
|
||||
"contract_name": PROVENANCE_CONTRACT_NAME,
|
||||
"contract_version": PROVENANCE_CONTRACT_VERSION,
|
||||
"record_fields": list(RECORD_FIELDS),
|
||||
"provenance_states": list(PROVENANCE_STATES),
|
||||
"trust_levels": list(TRUST_LEVELS),
|
||||
"source_kinds": list(SOURCE_KINDS),
|
||||
"firewall_statuses": list(FIREWALL_STATUSES),
|
||||
"dispositions": list(DISPOSITIONS),
|
||||
"hash_format": HASH_FORMAT,
|
||||
"attestation_issuer": COMPANION_ATTESTATION_ISSUER,
|
||||
"p3_contract_name": P3_CONTRACT_NAME,
|
||||
"p3_contract_version": P3_CONTRACT_VERSION,
|
||||
}
|
||||
|
||||
|
||||
def contract_fingerprint() -> str:
|
||||
"""Return a stable descriptor fingerprint without importing Companion."""
|
||||
|
||||
return _digest(contract_descriptor())
|
||||
|
||||
|
||||
def provenance_record_digest(record: Mapping[str, Any] | None) -> str:
|
||||
"""Return a digest only for an already-valid, metadata-only P5 record."""
|
||||
|
||||
checked = validate_provenance_record(record)
|
||||
if not checked["ok"]:
|
||||
return ""
|
||||
return _digest(checked["record"])
|
||||
|
||||
|
||||
def _safe_memory_id(value: Any) -> str:
|
||||
if isinstance(value, str) and _MEMORY_ID_RE.fullmatch(value):
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _safe_hash(value: Any) -> str:
|
||||
"""Accept only a canonical stored hash (no prefix and lowercase)."""
|
||||
|
||||
if isinstance(value, str) and _SHA256_RE.fullmatch(value) and value == value.lower():
|
||||
return value
|
||||
return ""
|
||||
|
||||
|
||||
def _normalize_input_hash(value: Any) -> str:
|
||||
"""Normalize a boundary hash; persisted records remain canonical only."""
|
||||
|
||||
if not isinstance(value, str):
|
||||
return ""
|
||||
raw = value.removeprefix("sha256:")
|
||||
if _SHA256_RE.fullmatch(raw):
|
||||
return raw.lower()
|
||||
return ""
|
||||
|
||||
|
||||
def _safe_revision(value: Any) -> int | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _known_token(value: Any, known: frozenset[str] | tuple[str, ...]) -> bool:
|
||||
return isinstance(value, str) and value in known
|
||||
|
||||
|
||||
def _has_forbidden_raw_field(value: Mapping[str, Any]) -> bool:
|
||||
return any(key in value for key in _FORBIDDEN_RAW_FIELDS)
|
||||
|
||||
|
||||
def _field_set_is_exact(value: Mapping[str, Any], fields: tuple[str, ...]) -> bool:
|
||||
return set(value) == set(fields)
|
||||
|
||||
|
||||
def _error_result(*codes: str) -> dict[str, Any]:
|
||||
# Error codes are static, so they neither echo a hostile value nor retain
|
||||
# prose/raw IDs supplied by a caller.
|
||||
return {"ok": False, "error_codes": sorted(set(codes))}
|
||||
|
||||
|
||||
def _blank_record(memory_id: Any = "", *, revision: int = 0, state: str = "invalid") -> dict[str, Any]:
|
||||
return {
|
||||
"contract_name": PROVENANCE_CONTRACT_NAME,
|
||||
"contract_version": PROVENANCE_CONTRACT_VERSION,
|
||||
"contract_fingerprint": contract_fingerprint(),
|
||||
"memory_id": _safe_memory_id(memory_id),
|
||||
"source_kind": "unknown",
|
||||
"source_trust": "T4",
|
||||
"firewall_status": "unknown",
|
||||
"source_event_ref_hash": "",
|
||||
"authority_attestation_ref_hash": "",
|
||||
"provenance_state": state,
|
||||
"migration_operation_ref": "",
|
||||
"recovery_operation_ref": "",
|
||||
"record_revision": revision if _safe_revision(revision) is not None else 0,
|
||||
}
|
||||
|
||||
|
||||
def validate_provenance_record(record: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""Validate a persisted P5 record without coercion or content parsing.
|
||||
|
||||
The result contains only static error codes. In particular it never
|
||||
returns an invalid input field, a source event ID, or message text.
|
||||
"""
|
||||
|
||||
if not isinstance(record, Mapping):
|
||||
return _error_result("record_not_mapping")
|
||||
errors: list[str] = []
|
||||
if _has_forbidden_raw_field(record):
|
||||
errors.append("forbidden_raw_field")
|
||||
if not _field_set_is_exact(record, RECORD_FIELDS):
|
||||
errors.append("record_fields_mismatch")
|
||||
if record.get("contract_name") != PROVENANCE_CONTRACT_NAME:
|
||||
errors.append("contract_name_mismatch")
|
||||
if record.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
|
||||
errors.append("contract_version_mismatch")
|
||||
if record.get("contract_fingerprint") != contract_fingerprint():
|
||||
errors.append("contract_fingerprint_mismatch")
|
||||
if not _safe_memory_id(record.get("memory_id")):
|
||||
errors.append("memory_id_invalid")
|
||||
if not _known_token(record.get("source_kind"), SOURCE_KINDS):
|
||||
errors.append("source_kind_invalid")
|
||||
if not _known_token(record.get("source_trust"), TRUST_LEVELS):
|
||||
errors.append("source_trust_invalid")
|
||||
if not _known_token(record.get("firewall_status"), FIREWALL_STATUSES):
|
||||
errors.append("firewall_status_invalid")
|
||||
if not _known_token(record.get("provenance_state"), PROVENANCE_STATES):
|
||||
errors.append("provenance_state_invalid")
|
||||
if _safe_revision(record.get("record_revision")) is None:
|
||||
errors.append("record_revision_invalid")
|
||||
|
||||
source_hash_value = record.get("source_event_ref_hash")
|
||||
authority_hash_value = record.get("authority_attestation_ref_hash")
|
||||
source_hash = _safe_hash(source_hash_value)
|
||||
authority_hash = _safe_hash(authority_hash_value)
|
||||
if source_hash_value != "" and not source_hash:
|
||||
errors.append("source_event_ref_hash_invalid")
|
||||
if authority_hash_value != "" and not authority_hash:
|
||||
errors.append("authority_attestation_ref_hash_invalid")
|
||||
migration_ref = record.get("migration_operation_ref")
|
||||
recovery_ref = record.get("recovery_operation_ref")
|
||||
if migration_ref != "" and not _safe_hash(migration_ref):
|
||||
errors.append("migration_operation_ref_invalid")
|
||||
if recovery_ref != "" and not _safe_hash(recovery_ref):
|
||||
errors.append("recovery_operation_ref_invalid")
|
||||
|
||||
state = record.get("provenance_state")
|
||||
if state == "legacy_unresolved":
|
||||
if (record.get("source_kind"), record.get("source_trust"), record.get("firewall_status")) != (
|
||||
"unknown",
|
||||
"T4",
|
||||
"unknown",
|
||||
):
|
||||
errors.append("legacy_defaults_required")
|
||||
if source_hash_value != "" or authority_hash_value != "":
|
||||
errors.append("legacy_hashes_forbidden")
|
||||
if not isinstance(migration_ref, str) or not isinstance(recovery_ref, str):
|
||||
errors.append("operation_ref_type_invalid")
|
||||
elif state == "observed" or state == "owner_recovered":
|
||||
if record.get("source_kind") == "unknown":
|
||||
errors.append("observed_source_kind_required")
|
||||
if not source_hash or not authority_hash:
|
||||
errors.append("attestation_hashes_required")
|
||||
if source_hash and authority_hash and source_hash == authority_hash:
|
||||
errors.append("attestation_hashes_conflict")
|
||||
if state == "owner_recovered" and not _safe_hash(recovery_ref):
|
||||
errors.append("recovery_operation_ref_required")
|
||||
elif state == "invalid":
|
||||
# Invalid is an explicit quarantine representation, never an accepted
|
||||
# migration target. Its safe shape is still useful for caller logs.
|
||||
if source_hash or authority_hash:
|
||||
errors.append("invalid_hashes_forbidden")
|
||||
if errors:
|
||||
return _error_result(*errors)
|
||||
return {"ok": True, "record": deepcopy(dict(record))}
|
||||
|
||||
|
||||
def legacy_unresolved(memory_id: str, *, record_revision: int = 0) -> dict[str, Any]:
|
||||
"""Represent a historical item without pretending it has known trust."""
|
||||
|
||||
record = _blank_record(memory_id, revision=record_revision, state="legacy_unresolved")
|
||||
if not record["memory_id"]:
|
||||
return _blank_record(state="invalid")
|
||||
return record
|
||||
|
||||
|
||||
def _snapshot_hash(snapshot: Mapping[str, Any], *keys: str) -> tuple[str, str]:
|
||||
present = [(key, snapshot.get(key)) for key in keys if key in snapshot]
|
||||
if not present:
|
||||
return "missing", ""
|
||||
values = [_normalize_input_hash(value) for _, value in present]
|
||||
if not all(values):
|
||||
return "invalid", ""
|
||||
if len(set(values)) != 1:
|
||||
return "conflict", ""
|
||||
return "present", values[0]
|
||||
|
||||
|
||||
def _snapshot_label(
|
||||
snapshot: Mapping[str, Any], known: frozenset[str] | tuple[str, ...], *keys: str
|
||||
) -> tuple[str, str]:
|
||||
present = [(key, snapshot.get(key)) for key in keys if key in snapshot]
|
||||
if not present:
|
||||
return "missing", ""
|
||||
values = [value for _, value in present]
|
||||
if not all(_known_token(value, known) for value in values):
|
||||
return "invalid", ""
|
||||
if len(set(values)) != 1:
|
||||
return "conflict", ""
|
||||
return "present", values[0]
|
||||
|
||||
|
||||
def _snapshot_diagnostics_valid(snapshot: Mapping[str, Any]) -> bool:
|
||||
"""Allow only bounded, non-prose diagnostics that are not persisted."""
|
||||
|
||||
# A disposition is an attestation-side decision label, distinct from the
|
||||
# upstream P2/P3 firewall status that is persisted in the record.
|
||||
if "disposition" not in snapshot:
|
||||
return False
|
||||
disposition = snapshot["disposition"]
|
||||
if not _known_token(disposition, DISPOSITIONS):
|
||||
return False
|
||||
if "reason_codes" in snapshot:
|
||||
codes = snapshot["reason_codes"]
|
||||
if not isinstance(codes, (list, tuple)) or len(codes) > 16:
|
||||
return False
|
||||
if not all(isinstance(code, str) and _TOKEN_RE.fullmatch(code) for code in codes):
|
||||
return False
|
||||
for key in ("request_hash", "session_hash"):
|
||||
if key in snapshot and not _normalize_input_hash(snapshot[key]):
|
||||
return False
|
||||
derived_values = [
|
||||
snapshot[key] for key in ("derived_from_ref_hash", "derived_hash") if key in snapshot
|
||||
]
|
||||
if derived_values:
|
||||
if any(value != "" and not _normalize_input_hash(value) for value in derived_values):
|
||||
return False
|
||||
normalized = [_normalize_input_hash(value) for value in derived_values if value != ""]
|
||||
if len(set(normalized)) > 1:
|
||||
return False
|
||||
if snapshot.get("provenance_state") != "observed":
|
||||
return False
|
||||
epoch = snapshot.get("issuer_epoch")
|
||||
if not _normalize_input_hash(epoch):
|
||||
return False
|
||||
sink = snapshot.get("sink")
|
||||
if not _known_token(sink, _ATTESTATION_SINKS):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _snapshot_mapping(snapshot: Any) -> Mapping[str, Any] | None:
|
||||
"""Accept the immutable Companion dataclass without importing Companion."""
|
||||
|
||||
if isinstance(snapshot, Mapping):
|
||||
return snapshot
|
||||
if is_dataclass(snapshot):
|
||||
value = asdict(snapshot)
|
||||
return value if isinstance(value, Mapping) else None
|
||||
return None
|
||||
|
||||
|
||||
def observed_from_companion_snapshot(
|
||||
memory_id: str, snapshot: Any, *, record_revision: int = 0
|
||||
) -> dict[str, Any]:
|
||||
"""Convert only a valid Companion-attested snapshot into ``observed``.
|
||||
|
||||
This function accepts hashes and bounded labels only. A raw source event
|
||||
ID, text body, or arbitrary P3 fingerprint rejects conversion and yields a
|
||||
safe ``invalid`` record rather than an observed one.
|
||||
"""
|
||||
|
||||
safe_id = _safe_memory_id(memory_id)
|
||||
revision = _safe_revision(record_revision)
|
||||
snapshot_mapping = _snapshot_mapping(snapshot)
|
||||
if not safe_id or revision is None or snapshot_mapping is None:
|
||||
return _blank_record(safe_id, revision=revision or 0)
|
||||
if (
|
||||
_has_forbidden_raw_field(snapshot_mapping)
|
||||
or not set(snapshot_mapping).issubset(_SNAPSHOT_FIELDS)
|
||||
or not _snapshot_diagnostics_valid(snapshot_mapping)
|
||||
):
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("schema_version") != ATTESTATION_SCHEMA_VERSION:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("issuer") != COMPANION_ATTESTATION_ISSUER:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("contract_name") != PROVENANCE_CONTRACT_NAME:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("contract_fingerprint") != contract_fingerprint():
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("p3_contract_name") != P3_CONTRACT_NAME:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("p3_contract_version") != P3_CONTRACT_VERSION:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
if snapshot_mapping.get("p3_contract_fingerprint") != P3_CONTRACT_FINGERPRINT:
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
|
||||
source_status, source_hash = _snapshot_hash(
|
||||
snapshot_mapping, "source_event_ref_hash", "source_hash"
|
||||
)
|
||||
authority_status, authority_hash = _snapshot_hash(
|
||||
snapshot_mapping, "authority_attestation_ref_hash", "attestation_ref_hash"
|
||||
)
|
||||
trust_status, source_trust = _snapshot_label(snapshot_mapping, TRUST_LEVELS, "source_trust", "trust")
|
||||
source_kind = snapshot_mapping.get("source_kind")
|
||||
firewall_status = snapshot_mapping.get("firewall_status")
|
||||
if (
|
||||
source_status != "present"
|
||||
or authority_status != "present"
|
||||
or trust_status != "present"
|
||||
or source_hash == authority_hash
|
||||
or not _known_token(source_kind, _SOURCE_KIND_SET - {"unknown"})
|
||||
or not _known_token(source_trust, TRUST_LEVELS)
|
||||
or not _known_token(firewall_status, FIREWALL_STATUSES)
|
||||
):
|
||||
return _blank_record(safe_id, revision=revision)
|
||||
|
||||
record = _blank_record(safe_id, revision=revision, state="observed")
|
||||
record.update(
|
||||
{
|
||||
"source_kind": source_kind,
|
||||
"source_trust": source_trust,
|
||||
"firewall_status": firewall_status,
|
||||
"source_event_ref_hash": source_hash,
|
||||
"authority_attestation_ref_hash": authority_hash,
|
||||
}
|
||||
)
|
||||
return record if validate_provenance_record(record)["ok"] else _blank_record(safe_id, revision=revision)
|
||||
|
||||
|
||||
def _record_or_none(value: Any) -> dict[str, Any] | None:
|
||||
if value is None:
|
||||
return None
|
||||
result = validate_provenance_record(value if isinstance(value, Mapping) else None)
|
||||
return deepcopy(result["record"]) if result["ok"] else None
|
||||
|
||||
|
||||
def _operation(
|
||||
*,
|
||||
operation_kind: str,
|
||||
operation_ref_hash: str,
|
||||
before_record: Mapping[str, Any] | None,
|
||||
after_record: Mapping[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
before = _record_or_none(before_record)
|
||||
after = _record_or_none(after_record)
|
||||
reference = _safe_hash(operation_ref_hash)
|
||||
if not _known_token(operation_kind, ("migration", "recovery")) or not reference or after is None:
|
||||
return None
|
||||
if before is not None and before["memory_id"] != after["memory_id"]:
|
||||
return None
|
||||
expected_revision = before["record_revision"] if before is not None else after["record_revision"] - 1
|
||||
if expected_revision < 0 or after["record_revision"] != expected_revision + 1:
|
||||
return None
|
||||
operation = {
|
||||
"contract_name": PROVENANCE_CONTRACT_NAME,
|
||||
"contract_version": PROVENANCE_CONTRACT_VERSION,
|
||||
"contract_fingerprint": contract_fingerprint(),
|
||||
"operation_kind": operation_kind,
|
||||
"operation_ref_hash": reference,
|
||||
"memory_id": after["memory_id"],
|
||||
"expected_revision": expected_revision,
|
||||
"before_record_digest": _digest(before) if before is not None else "",
|
||||
"after_record_digest": _digest(after),
|
||||
"before_record": before,
|
||||
"after_record": after,
|
||||
}
|
||||
return operation if validate_operation(operation)["ok"] else None
|
||||
|
||||
|
||||
def validate_operation(operation: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""Validate an I/O-free migration/recovery operation and its CAS guards."""
|
||||
|
||||
if not isinstance(operation, Mapping):
|
||||
return _error_result("operation_not_mapping")
|
||||
errors: list[str] = []
|
||||
if _has_forbidden_raw_field(operation):
|
||||
errors.append("forbidden_raw_field")
|
||||
if not _field_set_is_exact(operation, OPERATION_FIELDS):
|
||||
errors.append("operation_fields_mismatch")
|
||||
if operation.get("contract_name") != PROVENANCE_CONTRACT_NAME:
|
||||
errors.append("contract_name_mismatch")
|
||||
if operation.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
|
||||
errors.append("contract_version_mismatch")
|
||||
if operation.get("contract_fingerprint") != contract_fingerprint():
|
||||
errors.append("contract_fingerprint_mismatch")
|
||||
kind = operation.get("operation_kind")
|
||||
if not _known_token(kind, ("migration", "recovery")):
|
||||
errors.append("operation_kind_invalid")
|
||||
if not _safe_hash(operation.get("operation_ref_hash")):
|
||||
errors.append("operation_ref_invalid")
|
||||
if not _safe_memory_id(operation.get("memory_id")):
|
||||
errors.append("memory_id_invalid")
|
||||
expected_revision = _safe_revision(operation.get("expected_revision"))
|
||||
if expected_revision is None:
|
||||
errors.append("expected_revision_invalid")
|
||||
|
||||
before = operation.get("before_record")
|
||||
after = operation.get("after_record")
|
||||
before_record = _record_or_none(before)
|
||||
after_record = _record_or_none(after)
|
||||
if before is not None and before_record is None:
|
||||
errors.append("before_record_invalid")
|
||||
if after_record is None:
|
||||
errors.append("after_record_invalid")
|
||||
if before_record is not None and operation.get("before_record_digest") != _digest(before_record):
|
||||
errors.append("before_digest_mismatch")
|
||||
if before is None and operation.get("before_record_digest") != "":
|
||||
errors.append("before_digest_required_empty")
|
||||
if after_record is not None and operation.get("after_record_digest") != _digest(after_record):
|
||||
errors.append("after_digest_mismatch")
|
||||
if after_record is not None:
|
||||
if operation.get("memory_id") != after_record["memory_id"]:
|
||||
errors.append("memory_id_mismatch")
|
||||
if expected_revision is not None and after_record["record_revision"] != expected_revision + 1:
|
||||
errors.append("revision_progression_invalid")
|
||||
required_ref = "migration_operation_ref" if kind == "migration" else "recovery_operation_ref"
|
||||
if after_record.get(required_ref) != operation.get("operation_ref_hash"):
|
||||
errors.append("operation_ref_not_bound")
|
||||
if before_record is not None and after_record is not None:
|
||||
if before_record["memory_id"] != after_record["memory_id"]:
|
||||
errors.append("before_after_memory_mismatch")
|
||||
if expected_revision is not None and before_record["record_revision"] != expected_revision:
|
||||
errors.append("before_revision_mismatch")
|
||||
if errors:
|
||||
return _error_result(*errors)
|
||||
return {"ok": True, "operation": deepcopy(dict(operation))}
|
||||
|
||||
|
||||
def plan_legacy_migration(
|
||||
records: Iterable[Mapping[str, Any]], *, operation_ref_hash: str
|
||||
) -> dict[str, Any]:
|
||||
"""Build a read-only P5 legacy-backfill preview; never write or mutate.
|
||||
|
||||
``records`` are intentionally tiny metadata snapshots containing only a
|
||||
``memory_id`` and ``record_revision``. A valid existing P5 record may be
|
||||
supplied instead and is skipped; Store integration will decide how to
|
||||
obtain those snapshots in a future phase.
|
||||
"""
|
||||
|
||||
reference = _safe_hash(operation_ref_hash)
|
||||
result: dict[str, Any] = {
|
||||
"mode": "preview",
|
||||
"readonly": True,
|
||||
"write_count": 0,
|
||||
"operation_ref_hash": reference,
|
||||
"operations": [],
|
||||
"skipped": [],
|
||||
"error_codes": [],
|
||||
}
|
||||
if not reference:
|
||||
result["error_codes"].append("operation_ref_invalid")
|
||||
return result
|
||||
if isinstance(records, (str, bytes)) or not isinstance(records, Iterable):
|
||||
result["error_codes"].append("records_not_iterable")
|
||||
return result
|
||||
|
||||
for candidate in records:
|
||||
if not isinstance(candidate, Mapping) or _has_forbidden_raw_field(candidate):
|
||||
result["error_codes"].append("candidate_invalid")
|
||||
continue
|
||||
existing = _record_or_none(candidate)
|
||||
if existing is not None:
|
||||
result["skipped"].append({"memory_id": existing["memory_id"], "reason_code": "already_provenanced"})
|
||||
continue
|
||||
allowed_base_fields = {"memory_id", "record_revision"}
|
||||
if set(candidate) != allowed_base_fields:
|
||||
result["error_codes"].append("candidate_fields_mismatch")
|
||||
continue
|
||||
memory_id = _safe_memory_id(candidate.get("memory_id"))
|
||||
revision = _safe_revision(candidate.get("record_revision"))
|
||||
if not memory_id or revision is None:
|
||||
result["error_codes"].append("candidate_invalid")
|
||||
continue
|
||||
after = legacy_unresolved(memory_id, record_revision=revision + 1)
|
||||
after["migration_operation_ref"] = reference
|
||||
operation = _operation(
|
||||
operation_kind="migration",
|
||||
operation_ref_hash=reference,
|
||||
before_record=None,
|
||||
after_record=after,
|
||||
)
|
||||
if operation is None:
|
||||
result["error_codes"].append("operation_invalid")
|
||||
else:
|
||||
result["operations"].append(operation)
|
||||
result["error_codes"] = sorted(set(result["error_codes"]))
|
||||
return result
|
||||
|
||||
|
||||
def _minimal_current(current: Mapping[str, Any] | None) -> tuple[str, int | None, dict[str, Any] | None, str]:
|
||||
"""Extract a safe CAS view; never accept a raw current-record payload."""
|
||||
|
||||
if not isinstance(current, Mapping) or _has_forbidden_raw_field(current):
|
||||
return "", None, None, "current_invalid"
|
||||
p5 = _record_or_none(current)
|
||||
if p5 is not None:
|
||||
return p5["memory_id"], p5["record_revision"], p5, ""
|
||||
if set(current) != {"memory_id", "record_revision"}:
|
||||
return "", None, None, "current_fields_mismatch"
|
||||
return _safe_memory_id(current.get("memory_id")), _safe_revision(current.get("record_revision")), None, ""
|
||||
|
||||
|
||||
def apply_planned_operation(
|
||||
current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None
|
||||
) -> dict[str, Any]:
|
||||
"""Project a guarded apply result without touching storage.
|
||||
|
||||
Applying the same operation twice is idempotent. A different record or
|
||||
revision returns a conflict instead of overwriting a concurrent update.
|
||||
"""
|
||||
|
||||
checked = validate_operation(operation)
|
||||
if not checked["ok"]:
|
||||
return {"ok": False, "mode": "apply", "status": "invalid_operation", "error_codes": checked["error_codes"]}
|
||||
op = checked["operation"]
|
||||
memory_id, revision, current_record, error = _minimal_current(current)
|
||||
if error or not memory_id or revision is None:
|
||||
return {"ok": False, "mode": "apply", "status": "invalid_current", "error_codes": [error or "current_invalid"]}
|
||||
after = op["after_record"]
|
||||
if current_record is not None and _digest(current_record) == op["after_record_digest"]:
|
||||
return {"ok": True, "mode": "apply", "status": "idempotent", "record": deepcopy(after), "changed": False}
|
||||
if memory_id != op["memory_id"] or revision != op["expected_revision"]:
|
||||
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["revision_or_memory_mismatch"]}
|
||||
expected_before = op["before_record"]
|
||||
if expected_before is None:
|
||||
if current_record is not None:
|
||||
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["unexpected_existing_provenance"]}
|
||||
elif current_record is None or _digest(current_record) != op["before_record_digest"]:
|
||||
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["before_record_mismatch"]}
|
||||
return {"ok": True, "mode": "apply", "status": "applied", "record": deepcopy(after), "changed": True}
|
||||
|
||||
|
||||
def rollback_planned_operation(
|
||||
current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None
|
||||
) -> dict[str, Any]:
|
||||
"""Project a CAS-protected rollback without overwriting later changes."""
|
||||
|
||||
checked = validate_operation(operation)
|
||||
if not checked["ok"]:
|
||||
return {"ok": False, "mode": "rollback", "status": "invalid_operation", "error_codes": checked["error_codes"]}
|
||||
op = checked["operation"]
|
||||
memory_id, revision, current_record, error = _minimal_current(current)
|
||||
if error or current_record is None or revision is None:
|
||||
return {"ok": False, "mode": "rollback", "status": "invalid_current", "error_codes": [error or "current_invalid"]}
|
||||
if (
|
||||
memory_id != op["memory_id"]
|
||||
or revision != op["after_record"]["record_revision"]
|
||||
or _digest(current_record) != op["after_record_digest"]
|
||||
):
|
||||
return {"ok": False, "mode": "rollback", "status": "revision_conflict", "error_codes": ["concurrent_update_protected"]}
|
||||
before = op["before_record"]
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "rollback",
|
||||
"status": "rolled_back",
|
||||
"record": deepcopy(before) if before is not None else None,
|
||||
"removed": before is None,
|
||||
"changed": True,
|
||||
}
|
||||
|
||||
|
||||
def plan_owner_recovery(
|
||||
current: Mapping[str, Any],
|
||||
snapshot: Mapping[str, Any],
|
||||
*,
|
||||
recovery_operation_ref_hash: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Plan, but do not apply, a Companion-attested owner recovery.
|
||||
|
||||
Recovery is intentionally limited to a valid ``legacy_unresolved`` record.
|
||||
A later owner-authority workflow must supply the Companion snapshot; this
|
||||
pure contract has no owner command, storage, or authorization side effect.
|
||||
"""
|
||||
|
||||
existing = _record_or_none(current)
|
||||
reference = _safe_hash(recovery_operation_ref_hash)
|
||||
if existing is None or existing["provenance_state"] != "legacy_unresolved" or not reference:
|
||||
return {"ok": False, "mode": "preview", "error_codes": ["recovery_precondition_failed"]}
|
||||
after = observed_from_companion_snapshot(
|
||||
existing["memory_id"], snapshot, record_revision=existing["record_revision"] + 1
|
||||
)
|
||||
if after["provenance_state"] != "observed":
|
||||
return {"ok": False, "mode": "preview", "error_codes": ["companion_snapshot_invalid"]}
|
||||
after["provenance_state"] = "owner_recovered"
|
||||
after["migration_operation_ref"] = existing["migration_operation_ref"]
|
||||
after["recovery_operation_ref"] = reference
|
||||
operation = _operation(
|
||||
operation_kind="recovery",
|
||||
operation_ref_hash=reference,
|
||||
before_record=existing,
|
||||
after_record=after,
|
||||
)
|
||||
if operation is None:
|
||||
return {"ok": False, "mode": "preview", "error_codes": ["operation_invalid"]}
|
||||
return {"ok": True, "mode": "preview", "readonly": True, "write_count": 0, "operation": operation}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"COMPANION_ATTESTATION_ISSUER",
|
||||
"DISPOSITIONS",
|
||||
"FIREWALL_STATUSES",
|
||||
"HASH_FORMAT",
|
||||
"P3_CONTRACT_FINGERPRINT",
|
||||
"P3_CONTRACT_NAME",
|
||||
"P3_CONTRACT_VERSION",
|
||||
"PROVENANCE_CONTRACT_NAME",
|
||||
"PROVENANCE_CONTRACT_VERSION",
|
||||
"PROVENANCE_STATES",
|
||||
"SOURCE_KINDS",
|
||||
"TRUST_LEVELS",
|
||||
"apply_planned_operation",
|
||||
"contract_descriptor",
|
||||
"contract_fingerprint",
|
||||
"legacy_unresolved",
|
||||
"observed_from_companion_snapshot",
|
||||
"plan_legacy_migration",
|
||||
"plan_owner_recovery",
|
||||
"provenance_record_digest",
|
||||
"rollback_planned_operation",
|
||||
"validate_operation",
|
||||
"validate_provenance_record",
|
||||
]
|
||||
468
core/p5_provenance_persistence.py
普通文件
468
core/p5_provenance_persistence.py
普通文件
|
|
@ -0,0 +1,468 @@
|
|||
"""P5-B0 additive SQLite persistence and recoverable legacy backfill.
|
||||
|
||||
This is deliberately a storage/migration boundary. It does not consume a
|
||||
Companion attestation, change memory capture, alter retrieval, or evaluate a
|
||||
live sink. The only records it creates are explicit ``legacy_unresolved``
|
||||
records, so existing memory columns can never be used to infer source trust.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
from .models import utc_now
|
||||
from .p5_provenance import (
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
contract_fingerprint,
|
||||
legacy_unresolved,
|
||||
provenance_record_digest,
|
||||
validate_provenance_record,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .store import MemoryStore
|
||||
|
||||
|
||||
P5_PERSISTENCE_SCHEMA_VERSION = 1
|
||||
_OPERATION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$")
|
||||
_RECORD_COLUMNS = (
|
||||
"memory_id",
|
||||
"contract_name",
|
||||
"contract_version",
|
||||
"contract_fingerprint",
|
||||
"source_kind",
|
||||
"source_trust",
|
||||
"firewall_status",
|
||||
"source_event_ref_hash",
|
||||
"authority_attestation_ref_hash",
|
||||
"provenance_state",
|
||||
"migration_operation_ref",
|
||||
"recovery_operation_ref",
|
||||
"record_revision",
|
||||
"record_digest",
|
||||
)
|
||||
|
||||
|
||||
def ensure_p5_provenance_persistence_schema(connection: sqlite3.Connection) -> None:
|
||||
"""Create only additive P5-B0 tables and indexes.
|
||||
|
||||
This function neither scans memories nor creates provenance records; that
|
||||
work is possible only through the explicit migration apply operation.
|
||||
"""
|
||||
|
||||
connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS p5_provenance_records (
|
||||
memory_id TEXT PRIMARY KEY,
|
||||
contract_name TEXT NOT NULL,
|
||||
contract_version TEXT NOT NULL,
|
||||
contract_fingerprint TEXT NOT NULL,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_trust TEXT NOT NULL,
|
||||
firewall_status TEXT NOT NULL,
|
||||
source_event_ref_hash TEXT NOT NULL DEFAULT '',
|
||||
authority_attestation_ref_hash TEXT NOT NULL DEFAULT '',
|
||||
provenance_state TEXT NOT NULL,
|
||||
migration_operation_ref TEXT NOT NULL DEFAULT '',
|
||||
recovery_operation_ref TEXT NOT NULL DEFAULT '',
|
||||
record_revision INTEGER NOT NULL,
|
||||
record_digest TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(memory_id) REFERENCES memories(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_p5_provenance_state
|
||||
ON p5_provenance_records(provenance_state, source_trust, firewall_status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS p5_provenance_migration_runs (
|
||||
operation_id TEXT PRIMARY KEY,
|
||||
operation_ref_hash TEXT NOT NULL,
|
||||
contract_name TEXT NOT NULL,
|
||||
contract_version TEXT NOT NULL,
|
||||
contract_fingerprint TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
requested_limit INTEGER NOT NULL DEFAULT 0,
|
||||
candidate_count INTEGER NOT NULL DEFAULT 0,
|
||||
applied_count INTEGER NOT NULL DEFAULT 0,
|
||||
rollback_conflict_count INTEGER NOT NULL DEFAULT 0,
|
||||
backup_path TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
completed_at TEXT NOT NULL DEFAULT '',
|
||||
rolled_back_at TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS p5_provenance_migration_backups (
|
||||
operation_id TEXT NOT NULL,
|
||||
memory_id TEXT NOT NULL,
|
||||
before_exists INTEGER NOT NULL,
|
||||
before_record_json TEXT NOT NULL DEFAULT '',
|
||||
before_digest TEXT NOT NULL DEFAULT '',
|
||||
before_revision INTEGER NOT NULL DEFAULT 0,
|
||||
after_record_json TEXT NOT NULL,
|
||||
after_digest TEXT NOT NULL,
|
||||
after_revision INTEGER NOT NULL,
|
||||
apply_state TEXT NOT NULL DEFAULT 'prepared',
|
||||
rollback_state TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at TEXT NOT NULL,
|
||||
applied_at TEXT NOT NULL DEFAULT '',
|
||||
rolled_back_at TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(operation_id, memory_id),
|
||||
FOREIGN KEY(operation_id) REFERENCES p5_provenance_migration_runs(operation_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_p5_provenance_backup_operation
|
||||
ON p5_provenance_migration_backups(operation_id, rollback_state, memory_id);
|
||||
"""
|
||||
)
|
||||
run_columns = {
|
||||
str(row[1])
|
||||
for row in connection.execute("PRAGMA table_info(p5_provenance_migration_runs)").fetchall()
|
||||
}
|
||||
if "backup_path" not in run_columns:
|
||||
connection.execute(
|
||||
"ALTER TABLE p5_provenance_migration_runs ADD COLUMN backup_path TEXT NOT NULL DEFAULT ''"
|
||||
)
|
||||
|
||||
|
||||
def _canonical_json(value: Mapping[str, Any]) -> str:
|
||||
return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _operation_ref(operation_id: str) -> str:
|
||||
return hashlib.sha256(
|
||||
f"{PROVENANCE_CONTRACT_NAME}:migration:{operation_id}".encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _valid_operation_id(value: Any) -> str:
|
||||
return value if isinstance(value, str) and _OPERATION_ID_RE.fullmatch(value) else ""
|
||||
|
||||
|
||||
def _limit(value: Any) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
return 0
|
||||
return min(value, 5000)
|
||||
|
||||
|
||||
def _record_from_row(row: sqlite3.Row) -> dict[str, Any]:
|
||||
return {
|
||||
"contract_name": row["contract_name"],
|
||||
"contract_version": row["contract_version"],
|
||||
"contract_fingerprint": row["contract_fingerprint"],
|
||||
"memory_id": row["memory_id"],
|
||||
"source_kind": row["source_kind"],
|
||||
"source_trust": row["source_trust"],
|
||||
"firewall_status": row["firewall_status"],
|
||||
"source_event_ref_hash": row["source_event_ref_hash"],
|
||||
"authority_attestation_ref_hash": row["authority_attestation_ref_hash"],
|
||||
"provenance_state": row["provenance_state"],
|
||||
"migration_operation_ref": row["migration_operation_ref"],
|
||||
"recovery_operation_ref": row["recovery_operation_ref"],
|
||||
"record_revision": int(row["record_revision"]),
|
||||
}
|
||||
|
||||
|
||||
def _insert_record(connection: sqlite3.Connection, record: Mapping[str, Any], *, digest: str, now: str) -> None:
|
||||
values = {
|
||||
"memory_id": record["memory_id"],
|
||||
"contract_name": record["contract_name"],
|
||||
"contract_version": record["contract_version"],
|
||||
"contract_fingerprint": record["contract_fingerprint"],
|
||||
"source_kind": record["source_kind"],
|
||||
"source_trust": record["source_trust"],
|
||||
"firewall_status": record["firewall_status"],
|
||||
"source_event_ref_hash": record["source_event_ref_hash"],
|
||||
"authority_attestation_ref_hash": record["authority_attestation_ref_hash"],
|
||||
"provenance_state": record["provenance_state"],
|
||||
"migration_operation_ref": record["migration_operation_ref"],
|
||||
"recovery_operation_ref": record["recovery_operation_ref"],
|
||||
"record_revision": record["record_revision"],
|
||||
"record_digest": digest,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO p5_provenance_records(
|
||||
memory_id, contract_name, contract_version, contract_fingerprint,
|
||||
source_kind, source_trust, firewall_status, source_event_ref_hash,
|
||||
authority_attestation_ref_hash, provenance_state,
|
||||
migration_operation_ref, recovery_operation_ref, record_revision,
|
||||
record_digest, created_at, updated_at
|
||||
) VALUES(
|
||||
:memory_id, :contract_name, :contract_version, :contract_fingerprint,
|
||||
:source_kind, :source_trust, :firewall_status, :source_event_ref_hash,
|
||||
:authority_attestation_ref_hash, :provenance_state,
|
||||
:migration_operation_ref, :recovery_operation_ref, :record_revision,
|
||||
:record_digest, :created_at, :updated_at
|
||||
)
|
||||
""",
|
||||
values,
|
||||
)
|
||||
|
||||
|
||||
class P5ProvenanceMigration:
|
||||
"""Recoverable B0 legacy provenance backfill over an existing MemoryStore."""
|
||||
|
||||
def __init__(self, store: MemoryStore):
|
||||
self.store = store
|
||||
|
||||
async def preview(self, *, limit: int = 5000) -> dict[str, Any]:
|
||||
return await self.store._run_recoverable_database_operation(self._preview_sync, limit)
|
||||
|
||||
def _preview_sync(self, limit: int) -> dict[str, Any]:
|
||||
bounded_limit = _limit(limit)
|
||||
if not bounded_limit:
|
||||
return {"mode": "preview", "ok": False, "readonly": True, "error_code": "limit_invalid", "count": 0}
|
||||
with self.store._lock:
|
||||
rows = self.store._conn.execute(
|
||||
"""
|
||||
SELECT m.id
|
||||
FROM memories AS m
|
||||
LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id
|
||||
WHERE p.memory_id IS NULL
|
||||
ORDER BY m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(bounded_limit,),
|
||||
).fetchall()
|
||||
return {
|
||||
"mode": "preview",
|
||||
"ok": True,
|
||||
"readonly": True,
|
||||
"contract_name": PROVENANCE_CONTRACT_NAME,
|
||||
"contract_version": PROVENANCE_CONTRACT_VERSION,
|
||||
"contract_fingerprint": contract_fingerprint(),
|
||||
"count": len(rows),
|
||||
"write_count": 0,
|
||||
}
|
||||
|
||||
async def apply(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
|
||||
return await self.store._run_recoverable_database_operation(self._apply_sync, operation_id, limit)
|
||||
|
||||
def _apply_sync(self, operation_id: str, limit: int) -> dict[str, Any]:
|
||||
clean_operation_id = _valid_operation_id(operation_id)
|
||||
bounded_limit = _limit(limit)
|
||||
if not clean_operation_id:
|
||||
return {"mode": "apply", "ok": False, "operation_id": "", "error_code": "operation_id_required", "changed": 0}
|
||||
if not bounded_limit:
|
||||
return {"mode": "apply", "ok": False, "operation_id": clean_operation_id, "error_code": "limit_invalid", "changed": 0}
|
||||
reference = _operation_ref(clean_operation_id)
|
||||
with self.store._lock:
|
||||
existing = self.store._conn.execute(
|
||||
"SELECT state, applied_count, backup_path FROM p5_provenance_migration_runs WHERE operation_id=?",
|
||||
(clean_operation_id,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
if existing["state"] == "completed":
|
||||
return {
|
||||
"mode": "apply", "ok": True, "operation_id": clean_operation_id,
|
||||
"changed": 0, "deduplicated": True, "backup_count": int(existing["applied_count"] or 0),
|
||||
"backup_path": str(existing["backup_path"] or ""),
|
||||
}
|
||||
return {
|
||||
"mode": "apply", "ok": False, "operation_id": clean_operation_id,
|
||||
"error_code": "operation_not_reapplicable", "changed": 0,
|
||||
}
|
||||
try:
|
||||
# This commits only prior state and creates the physical SQLite
|
||||
# image before the first B0 mutation transaction is opened.
|
||||
backup_path = str(self.store.backup(f".before_p5_provenance_b0.{clean_operation_id}"))
|
||||
except Exception:
|
||||
return {
|
||||
"mode": "apply", "ok": False, "operation_id": clean_operation_id,
|
||||
"error_code": "physical_backup_failed", "changed": 0, "backup_path": "",
|
||||
}
|
||||
with self.store._transaction_sync():
|
||||
|
||||
candidates = self.store._conn.execute(
|
||||
"""
|
||||
SELECT m.id
|
||||
FROM memories AS m
|
||||
LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id
|
||||
WHERE p.memory_id IS NULL
|
||||
ORDER BY m.id
|
||||
LIMIT ?
|
||||
""",
|
||||
(bounded_limit,),
|
||||
).fetchall()
|
||||
now = utc_now()
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
INSERT INTO p5_provenance_migration_runs(
|
||||
operation_id, operation_ref_hash, contract_name, contract_version,
|
||||
contract_fingerprint, state, requested_limit, candidate_count,
|
||||
applied_count, rollback_conflict_count, backup_path, created_at, updated_at
|
||||
) VALUES(?,?,?,?,?,'running',?,?,0,0,?,?,?)
|
||||
""",
|
||||
(
|
||||
clean_operation_id, reference, PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION, contract_fingerprint(), bounded_limit,
|
||||
len(candidates), backup_path, now, now,
|
||||
),
|
||||
)
|
||||
changed = 0
|
||||
for row in candidates:
|
||||
record = legacy_unresolved(row["id"], record_revision=1)
|
||||
record["migration_operation_ref"] = reference
|
||||
checked = validate_provenance_record(record)
|
||||
digest = provenance_record_digest(record)
|
||||
if not checked["ok"] or not digest:
|
||||
raise RuntimeError("p5_provenance_legacy_record_invalid")
|
||||
record_json = _canonical_json(record)
|
||||
# The backup/manifest row is deliberately inserted before
|
||||
# the matching provenance record mutation in this transaction.
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
INSERT INTO p5_provenance_migration_backups(
|
||||
operation_id, memory_id, before_exists, before_record_json,
|
||||
before_digest, before_revision, after_record_json, after_digest,
|
||||
after_revision, apply_state, rollback_state, created_at
|
||||
) VALUES(?,?,0,'','',0,?,?,?,'prepared','pending',?)
|
||||
""",
|
||||
(clean_operation_id, record["memory_id"], record_json, digest, record["record_revision"], now),
|
||||
)
|
||||
_insert_record(self.store._conn, record, digest=digest, now=now)
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
UPDATE p5_provenance_migration_backups
|
||||
SET apply_state='applied', applied_at=?
|
||||
WHERE operation_id=? AND memory_id=?
|
||||
""",
|
||||
(now, clean_operation_id, record["memory_id"]),
|
||||
)
|
||||
changed += 1
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
UPDATE p5_provenance_migration_runs
|
||||
SET state='completed', applied_count=?, updated_at=?, completed_at=?
|
||||
WHERE operation_id=?
|
||||
""",
|
||||
(changed, now, now, clean_operation_id),
|
||||
)
|
||||
return {
|
||||
"mode": "apply", "ok": True, "operation_id": clean_operation_id,
|
||||
"changed": changed, "deduplicated": False, "backup_count": changed,
|
||||
"backup_path": backup_path,
|
||||
}
|
||||
|
||||
async def rollback(self, operation_id: str) -> dict[str, Any]:
|
||||
return await self.store._run_recoverable_database_operation(self._rollback_sync, operation_id)
|
||||
|
||||
def _rollback_sync(self, operation_id: str) -> dict[str, Any]:
|
||||
clean_operation_id = _valid_operation_id(operation_id)
|
||||
if not clean_operation_id:
|
||||
return {"mode": "rollback", "ok": False, "operation_id": "", "error_code": "operation_id_required", "restored": 0, "conflicts": 0}
|
||||
with self.store._lock:
|
||||
with self.store._transaction_sync():
|
||||
run = self.store._conn.execute(
|
||||
"SELECT state, operation_ref_hash FROM p5_provenance_migration_runs WHERE operation_id=?",
|
||||
(clean_operation_id,),
|
||||
).fetchone()
|
||||
if run is None:
|
||||
return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_found", "restored": 0, "conflicts": 0}
|
||||
if run["state"] == "rolled_back":
|
||||
return {"mode": "rollback", "ok": True, "operation_id": clean_operation_id, "restored": 0, "conflicts": 0, "deduplicated": True}
|
||||
if run["state"] not in {"completed", "rollback_conflicted"}:
|
||||
return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_rollbackable", "restored": 0, "conflicts": 0}
|
||||
|
||||
backups = self.store._conn.execute(
|
||||
"""
|
||||
SELECT * FROM p5_provenance_migration_backups
|
||||
WHERE operation_id=? AND rollback_state='pending'
|
||||
ORDER BY memory_id
|
||||
""",
|
||||
(clean_operation_id,),
|
||||
).fetchall()
|
||||
restored = 0
|
||||
conflicts = 0
|
||||
now = utc_now()
|
||||
for backup in backups:
|
||||
current = self.store._conn.execute(
|
||||
"SELECT * FROM p5_provenance_records WHERE memory_id=?",
|
||||
(backup["memory_id"],),
|
||||
).fetchone()
|
||||
expected = (
|
||||
current is not None
|
||||
and current["record_digest"] == backup["after_digest"]
|
||||
and int(current["record_revision"]) == int(backup["after_revision"])
|
||||
and current["migration_operation_ref"] == run["operation_ref_hash"]
|
||||
)
|
||||
if not expected:
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
UPDATE p5_provenance_migration_backups
|
||||
SET rollback_state='conflict'
|
||||
WHERE operation_id=? AND memory_id=?
|
||||
""",
|
||||
(clean_operation_id, backup["memory_id"]),
|
||||
)
|
||||
conflicts += 1
|
||||
continue
|
||||
if int(backup["before_exists"]):
|
||||
try:
|
||||
before = json.loads(backup["before_record_json"])
|
||||
except (TypeError, ValueError):
|
||||
before = None
|
||||
before_digest = provenance_record_digest(before if isinstance(before, Mapping) else None)
|
||||
if not before_digest or before_digest != backup["before_digest"]:
|
||||
self.store._conn.execute(
|
||||
"UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?",
|
||||
(clean_operation_id, backup["memory_id"]),
|
||||
)
|
||||
conflicts += 1
|
||||
continue
|
||||
self.store._conn.execute("DELETE FROM p5_provenance_records WHERE memory_id=?", (backup["memory_id"],))
|
||||
_insert_record(self.store._conn, before, digest=before_digest, now=now)
|
||||
else:
|
||||
deleted = self.store._conn.execute(
|
||||
"DELETE FROM p5_provenance_records WHERE memory_id=? AND record_digest=? AND record_revision=? AND migration_operation_ref=?",
|
||||
(backup["memory_id"], backup["after_digest"], backup["after_revision"], run["operation_ref_hash"]),
|
||||
)
|
||||
if deleted.rowcount != 1:
|
||||
# The earlier SELECT predicate should make this
|
||||
# unreachable under the store lock, but retain a
|
||||
# conservative conflict if an adapter changes it.
|
||||
self.store._conn.execute(
|
||||
"UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?",
|
||||
(clean_operation_id, backup["memory_id"]),
|
||||
)
|
||||
conflicts += 1
|
||||
continue
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
UPDATE p5_provenance_migration_backups
|
||||
SET rollback_state='restored', rolled_back_at=?
|
||||
WHERE operation_id=? AND memory_id=?
|
||||
""",
|
||||
(now, clean_operation_id, backup["memory_id"]),
|
||||
)
|
||||
restored += 1
|
||||
state = "rolled_back" if conflicts == 0 else "rollback_conflicted"
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
UPDATE p5_provenance_migration_runs
|
||||
SET state=?, rollback_conflict_count=rollback_conflict_count + ?,
|
||||
updated_at=?, rolled_back_at=?
|
||||
WHERE operation_id=?
|
||||
""",
|
||||
(state, conflicts, now, now if conflicts == 0 else "", clean_operation_id),
|
||||
)
|
||||
return {
|
||||
"mode": "rollback", "ok": conflicts == 0, "operation_id": clean_operation_id,
|
||||
"restored": restored, "conflicts": conflicts, "deduplicated": False,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"P5ProvenanceMigration",
|
||||
"P5_PERSISTENCE_SCHEMA_VERSION",
|
||||
"ensure_p5_provenance_persistence_schema",
|
||||
]
|
||||
|
|
@ -41,6 +41,7 @@ class RetrievalEngine:
|
|||
keyword_fallback_min_fts_candidates: int = 80,
|
||||
knowledge_graph_enabled: bool = True,
|
||||
knowledge_graph_expansion_limit: int = 12,
|
||||
p5_recall_provenance_required: bool = False,
|
||||
usage_recorder: Any | None = None,
|
||||
):
|
||||
self.store = store
|
||||
|
|
@ -69,6 +70,7 @@ class RetrievalEngine:
|
|||
)
|
||||
self.knowledge_graph_enabled = bool(knowledge_graph_enabled)
|
||||
self.knowledge_graph_expansion_limit = max(0, int(knowledge_graph_expansion_limit or 0))
|
||||
self.p5_recall_provenance_required = bool(p5_recall_provenance_required)
|
||||
self.usage_recorder = usage_recorder if callable(usage_recorder) else None
|
||||
self._rank_path_info: dict[str, Any] = {}
|
||||
self.last_path_info: dict[str, Any] = {
|
||||
|
|
@ -102,7 +104,11 @@ class RetrievalEngine:
|
|||
clean_text(getattr(getattr(item, "memory", None), "id", ""), 120)
|
||||
for item in results or []
|
||||
]
|
||||
current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id])
|
||||
active_ids = [memory_id for memory_id in memory_ids if memory_id]
|
||||
if self.p5_recall_provenance_required:
|
||||
current = await self.store.p5_recall_eligible_memories_by_ids(active_ids)
|
||||
else:
|
||||
current = await self.store.get_memories_by_ids(active_ids)
|
||||
acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state()
|
||||
validated: list[SearchResult] = []
|
||||
for item, memory_id in zip(results or [], memory_ids):
|
||||
|
|
@ -126,7 +132,11 @@ class RetrievalEngine:
|
|||
) -> list[dict[str, str]]:
|
||||
"""Drop cached diagnostic snippets that are no longer readable."""
|
||||
memory_ids = [clean_text(item.get("id"), 120) for item in blocked if isinstance(item, dict)]
|
||||
current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id])
|
||||
active_ids = [memory_id for memory_id in memory_ids if memory_id]
|
||||
if self.p5_recall_provenance_required:
|
||||
current = await self.store.p5_recall_eligible_memories_by_ids(active_ids)
|
||||
else:
|
||||
current = await self.store.get_memories_by_ids(active_ids)
|
||||
acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state()
|
||||
validated: list[dict[str, str]] = []
|
||||
for item in blocked or []:
|
||||
|
|
@ -866,7 +876,11 @@ class RetrievalEngine:
|
|||
query = clean_text(query, 1000)
|
||||
terms = self._terms(query)
|
||||
graph_terms = []
|
||||
if self.knowledge_graph_enabled and self.knowledge_graph_expansion_limit > 0:
|
||||
if (
|
||||
not self.p5_recall_provenance_required
|
||||
and self.knowledge_graph_enabled
|
||||
and self.knowledge_graph_expansion_limit > 0
|
||||
):
|
||||
graph_labels = await self.store.related_knowledge_terms(
|
||||
terms,
|
||||
scope=ctx.scope,
|
||||
|
|
@ -881,7 +895,11 @@ class RetrievalEngine:
|
|||
)
|
||||
expanded_terms = self._merge_terms(terms, graph_terms)
|
||||
include_pending = not self.policy.hide_pending_review
|
||||
ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending)
|
||||
ranked_candidates = await self.store.list_candidate_memories(
|
||||
limit=2000,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
current_window_candidates = await self.store.list_current_window_candidate_memories(
|
||||
scope=ctx.scope,
|
||||
session_id=ctx.session_id,
|
||||
|
|
@ -889,6 +907,7 @@ class RetrievalEngine:
|
|||
group_id=ctx.group_id,
|
||||
limit=self.current_window_candidate_limit,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
time_window_candidates: list[MemoryRecord] = []
|
||||
if time_intent is not None and time_intent.active:
|
||||
|
|
@ -897,12 +916,14 @@ class RetrievalEngine:
|
|||
time_intent.end_at,
|
||||
limit=1600,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
keyword_terms = self._keyword_candidate_terms(query, expanded_terms)
|
||||
fts_candidates = await self.store.list_fts_candidate_memories(
|
||||
keyword_terms,
|
||||
limit=1200,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
use_keyword_fallback = len(fts_candidates) < self.keyword_fallback_min_fts_candidates
|
||||
keyword_candidates = (
|
||||
|
|
@ -910,6 +931,7 @@ class RetrievalEngine:
|
|||
keyword_terms,
|
||||
limit=1200,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
if use_keyword_fallback
|
||||
else []
|
||||
|
|
@ -917,6 +939,7 @@ class RetrievalEngine:
|
|||
vector_candidates, vector_scores, embedding_info = await self._embedding_candidate_memories(
|
||||
query,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=self.p5_recall_provenance_required,
|
||||
)
|
||||
self._rank_path_info = embedding_info
|
||||
self._rank_path_info.update(
|
||||
|
|
@ -1010,6 +1033,7 @@ class RetrievalEngine:
|
|||
query: str,
|
||||
*,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> tuple[list[MemoryRecord], dict[str, float], dict[str, Any]]:
|
||||
info: dict[str, Any] = {
|
||||
"embedding_enabled": bool(self.embedding_enabled),
|
||||
|
|
@ -1046,6 +1070,7 @@ class RetrievalEngine:
|
|||
provider_id=self.embedding_provider_id,
|
||||
limit=self.embedding_candidate_limit,
|
||||
include_pending=include_pending,
|
||||
require_p5_recall_provenance=require_p5_recall_provenance,
|
||||
)
|
||||
except Exception as error:
|
||||
info["embedding_reason"] = f"embedding_store_error:{clean_text(error, 120)}"
|
||||
|
|
|
|||
179
core/service.py
179
core/service.py
|
|
@ -9,6 +9,7 @@ import json
|
|||
import hashlib
|
||||
import inspect
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -92,9 +93,13 @@ from .user_memory_boundary import UserMemoryBoundaryError, validate_user_memory_
|
|||
from .user_memory_migration import UserMemoryMigration
|
||||
from .unified_person_persistence import UnifiedPersonPersistence, UnifiedPersonReferenceMigration
|
||||
from .person_memory import PersonMemoryFacts, PersonMemoryMigration
|
||||
from .p5_provenance_persistence import P5ProvenanceMigration
|
||||
from .p5_b1_recall_gate import evaluate_bridge_serialization_gate, evaluate_normal_recall_gate
|
||||
from .visibility import VisibilityPolicy
|
||||
|
||||
|
||||
_P5_ATTESTATION_BIND_FALLBACK_LOCK = threading.RLock()
|
||||
|
||||
_RECENT_FACT_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
||||
("meal", re.compile(r"吃过|吃完|吃了|没吃|还没吃|正在吃|吃饭|早饭|早餐|午饭|午餐|晚饭|晚餐|夜宵")),
|
||||
("bath", re.compile(r"洗澡|洗完澡|冲澡|沐浴")),
|
||||
|
|
@ -198,10 +203,14 @@ class MemoryCompanionService:
|
|||
self._retrieval_result_cache_stats: dict[str, int] = {"hits": 0, "misses": 0, "evictions": 0}
|
||||
self._RETRIEVAL_RESULT_CACHE_TTL: float = 45.0
|
||||
self._RETRIEVAL_RESULT_CACHE_MAX: int = 128
|
||||
self._p5_attestation_verifier: Any = None
|
||||
self._p5_attestation_verifier_bound = False
|
||||
self._p5_attestation_verifier_lock = threading.RLock()
|
||||
self.user_memory_migration = UserMemoryMigration(self.store)
|
||||
self.unified_person_persistence = UnifiedPersonPersistence(self.store)
|
||||
self.unified_person_reference_migration = UnifiedPersonReferenceMigration(self.store)
|
||||
self.person_memory_migration = PersonMemoryMigration(self.store)
|
||||
self.p5_provenance_migration = P5ProvenanceMigration(self.store)
|
||||
self.person_memory_facts = PersonMemoryFacts(self.store)
|
||||
self.portable_archive = PortableMemoryArchive(self.store, self.data_dir)
|
||||
self.chat_importer = HistoricalChatImporter(self)
|
||||
|
|
@ -281,6 +290,28 @@ class MemoryCompanionService:
|
|||
async def rollback_person_memory_migration(self, migration_id: str) -> dict[str, Any]:
|
||||
return await self.person_memory_migration.rollback(migration_id)
|
||||
|
||||
async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]:
|
||||
return await self.p5_provenance_migration.preview(limit=limit)
|
||||
|
||||
async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
|
||||
return await self.p5_provenance_migration.apply(operation_id=operation_id, limit=limit)
|
||||
|
||||
async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]:
|
||||
return await self.p5_provenance_migration.rollback(operation_id)
|
||||
|
||||
def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool:
|
||||
"""Pin the first Bridge-provided in-process B1 verifier."""
|
||||
|
||||
if not callable(consume_callable):
|
||||
return False
|
||||
bind_lock = getattr(self, "_p5_attestation_verifier_lock", _P5_ATTESTATION_BIND_FALLBACK_LOCK)
|
||||
with bind_lock:
|
||||
if getattr(self, "_p5_attestation_verifier_bound", False):
|
||||
return False
|
||||
self._p5_attestation_verifier = consume_callable
|
||||
self._p5_attestation_verifier_bound = True
|
||||
return True
|
||||
|
||||
async def bridge_correct_person_memory(
|
||||
self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord,
|
||||
) -> dict[str, Any]:
|
||||
|
|
@ -500,6 +531,84 @@ class MemoryCompanionService:
|
|||
ctx.context_orchestration = project_context_state(state) or {}
|
||||
ctx.context_orchestration_status = "accepted"
|
||||
|
||||
def _p5_b1_record_normal_recall_audit(self, *, event: Any, req: Any, reason: str) -> None:
|
||||
"""Expose only a static gate reason; never retain a handle/snapshot."""
|
||||
|
||||
for carrier in (req, event):
|
||||
if carrier is None:
|
||||
continue
|
||||
try:
|
||||
setattr(carrier, "memory_companion_p5_b1_recall_gate_reason", reason)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _p5_b1_normal_recall_injection(
|
||||
self,
|
||||
ctx: SessionContext,
|
||||
req: Any,
|
||||
*,
|
||||
event: Any,
|
||||
p3_state: Any,
|
||||
) -> tuple[str | None, str]:
|
||||
"""Gate only the normal request-time memory injection seam.
|
||||
|
||||
With the default switch off this calls the historical seam unchanged.
|
||||
With it on, the pinned verifier must consume the opaque handle using
|
||||
these exact request/event/P3 object identities before recall proceeds.
|
||||
"""
|
||||
|
||||
if not self.config.bool("enable_p5_b1_recall_gate", False):
|
||||
return await self.inject_memories(ctx, req, event=event), ""
|
||||
reason = evaluate_normal_recall_gate(
|
||||
getattr(self, "_p5_attestation_verifier", None),
|
||||
req=req,
|
||||
event=event,
|
||||
p3_state=p3_state,
|
||||
)
|
||||
if reason:
|
||||
self._p5_b1_record_normal_recall_audit(event=event, req=req, reason=reason)
|
||||
return None, reason
|
||||
return await self.inject_memories(ctx, req, event=event), ""
|
||||
|
||||
def _p5_b1_bridge_gate_reason(self, invocation: Any) -> str:
|
||||
"""Consume an explicitly carried B1b invocation before Bridge recall.
|
||||
|
||||
``None`` means a legacy/background Bridge caller which is outside the
|
||||
current-request B1b scope. A protected Companion path always carries
|
||||
an opaque invocation, including an intentionally unusable one when
|
||||
minting or verifier binding failed, so it fails closed here.
|
||||
"""
|
||||
|
||||
if invocation is None:
|
||||
return ""
|
||||
if not self.config.bool("enable_p5_b1_bridge_gate", False):
|
||||
return ""
|
||||
return evaluate_bridge_serialization_gate(
|
||||
getattr(self, "_p5_attestation_verifier", None),
|
||||
invocation=invocation,
|
||||
)
|
||||
|
||||
def _p5_b1_tool_recall_gate_reason(self) -> str:
|
||||
"""Fail closed because tool callbacks do not carry a request invocation."""
|
||||
|
||||
config = getattr(self, "config", None)
|
||||
getter = getattr(config, "bool", None)
|
||||
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b1_tool_recall_gate", False) else ""
|
||||
|
||||
def _p5_b2_archive_read_gate_reason(self) -> str:
|
||||
"""Reject scope-dict archive reads until a trusted identity contract exists."""
|
||||
|
||||
config = getattr(self, "config", None)
|
||||
getter = getattr(config, "bool", None)
|
||||
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_archive_read_gate", False) else ""
|
||||
|
||||
def _p5_b2_cross_user_read_gate_reason(self) -> str:
|
||||
"""Reject scope-dict cross-user reads until a trusted identity contract exists."""
|
||||
|
||||
config = getattr(self, "config", None)
|
||||
getter = getattr(config, "bool", None)
|
||||
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_cross_user_read_gate", False) else ""
|
||||
|
||||
async def handle_llm_request(self, event: Any, req: Any) -> None:
|
||||
ctx = await self.identity.resolve_event_context(event)
|
||||
self._sanitize_session_context_message_text(ctx)
|
||||
|
|
@ -533,8 +642,13 @@ class MemoryCompanionService:
|
|||
# this memory plugin no longer advances relationship authority.
|
||||
|
||||
try:
|
||||
injection = await self.inject_memories(ctx, req, event=event)
|
||||
if injection is not None:
|
||||
injection, p5_b1_reason = await self._p5_b1_normal_recall_injection(
|
||||
ctx, req, event=event, p3_state=p3_state
|
||||
)
|
||||
if p5_b1_reason:
|
||||
self._mark_memory_companion_injection_state(event, req, injected=False, conversation_memory=False, slot_map={})
|
||||
self._p3_complete_memory(p3_state, ctx=ctx, injection="", event=event, req=req)
|
||||
elif injection is not None:
|
||||
self._p3_complete_memory(p3_state, ctx=ctx, injection=injection, event=event, req=req)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
|
|
@ -1354,6 +1468,16 @@ class MemoryCompanionService:
|
|||
time_range: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload = dict(scope or {})
|
||||
p5_b2_reason = self._p5_b2_cross_user_read_gate_reason()
|
||||
if p5_b2_reason:
|
||||
return {
|
||||
**user_fact_query_capabilities(available=True, warnings=[p5_b2_reason]),
|
||||
"query": clean_text(query, 1400),
|
||||
"scope": payload,
|
||||
"memories": [],
|
||||
"excluded": [],
|
||||
"warnings": [p5_b2_reason],
|
||||
}
|
||||
requested_scope = clean_text(payload.get("scope"), 40).lower()
|
||||
user_id = clean_text(payload.get("user_id"), 120)
|
||||
group_id = clean_text(payload.get("group_id"), 120)
|
||||
|
|
@ -1445,6 +1569,15 @@ class MemoryCompanionService:
|
|||
time_range: dict[str, str] | None = None,
|
||||
include_timeline: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
|
||||
if p5_b2_reason:
|
||||
return {
|
||||
"archive_api_version": ARCHIVE_API_VERSION,
|
||||
"scope": await self.bridge_get_archive_scope(scope=scope),
|
||||
"memories": [],
|
||||
"excluded": [],
|
||||
"warnings": [p5_b2_reason],
|
||||
}
|
||||
ctx = self._archive_context(scope)
|
||||
query_text = clean_text(query, 1400)
|
||||
gate_reason = self._p3_gate_reason(
|
||||
|
|
@ -1515,6 +1648,13 @@ class MemoryCompanionService:
|
|||
scope: dict[str, Any],
|
||||
include_timeline: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
|
||||
if p5_b2_reason:
|
||||
return {
|
||||
"archive_api_version": "1.0",
|
||||
"scope": await self.bridge_get_archive_scope(scope=scope),
|
||||
"memories": [], "excluded": [], "warnings": [p5_b2_reason],
|
||||
}
|
||||
ctx = self._archive_context(scope)
|
||||
gate_reason = self._p3_gate_reason(
|
||||
ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None,
|
||||
|
|
@ -1628,6 +1768,14 @@ class MemoryCompanionService:
|
|||
limit: int = 30,
|
||||
include_timeline: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
|
||||
if p5_b2_reason:
|
||||
return {
|
||||
"archive_api_version": "1.0",
|
||||
"scope": await self.bridge_get_archive_scope(scope=scope),
|
||||
"start_at": clean_text(start_at, 80), "end_at": clean_text(end_at, 80),
|
||||
"items": [], "warnings": [p5_b2_reason],
|
||||
}
|
||||
ctx = self._archive_context(scope)
|
||||
gate_reason = self._p3_gate_reason(
|
||||
ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None,
|
||||
|
|
@ -1694,7 +1842,10 @@ class MemoryCompanionService:
|
|||
max_chars: int | None = None,
|
||||
companion_bot_mood: str = "",
|
||||
companion_bot_energy: float = 0.0,
|
||||
p5_b1_bridge_invocation: Any = None,
|
||||
) -> str:
|
||||
if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation):
|
||||
return ""
|
||||
ctx = self.session_context_from_bridge(session_context)
|
||||
if self._p3_bridge_gate_reason(ctx):
|
||||
return ""
|
||||
|
|
@ -1722,6 +1873,7 @@ class MemoryCompanionService:
|
|||
companion_bot_mood: str = "",
|
||||
companion_bot_energy: float = 0.0,
|
||||
retrieval_profile: str = "",
|
||||
p5_b1_bridge_invocation: Any = None,
|
||||
) -> str:
|
||||
profile = clean_text(retrieval_profile, 40).lower()
|
||||
fast_profile_enabled = {
|
||||
|
|
@ -1743,6 +1895,7 @@ class MemoryCompanionService:
|
|||
companion_bot_mood=companion_bot_mood,
|
||||
companion_bot_energy=companion_bot_energy,
|
||||
context_kind="outfit" if profile == "outfit_fast" else "schedule",
|
||||
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
|
||||
)
|
||||
return await self.bridge_compose_injection(
|
||||
query,
|
||||
|
|
@ -1751,6 +1904,7 @@ class MemoryCompanionService:
|
|||
max_chars=max_chars,
|
||||
companion_bot_mood=companion_bot_mood,
|
||||
companion_bot_energy=companion_bot_energy,
|
||||
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
|
||||
)
|
||||
|
||||
async def _bridge_compose_schedule_fast_context(
|
||||
|
|
@ -1763,7 +1917,10 @@ class MemoryCompanionService:
|
|||
companion_bot_mood: str,
|
||||
companion_bot_energy: float,
|
||||
context_kind: str = "schedule",
|
||||
p5_b1_bridge_invocation: Any = None,
|
||||
) -> str:
|
||||
if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation):
|
||||
return ""
|
||||
started = time.monotonic()
|
||||
ctx = self._normalized_session_context(self.session_context_from_bridge(session_context))
|
||||
if self._p3_bridge_gate_reason(ctx):
|
||||
|
|
@ -1781,6 +1938,7 @@ class MemoryCompanionService:
|
|||
bot_id=ctx.bot_id,
|
||||
limit=max(72, limit * 12) if outfit_focus else max(24, limit * 6),
|
||||
strict_session_only=ctx.strict_session_only,
|
||||
require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False),
|
||||
)
|
||||
policy = self.visibility_policy()
|
||||
visible = [record for record in records if policy.is_visible(record, ctx)[0]]
|
||||
|
|
@ -2453,6 +2611,7 @@ class MemoryCompanionService:
|
|||
retrieval_mode="basic",
|
||||
embedding_enabled=False,
|
||||
knowledge_graph_enabled=False,
|
||||
p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False),
|
||||
)
|
||||
|
||||
async def _retrieval_engine(self, ctx: SessionContext, *, admin_read_all: bool = False) -> RetrievalEngine:
|
||||
|
|
@ -2494,6 +2653,7 @@ class MemoryCompanionService:
|
|||
),
|
||||
knowledge_graph_enabled=self.config.bool("knowledge_graph.retrieval_expansion_enabled", True),
|
||||
knowledge_graph_expansion_limit=self.config.int("knowledge_graph.expansion_limit", 12),
|
||||
p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False),
|
||||
usage_recorder=self._record_token_usage,
|
||||
)
|
||||
|
||||
|
|
@ -2986,6 +3146,7 @@ class MemoryCompanionService:
|
|||
provider_id=provider_id,
|
||||
limit=self.config.int("retrieval.embedding_backfill_batch_size", 50),
|
||||
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
|
||||
require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False),
|
||||
)
|
||||
for record in records:
|
||||
if await self._embed_memory_record(provider, provider_id, record):
|
||||
|
|
@ -3004,12 +3165,23 @@ class MemoryCompanionService:
|
|||
return False
|
||||
if record.review_status == "pending" and not self.config.bool("retrieval.embedding_index_pending", False):
|
||||
return False
|
||||
p5_recall_provenance_required = self.config.bool("enable_p5_b1_recall_gate", False)
|
||||
if p5_recall_provenance_required:
|
||||
eligible = await self.store.p5_recall_eligible_memory_ids([record.id])
|
||||
if record.id not in eligible:
|
||||
return False
|
||||
text = self._memory_embedding_text(record)
|
||||
text_hash = self._memory_embedding_text_hash(record)
|
||||
if not text or not text_hash:
|
||||
return False
|
||||
async with self._embedding_background_semaphore:
|
||||
try:
|
||||
# Revalidate immediately before external egress. This closes
|
||||
# the interval while waiting for the shared provider slot.
|
||||
if p5_recall_provenance_required:
|
||||
eligible = await self.store.p5_recall_eligible_memory_ids([record.id])
|
||||
if record.id not in eligible:
|
||||
return False
|
||||
vector = await self._embed_text_with_provider(provider, text, provider_id=provider_id)
|
||||
vector = self._normalize_embedding_vector(vector)
|
||||
if not vector:
|
||||
|
|
@ -4060,6 +4232,9 @@ class MemoryCompanionService:
|
|||
reason = self._p3_gate_reason(ctx, event=event, state=state)
|
||||
if reason:
|
||||
return {"ok": False, "error": reason, "memories": []}
|
||||
p5_b1_reason = self._p5_b1_tool_recall_gate_reason()
|
||||
if p5_b1_reason:
|
||||
return {"ok": False, "error": p5_b1_reason, "memories": []}
|
||||
query = clean_text(query, 1000)
|
||||
if not query:
|
||||
return {"ok": False, "error": "empty query", "memories": []}
|
||||
|
|
|
|||
203
core/store.py
203
core/store.py
|
|
@ -14,6 +14,66 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads,
|
|||
from .unified_person_persistence import ensure_unified_person_persistence_schema
|
||||
from .person_memory import apply_envelope, ensure_person_memory_schema, indexable, normalize_record
|
||||
from .person_memory_contract import consume_person_memory_envelope
|
||||
from .p5_provenance_persistence import ensure_p5_provenance_persistence_schema
|
||||
from .p5_provenance import (
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
SOURCE_KINDS as P5_SOURCE_KINDS,
|
||||
contract_fingerprint as p5_provenance_contract_fingerprint,
|
||||
)
|
||||
|
||||
|
||||
def _p5_recall_eligibility_predicate(memory_alias: str = "") -> tuple[str, list[str]]:
|
||||
"""Return the B1e SQL predicate for complete, valid recall records only."""
|
||||
|
||||
prefix = f"{memory_alias}." if memory_alias else ""
|
||||
observed_source_kinds = tuple(kind for kind in P5_SOURCE_KINDS if kind != "unknown")
|
||||
source_kind_placeholders = ", ".join("?" for _ in observed_source_kinds)
|
||||
return (
|
||||
f"""
|
||||
EXISTS(
|
||||
SELECT 1
|
||||
FROM p5_provenance_records AS p5
|
||||
WHERE p5.memory_id={prefix}id
|
||||
AND p5.contract_name=?
|
||||
AND p5.contract_version=?
|
||||
AND p5.contract_fingerprint=?
|
||||
AND p5.provenance_state='observed'
|
||||
AND p5.source_kind IN ({source_kind_placeholders})
|
||||
AND p5.source_trust IN ('T0', 'T1', 'T2')
|
||||
AND p5.firewall_status IN ('allowed', 'sanitized')
|
||||
AND length(p5.source_event_ref_hash)=64
|
||||
AND p5.source_event_ref_hash NOT GLOB '*[^0-9a-f]*'
|
||||
AND length(p5.authority_attestation_ref_hash)=64
|
||||
AND p5.authority_attestation_ref_hash NOT GLOB '*[^0-9a-f]*'
|
||||
AND p5.source_event_ref_hash != p5.authority_attestation_ref_hash
|
||||
AND typeof(p5.record_revision)='integer'
|
||||
AND p5.record_revision >= 0
|
||||
AND length(p5.record_digest)=64
|
||||
AND p5.record_digest NOT GLOB '*[^0-9a-f]*'
|
||||
AND (
|
||||
p5.migration_operation_ref=''
|
||||
OR (
|
||||
length(p5.migration_operation_ref)=64
|
||||
AND p5.migration_operation_ref NOT GLOB '*[^0-9a-f]*'
|
||||
)
|
||||
)
|
||||
AND (
|
||||
p5.recovery_operation_ref=''
|
||||
OR (
|
||||
length(p5.recovery_operation_ref)=64
|
||||
AND p5.recovery_operation_ref NOT GLOB '*[^0-9a-f]*'
|
||||
)
|
||||
)
|
||||
)
|
||||
""",
|
||||
[
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
p5_provenance_contract_fingerprint(),
|
||||
*observed_source_kinds,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class MemoryStore:
|
||||
|
|
@ -495,6 +555,7 @@ class MemoryStore:
|
|||
self._ensure_memory_columns_sync()
|
||||
ensure_unified_person_persistence_schema(self._conn)
|
||||
ensure_person_memory_schema(self._conn)
|
||||
ensure_p5_provenance_persistence_schema(self._conn)
|
||||
self._conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_memories_domain ON memories(memory_domain, lifecycle, scope, group_id, subject_id)"
|
||||
)
|
||||
|
|
@ -667,6 +728,22 @@ class MemoryStore:
|
|||
BEGIN
|
||||
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_insert
|
||||
AFTER INSERT ON p5_provenance_records
|
||||
BEGIN
|
||||
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_update
|
||||
AFTER UPDATE ON p5_provenance_records
|
||||
BEGIN
|
||||
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_delete
|
||||
AFTER DELETE ON p5_provenance_records
|
||||
BEGIN
|
||||
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
|
||||
END;
|
||||
"""
|
||||
)
|
||||
|
||||
|
|
@ -3492,14 +3569,34 @@ class MemoryStore:
|
|||
result.append(item)
|
||||
return result
|
||||
|
||||
async def list_candidate_memories(self, limit: int = 500, include_pending: bool = False) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(self._list_candidate_memories_sync, limit, include_pending)
|
||||
async def list_candidate_memories(
|
||||
self,
|
||||
limit: int = 500,
|
||||
include_pending: bool = False,
|
||||
*,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_candidate_memories_sync,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_candidate_memories_sync(self, limit: int, include_pending: bool) -> list[MemoryRecord]:
|
||||
def _list_candidate_memories_sync(
|
||||
self,
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
where = "lifecycle != 'archived'"
|
||||
params: list[Any] = []
|
||||
if not include_pending:
|
||||
where += " AND review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate()
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"""
|
||||
|
|
@ -3520,6 +3617,7 @@ class MemoryStore:
|
|||
bot_id: str = "",
|
||||
limit: int = 36,
|
||||
strict_session_only: bool = False,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
"""Read a small schedule-focused snapshot without waiting on the shared writer lock."""
|
||||
return await asyncio.to_thread(
|
||||
|
|
@ -3529,6 +3627,7 @@ class MemoryStore:
|
|||
bot_id,
|
||||
limit,
|
||||
strict_session_only,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_schedule_context_memories_sync(
|
||||
|
|
@ -3538,6 +3637,7 @@ class MemoryStore:
|
|||
bot_id: str,
|
||||
limit: int,
|
||||
strict_session_only: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
session_id = clean_text(session_id, 200)
|
||||
user_id = clean_text(user_id, 120)
|
||||
|
|
@ -3623,6 +3723,10 @@ class MemoryStore:
|
|||
return []
|
||||
where += " AND session_id=?"
|
||||
params.append(session_id)
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("memories")
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
uri = self.db_path.resolve().as_uri() + "?mode=ro"
|
||||
with closing(sqlite3.connect(uri, uri=True, timeout=0.35, check_same_thread=False)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
|
|
@ -3664,6 +3768,7 @@ class MemoryStore:
|
|||
group_id: str = "",
|
||||
limit: int = 600,
|
||||
include_pending: bool = False,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_current_window_candidate_memories_sync,
|
||||
|
|
@ -3673,6 +3778,7 @@ class MemoryStore:
|
|||
group_id,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_current_window_candidate_memories_sync(
|
||||
|
|
@ -3683,6 +3789,7 @@ class MemoryStore:
|
|||
group_id: str,
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
scope = clean_text(scope, 40).lower()
|
||||
session_id = clean_text(session_id, 200)
|
||||
|
|
@ -3704,6 +3811,10 @@ class MemoryStore:
|
|||
where = "lifecycle != 'archived' AND (" + " OR ".join(clauses) + ")"
|
||||
if not include_pending:
|
||||
where += " AND review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate()
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"""
|
||||
|
|
@ -3724,6 +3835,7 @@ class MemoryStore:
|
|||
end_at: str,
|
||||
limit: int = 1200,
|
||||
include_pending: bool = False,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_time_window_candidate_memories_sync,
|
||||
|
|
@ -3731,6 +3843,7 @@ class MemoryStore:
|
|||
end_at,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_time_window_candidate_memories_sync(
|
||||
|
|
@ -3739,15 +3852,21 @@ class MemoryStore:
|
|||
end_at: str,
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
start = clean_text(start_at, 80)
|
||||
end = clean_text(end_at, 80)
|
||||
if not start or not end:
|
||||
return []
|
||||
where = "lifecycle != 'archived'"
|
||||
params: list[Any] = [start, end, start, end, start, end]
|
||||
time_params: list[Any] = [start, end, start, end, start, end]
|
||||
params: list[Any] = []
|
||||
if not include_pending:
|
||||
where += " AND review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate()
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"""
|
||||
|
|
@ -3764,7 +3883,7 @@ class MemoryStore:
|
|||
importance DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
params + [max(1, int(limit or 1))],
|
||||
params + time_params + [max(1, int(limit or 1))],
|
||||
).fetchall()
|
||||
return [MemoryRecord.from_row(row) for row in rows]
|
||||
|
||||
|
|
@ -3773,12 +3892,15 @@ class MemoryStore:
|
|||
terms: list[str],
|
||||
limit: int = 800,
|
||||
include_pending: bool = False,
|
||||
*,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_fts_candidate_memories_sync,
|
||||
terms,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_fts_candidate_memories_sync(
|
||||
|
|
@ -3786,6 +3908,7 @@ class MemoryStore:
|
|||
terms: list[str],
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
if not self._fts_enabled:
|
||||
return []
|
||||
|
|
@ -3796,6 +3919,10 @@ class MemoryStore:
|
|||
params: list[Any] = [query]
|
||||
if not include_pending:
|
||||
where += " AND m.review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
try:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
|
|
@ -3845,12 +3972,15 @@ class MemoryStore:
|
|||
terms: list[str],
|
||||
limit: int = 800,
|
||||
include_pending: bool = False,
|
||||
*,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_keyword_candidate_memories_sync,
|
||||
terms,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_keyword_candidate_memories_sync(
|
||||
|
|
@ -3858,6 +3988,7 @@ class MemoryStore:
|
|||
terms: list[str],
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
cleaned_terms = []
|
||||
for term in terms or []:
|
||||
|
|
@ -3894,6 +4025,10 @@ class MemoryStore:
|
|||
)
|
||||
params.extend([like] * len(columns))
|
||||
where += " AND (" + " OR ".join(term_clauses) + ")"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate()
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
|
|
@ -4585,6 +4720,48 @@ class MemoryStore:
|
|||
async def get_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(self._get_memories_by_ids_sync, memory_ids)
|
||||
|
||||
async def p5_recall_eligible_memory_ids(self, memory_ids: list[str]) -> set[str]:
|
||||
"""Return only IDs that still satisfy the B1e SQL provenance rule."""
|
||||
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._p5_recall_eligible_memory_ids_sync,
|
||||
memory_ids,
|
||||
)
|
||||
|
||||
def _p5_recall_eligible_memory_ids_sync(self, memory_ids: list[str]) -> set[str]:
|
||||
ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120)))
|
||||
if not ids:
|
||||
return set()
|
||||
placeholders = ",".join("?" for _ in ids)
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"SELECT m.id FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}",
|
||||
[*ids, *predicate_params],
|
||||
).fetchall()
|
||||
return {clean_text(row["id"], 120) for row in rows if clean_text(row["id"], 120)}
|
||||
|
||||
async def p5_recall_eligible_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
|
||||
"""Atomically re-read cached bodies under the B1e provenance predicate."""
|
||||
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._p5_recall_eligible_memories_by_ids_sync,
|
||||
memory_ids,
|
||||
)
|
||||
|
||||
def _p5_recall_eligible_memories_by_ids_sync(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
|
||||
ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120)))
|
||||
if not ids:
|
||||
return {}
|
||||
placeholders = ",".join("?" for _ in ids)
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"SELECT m.* FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}",
|
||||
[*ids, *predicate_params],
|
||||
).fetchall()
|
||||
return {row["id"]: MemoryRecord.from_row(row) for row in rows}
|
||||
|
||||
async def get_timeline_event(self, event_id: str) -> dict[str, Any] | None:
|
||||
return await self._run_recoverable_database_operation(self._get_timeline_event_sync, event_id)
|
||||
|
||||
|
|
@ -5252,12 +5429,14 @@ class MemoryStore:
|
|||
provider_id: str,
|
||||
limit: int = 3000,
|
||||
include_pending: bool = False,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[tuple[MemoryRecord, list[float], str]]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_embedding_candidate_rows_sync,
|
||||
provider_id,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_embedding_candidate_rows_sync(
|
||||
|
|
@ -5265,6 +5444,7 @@ class MemoryStore:
|
|||
provider_id: str,
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[tuple[MemoryRecord, list[float], str]]:
|
||||
provider_id = clean_text(provider_id, 160)
|
||||
if not provider_id:
|
||||
|
|
@ -5275,7 +5455,7 @@ class MemoryStore:
|
|||
if revision != self._embedding_candidate_cache_revision:
|
||||
self._embedding_candidate_cache.clear()
|
||||
self._embedding_candidate_cache_revision = revision
|
||||
cache_key = (provider_id, bool(include_pending), safe_limit)
|
||||
cache_key = (provider_id, bool(include_pending), safe_limit, bool(require_p5_recall_provenance))
|
||||
cached = self._embedding_candidate_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return deepcopy(cached)
|
||||
|
|
@ -5283,6 +5463,10 @@ class MemoryStore:
|
|||
params: list[Any] = [provider_id]
|
||||
if not include_pending:
|
||||
where += " AND m.review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"""
|
||||
|
|
@ -5323,12 +5507,14 @@ class MemoryStore:
|
|||
provider_id: str,
|
||||
limit: int = 80,
|
||||
include_pending: bool = False,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
return await self._run_recoverable_database_operation(
|
||||
self._list_memories_missing_embeddings_sync,
|
||||
provider_id,
|
||||
limit,
|
||||
include_pending,
|
||||
require_p5_recall_provenance,
|
||||
)
|
||||
|
||||
def _list_memories_missing_embeddings_sync(
|
||||
|
|
@ -5336,6 +5522,7 @@ class MemoryStore:
|
|||
provider_id: str,
|
||||
limit: int,
|
||||
include_pending: bool,
|
||||
require_p5_recall_provenance: bool = False,
|
||||
) -> list[MemoryRecord]:
|
||||
provider_id = clean_text(provider_id, 160)
|
||||
if not provider_id:
|
||||
|
|
@ -5344,6 +5531,10 @@ class MemoryStore:
|
|||
params: list[Any] = [provider_id]
|
||||
if not include_pending:
|
||||
where += " AND m.review_status != 'pending'"
|
||||
if require_p5_recall_provenance:
|
||||
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
|
||||
where += f" AND {predicate}"
|
||||
params.extend(predicate_params)
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
f"""
|
||||
|
|
|
|||
263
tests/test_p5_b1_candidate_gate.py
普通文件
263
tests/test_p5_b1_candidate_gate.py
普通文件
|
|
@ -0,0 +1,263 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from core.models import EntityRef, MemoryRecord, SearchResult, SessionContext
|
||||
from core.p5_provenance import (
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
contract_fingerprint,
|
||||
)
|
||||
from core.retrieval import RetrievalEngine
|
||||
from core.service import MemoryCompanionService
|
||||
from core.store import MemoryStore
|
||||
from core.visibility import VisibilityPolicy
|
||||
|
||||
|
||||
_HASH = "a" * 64
|
||||
_OTHER_HASH = "b" * 64
|
||||
|
||||
|
||||
class _RerankProvider:
|
||||
model = "test-reranker"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.documents: list[str] = []
|
||||
|
||||
async def rerank(self, **kwargs):
|
||||
self.documents = list(kwargs.get("documents") or [])
|
||||
return [{"index": index, "relevance_score": 1.0} for index in range(len(self.documents))]
|
||||
|
||||
|
||||
class _ServiceConfig:
|
||||
def __init__(self, p5_required: bool) -> None:
|
||||
self.p5_required = p5_required
|
||||
|
||||
def bool(self, key: str, default: bool = False) -> bool:
|
||||
if key == "enable_p5_b1_recall_gate":
|
||||
return self.p5_required
|
||||
return default
|
||||
|
||||
def int(self, _key: str, default: int = 0) -> int:
|
||||
return default
|
||||
|
||||
def float(self, _key: str, default: float = 0.0) -> float:
|
||||
return default
|
||||
|
||||
def get(self, _key: str, default=None):
|
||||
return default
|
||||
|
||||
|
||||
class P5B1CandidateGateTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self) -> None:
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.store = MemoryStore(Path(self.temp.name) / "memory.db")
|
||||
self.store.initialize()
|
||||
for identifier in ("eligible", "legacy", "t3", "t4", "missing", "invalid", "bad_kind", "same_hash"):
|
||||
await self.store.insert_memory(
|
||||
MemoryRecord(
|
||||
id=identifier,
|
||||
memory_domain="user_memory",
|
||||
subject=EntityRef(kind="user", id="u1"),
|
||||
scope="private",
|
||||
session_id="s1",
|
||||
content=f"needle {identifier} private body",
|
||||
evidence="test",
|
||||
lifecycle="stable_memory",
|
||||
visibility="shareable",
|
||||
importance=0.8,
|
||||
)
|
||||
)
|
||||
self._put_provenance("eligible")
|
||||
self._put_provenance("legacy", state="legacy_unresolved", trust="T4", firewall="unknown", reference="")
|
||||
self._put_provenance("t3", trust="T3")
|
||||
self._put_provenance("t4", trust="T4")
|
||||
self._put_provenance("invalid", reference=_HASH.upper())
|
||||
self._put_provenance("bad_kind", source_kind="message")
|
||||
self._put_provenance("same_hash", source_event_hash=_HASH)
|
||||
|
||||
async def asyncTearDown(self) -> None:
|
||||
self.store.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def _put_provenance(
|
||||
self,
|
||||
memory_id: str,
|
||||
*,
|
||||
state: str = "observed",
|
||||
trust: str = "T1",
|
||||
firewall: str = "allowed",
|
||||
reference: str = _HASH,
|
||||
source_kind: str = "current_user_intent",
|
||||
source_event_hash: str = _OTHER_HASH,
|
||||
) -> None:
|
||||
self.store._conn.execute(
|
||||
"""
|
||||
INSERT INTO p5_provenance_records(
|
||||
memory_id, contract_name, contract_version, contract_fingerprint,
|
||||
source_kind, source_trust, firewall_status, source_event_ref_hash,
|
||||
authority_attestation_ref_hash, provenance_state,
|
||||
migration_operation_ref, recovery_operation_ref, record_revision,
|
||||
record_digest, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', '', 1, ?,
|
||||
'2026-07-29T00:00:00+00:00', '2026-07-29T00:00:00+00:00')
|
||||
""",
|
||||
(
|
||||
memory_id,
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
contract_fingerprint(),
|
||||
source_kind,
|
||||
trust,
|
||||
firewall,
|
||||
source_event_hash,
|
||||
reference,
|
||||
state,
|
||||
_HASH,
|
||||
),
|
||||
)
|
||||
self.store._conn.commit()
|
||||
|
||||
@staticmethod
|
||||
def _ids(rows) -> set[str]:
|
||||
return {row.id if hasattr(row, "id") else row[0].id for row in rows}
|
||||
|
||||
async def test_store_sql_fts_keyword_vector_and_windows_fail_closed(self) -> None:
|
||||
self.assertGreaterEqual(len(await self.store.list_candidate_memories(limit=20)), 6)
|
||||
self.assertEqual(
|
||||
{"eligible"}, self._ids(await self.store.list_candidate_memories(limit=20, require_p5_recall_provenance=True))
|
||||
)
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(
|
||||
await self.store.list_current_window_candidate_memories(
|
||||
scope="private", session_id="s1", user_id="u1", limit=20,
|
||||
require_p5_recall_provenance=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(
|
||||
await self.store.list_time_window_candidate_memories(
|
||||
"2026-01-01T00:00:00+00:00", "2027-01-01T00:00:00+00:00", limit=20,
|
||||
require_p5_recall_provenance=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(await self.store.list_fts_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)),
|
||||
)
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(await self.store.list_keyword_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)),
|
||||
)
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(
|
||||
await self.store.list_memories_missing_embeddings(
|
||||
provider_id="embed", limit=20, require_p5_recall_provenance=True
|
||||
)
|
||||
),
|
||||
)
|
||||
await self.store.upsert_memory_embedding(memory_id="eligible", provider_id="embed", text_hash="e", vector=[1.0, 0.0])
|
||||
await self.store.upsert_memory_embedding(memory_id="legacy", provider_id="embed", text_hash="l", vector=[1.0, 0.0])
|
||||
self.assertEqual(
|
||||
{"eligible"},
|
||||
self._ids(
|
||||
await self.store.list_embedding_candidate_rows(
|
||||
provider_id="embed", limit=20, require_p5_recall_provenance=True
|
||||
)
|
||||
),
|
||||
)
|
||||
self.store._conn.execute("UPDATE p5_provenance_records SET source_trust='T3' WHERE memory_id='eligible'")
|
||||
self.store._conn.commit()
|
||||
self.assertEqual(
|
||||
set(),
|
||||
self._ids(
|
||||
await self.store.list_embedding_candidate_rows(
|
||||
provider_id="embed", limit=20, require_p5_recall_provenance=True
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
async def test_engine_excludes_unattested_body_before_rerank_and_revalidates_cache(self) -> None:
|
||||
provider = _RerankProvider()
|
||||
engine = RetrievalEngine(
|
||||
self.store,
|
||||
VisibilityPolicy(enable_acl_rules=False),
|
||||
retrieval_mode="rerank",
|
||||
rerank_provider=provider,
|
||||
rerank_provider_id="test",
|
||||
embedding_enabled=False,
|
||||
knowledge_graph_enabled=True,
|
||||
p5_recall_provenance_required=True,
|
||||
)
|
||||
ctx = SessionContext(session_id="s1", scope="private", user_id="u1")
|
||||
results = await engine.search("needle", ctx, 10)
|
||||
self.assertEqual({"eligible"}, {item.memory.id for item in results})
|
||||
self.assertEqual(1, len(provider.documents))
|
||||
self.assertIn("needle eligible private body", provider.documents[0])
|
||||
self.assertNotIn("needle legacy private body", provider.documents[0])
|
||||
cached = await engine.revalidate_cached_results(
|
||||
[SearchResult(memory=(await self.store.get_memory("legacy")), score=1.0)], ctx
|
||||
)
|
||||
self.assertEqual([], cached)
|
||||
|
||||
default_off = RetrievalEngine(
|
||||
self.store,
|
||||
VisibilityPolicy(enable_acl_rules=False),
|
||||
retrieval_mode="basic",
|
||||
embedding_enabled=False,
|
||||
knowledge_graph_enabled=False,
|
||||
p5_recall_provenance_required=False,
|
||||
)
|
||||
legacy_visible = await default_off.search("needle legacy", ctx, 10)
|
||||
self.assertIn("legacy", {item.memory.id for item in legacy_visible})
|
||||
|
||||
async def test_service_propagates_recall_switch_to_retrieval_and_cache_validation_engines(self) -> None:
|
||||
async def no_rerank_provider(*_args, **_kwargs):
|
||||
return None, ""
|
||||
|
||||
for enabled in (False, True):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.store = self.store
|
||||
service.config = _ServiceConfig(enabled)
|
||||
service.visibility_policy = lambda **_kwargs: VisibilityPolicy(enable_acl_rules=False)
|
||||
service._resolve_rerank_provider = no_rerank_provider
|
||||
service._record_token_usage = lambda *_args, **_kwargs: None
|
||||
service._embedding_timeout_ms = lambda: 1200
|
||||
engine = await service._retrieval_engine(SessionContext())
|
||||
validation = service._retrieval_validation_engine()
|
||||
self.assertEqual(enabled, engine.p5_recall_provenance_required)
|
||||
self.assertEqual(enabled, validation.p5_recall_provenance_required)
|
||||
|
||||
async def test_embedding_provider_rechecks_provenance_before_body_egress(self) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
async def embed(_provider, text: str, *, provider_id: str):
|
||||
calls.append(text)
|
||||
return [1.0, 0.0]
|
||||
|
||||
def build_service(enabled: bool):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.store = self.store
|
||||
service.config = _ServiceConfig(enabled)
|
||||
service._embedding_background_semaphore = asyncio.Semaphore(1)
|
||||
service._embed_text_with_provider = embed
|
||||
return service
|
||||
|
||||
legacy = await self.store.get_memory("legacy")
|
||||
self.assertFalse(await build_service(True)._embed_memory_record(object(), "embed-guard", legacy))
|
||||
self.assertEqual([], calls)
|
||||
|
||||
self.assertTrue(await build_service(False)._embed_memory_record(object(), "embed-default-off", legacy))
|
||||
self.assertEqual(1, len(calls))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
337
tests/test_p5_b1_recall_gate.py
普通文件
337
tests/test_p5_b1_recall_gate.py
普通文件
|
|
@ -0,0 +1,337 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
import threading
|
||||
import unittest
|
||||
|
||||
from core.bridge import MemoryCompanionBridge
|
||||
from core.p5_b1_recall_gate import (
|
||||
BRIDGE_SERIALIZATION_SINK,
|
||||
MEMORY_RECALL_SINK,
|
||||
P5_B1_ATTESTATION_HANDLES_ATTRIBUTE,
|
||||
)
|
||||
from core.service import MemoryCompanionService
|
||||
|
||||
|
||||
class _Config:
|
||||
def __init__(self, enabled: bool, *, bridge_enabled: bool | None = None):
|
||||
self.enabled = enabled
|
||||
self.bridge_enabled = enabled if bridge_enabled is None else bridge_enabled
|
||||
|
||||
def bool(self, key: str, default: bool = False) -> bool:
|
||||
if key == "enable_p5_b1_recall_gate":
|
||||
return self.enabled
|
||||
if key == "enable_p5_b1_bridge_gate":
|
||||
return self.bridge_enabled
|
||||
return default
|
||||
|
||||
def int(self, key: str, default: int = 0) -> int:
|
||||
return default
|
||||
|
||||
|
||||
class _Verifier:
|
||||
def __init__(self, *, disposition: str = "allow", sink: str = MEMORY_RECALL_SINK):
|
||||
self.disposition = disposition
|
||||
self.sink = sink
|
||||
self.calls: list[tuple[object, object, object, object, str]] = []
|
||||
self.consumed: set[int] = set()
|
||||
|
||||
def __call__(self, handle, request, event, p3_state, sink):
|
||||
self.calls.append((handle, request, event, p3_state, sink))
|
||||
marker = id(handle)
|
||||
if marker in self.consumed:
|
||||
return None
|
||||
self.consumed.add(marker)
|
||||
return SimpleNamespace(disposition=self.disposition, sink=self.sink, secret_snapshot_prose="never retained")
|
||||
|
||||
|
||||
def _service(*, enabled: bool, verifier=None):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.config = _Config(enabled)
|
||||
service._p5_attestation_verifier = verifier
|
||||
service._p5_attestation_verifier_bound = verifier is not None
|
||||
service.inject_calls: list[tuple[object, object, object]] = []
|
||||
|
||||
async def inject(ctx, req, *, event):
|
||||
service.inject_calls.append((ctx, req, event))
|
||||
return "normal-memory-injection"
|
||||
|
||||
service.inject_memories = inject
|
||||
return service
|
||||
|
||||
|
||||
class _BridgeInvocation:
|
||||
def __init__(self, handle, request, event, p3_state):
|
||||
self.values = (handle, request, event, p3_state)
|
||||
|
||||
def _p5_b1_bridge_attestation_arguments(self):
|
||||
return self.values
|
||||
|
||||
|
||||
def _carriers(handle=object()):
|
||||
request = SimpleNamespace()
|
||||
event = SimpleNamespace()
|
||||
setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {MEMORY_RECALL_SINK: handle})
|
||||
return request, event, handle
|
||||
|
||||
|
||||
class P5B1NormalRecallGateTests(unittest.TestCase):
|
||||
def test_first_bind_pins_service_and_rejects_rebind_or_replacement(self):
|
||||
service = _service(enabled=False)
|
||||
bridge = MemoryCompanionBridge(service)
|
||||
first = _Verifier()
|
||||
second = _Verifier()
|
||||
self.assertTrue(bridge.bind_p5_attestation_verifier(first))
|
||||
self.assertIs(first, service._p5_attestation_verifier)
|
||||
self.assertFalse(bridge.bind_p5_attestation_verifier(first))
|
||||
self.assertFalse(bridge.bind_p5_attestation_verifier(second))
|
||||
self.assertIs(first, service._p5_attestation_verifier)
|
||||
self.assertFalse(service.bind_p5_attestation_verifier(second))
|
||||
|
||||
def test_concurrent_first_bind_has_one_winner_and_never_replaces_it(self):
|
||||
service = _service(enabled=False)
|
||||
bridge = MemoryCompanionBridge(service)
|
||||
first = _Verifier()
|
||||
second = _Verifier()
|
||||
start = threading.Barrier(3)
|
||||
results: list[tuple[object, bool]] = []
|
||||
result_lock = threading.Lock()
|
||||
|
||||
def bind(verifier):
|
||||
start.wait()
|
||||
value = bridge.bind_p5_attestation_verifier(verifier)
|
||||
with result_lock:
|
||||
results.append((verifier, value))
|
||||
|
||||
first_thread = threading.Thread(target=bind, args=(first,))
|
||||
second_thread = threading.Thread(target=bind, args=(second,))
|
||||
first_thread.start()
|
||||
second_thread.start()
|
||||
start.wait()
|
||||
first_thread.join(timeout=5)
|
||||
second_thread.join(timeout=5)
|
||||
self.assertFalse(first_thread.is_alive())
|
||||
self.assertFalse(second_thread.is_alive())
|
||||
winners = [verifier for verifier, accepted in results if accepted]
|
||||
self.assertEqual(1, len(winners))
|
||||
self.assertIs(winners[0], bridge._p5_attestation_verifier)
|
||||
self.assertIs(winners[0], service._p5_attestation_verifier)
|
||||
|
||||
direct_service = _service(enabled=False)
|
||||
direct_start = threading.Barrier(3)
|
||||
direct_results: list[tuple[object, bool]] = []
|
||||
|
||||
def direct_bind(verifier):
|
||||
direct_start.wait()
|
||||
with result_lock:
|
||||
direct_results.append((verifier, direct_service.bind_p5_attestation_verifier(verifier)))
|
||||
|
||||
direct_first = threading.Thread(target=direct_bind, args=(first,))
|
||||
direct_second = threading.Thread(target=direct_bind, args=(second,))
|
||||
direct_first.start()
|
||||
direct_second.start()
|
||||
direct_start.wait()
|
||||
direct_first.join(timeout=5)
|
||||
direct_second.join(timeout=5)
|
||||
direct_winners = [verifier for verifier, accepted in direct_results if accepted]
|
||||
self.assertEqual(1, len(direct_winners))
|
||||
self.assertIs(direct_winners[0], direct_service._p5_attestation_verifier)
|
||||
|
||||
def test_default_off_preserves_normal_injection_without_verifier_or_handle(self):
|
||||
service = _service(enabled=False)
|
||||
request, event, _ = _carriers()
|
||||
setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {})
|
||||
injection, reason = asyncio.run(
|
||||
service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=None)
|
||||
)
|
||||
self.assertEqual(("normal-memory-injection", ""), (injection, reason))
|
||||
self.assertEqual(1, len(service.inject_calls))
|
||||
|
||||
def test_valid_allow_uses_exact_runtime_identities_and_reaches_normal_seam(self):
|
||||
verifier = _Verifier()
|
||||
service = _service(enabled=True, verifier=verifier)
|
||||
request, event, handle = _carriers()
|
||||
p3_state = object()
|
||||
ctx = object()
|
||||
injection, reason = asyncio.run(
|
||||
service._p5_b1_normal_recall_injection(ctx, request, event=event, p3_state=p3_state)
|
||||
)
|
||||
self.assertEqual(("normal-memory-injection", ""), (injection, reason))
|
||||
self.assertEqual([(handle, request, event, p3_state, MEMORY_RECALL_SINK)], verifier.calls)
|
||||
self.assertEqual([(ctx, request, event)], service.inject_calls)
|
||||
self.assertFalse(hasattr(request, "memory_companion_p5_b1_recall_gate_reason"))
|
||||
|
||||
def test_missing_fake_replay_wrong_identity_sink_and_nonallow_fail_closed(self):
|
||||
cases = []
|
||||
|
||||
missing_service = _service(enabled=True, verifier=_Verifier())
|
||||
missing_request, missing_event, _ = _carriers()
|
||||
setattr(missing_request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {})
|
||||
cases.append((missing_service, missing_request, missing_event, object(), "p5_b1_handle_missing"))
|
||||
|
||||
fake_service = _service(enabled=True, verifier=lambda *args: None)
|
||||
fake_request, fake_event, _ = _carriers()
|
||||
cases.append((fake_service, fake_request, fake_event, object(), "p5_b1_verifier_rejected"))
|
||||
|
||||
sink_service = _service(enabled=True, verifier=_Verifier(sink="bridge_serialization"))
|
||||
sink_request, sink_event, _ = _carriers()
|
||||
cases.append((sink_service, sink_request, sink_event, object(), "p5_b1_sink_mismatch"))
|
||||
|
||||
denied_service = _service(enabled=True, verifier=_Verifier(disposition="shadow_quarantine"))
|
||||
denied_request, denied_event, _ = _carriers()
|
||||
cases.append((denied_service, denied_request, denied_event, object(), "p5_b1_disposition_denied"))
|
||||
|
||||
p3_service = _service(enabled=True, verifier=_Verifier())
|
||||
p3_request, p3_event, _ = _carriers()
|
||||
cases.append((p3_service, p3_request, p3_event, None, "p5_b1_p3_unavailable"))
|
||||
|
||||
for service, request, event, p3_state, expected in cases:
|
||||
with self.subTest(expected=expected):
|
||||
injection, reason = asyncio.run(
|
||||
service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=p3_state)
|
||||
)
|
||||
self.assertIsNone(injection)
|
||||
self.assertEqual(expected, reason)
|
||||
self.assertEqual([], service.inject_calls)
|
||||
self.assertEqual(expected, request.memory_companion_p5_b1_recall_gate_reason)
|
||||
rendered = repr(vars(request)) + repr(vars(event))
|
||||
self.assertNotIn("secret_snapshot_prose", rendered)
|
||||
|
||||
replay_verifier = _Verifier()
|
||||
replay_service = _service(enabled=True, verifier=replay_verifier)
|
||||
replay_request, replay_event, _ = _carriers()
|
||||
first, first_reason = asyncio.run(
|
||||
replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object())
|
||||
)
|
||||
second, second_reason = asyncio.run(
|
||||
replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object())
|
||||
)
|
||||
self.assertEqual(("normal-memory-injection", ""), (first, first_reason))
|
||||
self.assertEqual((None, "p5_b1_verifier_rejected"), (second, second_reason))
|
||||
|
||||
def test_rejected_identity_verifier_receives_no_fallback_or_snapshot(self):
|
||||
expected_request, expected_event, _ = _carriers()
|
||||
|
||||
def exact_only(handle, request, event, p3_state, sink):
|
||||
if request is not expected_request or event is not expected_event or sink != MEMORY_RECALL_SINK:
|
||||
return None
|
||||
return SimpleNamespace(disposition="allow", sink=MEMORY_RECALL_SINK)
|
||||
|
||||
service = _service(enabled=True, verifier=exact_only)
|
||||
replacement_request, replacement_event, _ = _carriers()
|
||||
injection, reason = asyncio.run(
|
||||
service._p5_b1_normal_recall_injection(
|
||||
object(), replacement_request, event=replacement_event, p3_state=object()
|
||||
)
|
||||
)
|
||||
self.assertEqual((None, "p5_b1_verifier_rejected"), (injection, reason))
|
||||
self.assertEqual([], service.inject_calls)
|
||||
|
||||
def test_bridge_gate_is_default_off_and_enabled_denials_stop_before_retrieval(self):
|
||||
request = SimpleNamespace()
|
||||
event = SimpleNamespace()
|
||||
p3_state = object()
|
||||
handle = object()
|
||||
|
||||
def build_bridge_service(*, bridge_enabled: bool, verifier):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.config = _Config(False, bridge_enabled=bridge_enabled)
|
||||
service._p5_attestation_verifier = verifier
|
||||
service._p5_attestation_verifier_bound = verifier is not None
|
||||
service.retrieval_calls = 0
|
||||
service.session_context_from_bridge = lambda _value: SimpleNamespace(message_text="query")
|
||||
service._p3_bridge_gate_reason = lambda _ctx: ""
|
||||
|
||||
async def compose(*_args, **_kwargs):
|
||||
service.retrieval_calls += 1
|
||||
return "bridge-memory"
|
||||
|
||||
service._compose_memory_injection = compose
|
||||
return service
|
||||
|
||||
allowed = _Verifier(sink=BRIDGE_SERIALIZATION_SINK)
|
||||
enabled_service = build_bridge_service(
|
||||
bridge_enabled=True,
|
||||
verifier=allowed,
|
||||
)
|
||||
result = asyncio.run(
|
||||
enabled_service.bridge_compose_injection(
|
||||
"query",
|
||||
session_context={},
|
||||
p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state),
|
||||
)
|
||||
)
|
||||
self.assertEqual("bridge-memory", result)
|
||||
self.assertEqual(1, enabled_service.retrieval_calls)
|
||||
self.assertEqual(
|
||||
[(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK)],
|
||||
allowed.calls,
|
||||
)
|
||||
|
||||
replay = build_bridge_service(bridge_enabled=True, verifier=allowed)
|
||||
replay_invocation = _BridgeInvocation(handle, request, event, p3_state)
|
||||
self.assertEqual(
|
||||
"",
|
||||
asyncio.run(
|
||||
replay.bridge_compose_injection(
|
||||
"query", session_context={}, p5_b1_bridge_invocation=replay_invocation
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertEqual(0, replay.retrieval_calls)
|
||||
|
||||
for verifier in (
|
||||
_Verifier(disposition="shadow_quarantine", sink=BRIDGE_SERIALIZATION_SINK),
|
||||
_Verifier(sink=MEMORY_RECALL_SINK),
|
||||
):
|
||||
with self.subTest(verifier=verifier.disposition + verifier.sink):
|
||||
denied = build_bridge_service(bridge_enabled=True, verifier=verifier)
|
||||
self.assertEqual(
|
||||
"",
|
||||
asyncio.run(
|
||||
denied.bridge_compose_injection(
|
||||
"query",
|
||||
session_context={},
|
||||
p5_b1_bridge_invocation=_BridgeInvocation(object(), request, event, p3_state),
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertEqual(0, denied.retrieval_calls)
|
||||
|
||||
for invocation in (
|
||||
None,
|
||||
{},
|
||||
_BridgeInvocation(object(), SimpleNamespace(), event, p3_state),
|
||||
_BridgeInvocation(object(), request, event, p3_state),
|
||||
):
|
||||
with self.subTest(invocation=type(invocation).__name__):
|
||||
denied = build_bridge_service(bridge_enabled=True, verifier=lambda *args: None)
|
||||
blocked = asyncio.run(
|
||||
denied.bridge_compose_injection(
|
||||
"query",
|
||||
session_context={},
|
||||
p5_b1_bridge_invocation=invocation,
|
||||
)
|
||||
)
|
||||
if invocation is None:
|
||||
self.assertEqual("bridge-memory", blocked)
|
||||
self.assertEqual(1, denied.retrieval_calls)
|
||||
else:
|
||||
self.assertEqual("", blocked)
|
||||
self.assertEqual(0, denied.retrieval_calls)
|
||||
|
||||
default_off = build_bridge_service(bridge_enabled=False, verifier=None)
|
||||
legacy = asyncio.run(
|
||||
default_off.bridge_compose_injection(
|
||||
"query",
|
||||
session_context={},
|
||||
p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state),
|
||||
)
|
||||
)
|
||||
self.assertEqual("bridge-memory", legacy)
|
||||
self.assertEqual(1, default_off.retrieval_calls)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
|
||||
from core.service import MemoryCompanionService
|
||||
|
||||
|
||||
class _Config:
|
||||
def __init__(self, **flags: bool) -> None:
|
||||
self.flags = flags
|
||||
|
||||
def bool(self, key: str, default: bool = False) -> bool:
|
||||
return bool(self.flags.get(key, default))
|
||||
|
||||
def int(self, _key: str, default: int = 0) -> int:
|
||||
return default
|
||||
|
||||
|
||||
class _Identity:
|
||||
async def resolve_event_context(self, _event):
|
||||
return SimpleNamespace()
|
||||
|
||||
|
||||
def _service(**flags: bool):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.config = _Config(**flags)
|
||||
service.identity = _Identity()
|
||||
service._p3_prepare_context = lambda _ctx, **_kwargs: object()
|
||||
service._p3_gate_reason = lambda *_args, **_kwargs: ""
|
||||
service._archive_context = lambda _scope: SimpleNamespace(
|
||||
session_id="s1", scope="private", platform="test", user_id="u1", group_id="", bot_id="b1",
|
||||
message_id="m1", strict_session_only=True, context_orchestration=None,
|
||||
)
|
||||
|
||||
async def archive_scope(*, scope=None):
|
||||
return {"scope": dict(scope or {})}
|
||||
|
||||
service.bridge_get_archive_scope = archive_scope
|
||||
return service
|
||||
|
||||
|
||||
class P5B1UnverifiableReadGateTests(unittest.TestCase):
|
||||
def test_tool_recall_default_off_preserves_search_and_enabled_stops_before_search(self):
|
||||
default_off = _service()
|
||||
default_off.search_calls = 0
|
||||
|
||||
async def search(*_args, **_kwargs):
|
||||
default_off.search_calls += 1
|
||||
return []
|
||||
|
||||
default_off.search = search
|
||||
self.assertEqual(
|
||||
{"ok": True, "memories": []}, asyncio.run(default_off.tool_recall(object(), "needle"))
|
||||
)
|
||||
self.assertEqual(1, default_off.search_calls)
|
||||
|
||||
enabled = _service(enable_p5_b1_tool_recall_gate=True)
|
||||
|
||||
async def forbidden_search(*_args, **_kwargs):
|
||||
raise AssertionError("tool gate must stop before search")
|
||||
|
||||
enabled.search = forbidden_search
|
||||
self.assertEqual(
|
||||
{"ok": False, "error": "p5_attestation_missing", "memories": []},
|
||||
asyncio.run(enabled.tool_recall(object(), "needle")),
|
||||
)
|
||||
|
||||
def test_archive_gate_stops_search_evidence_and_timeline_before_data_reads(self):
|
||||
service = _service(enable_p5_b2_archive_read_gate=True)
|
||||
|
||||
async def forbidden(*_args, **_kwargs):
|
||||
raise AssertionError("archive gate must stop before a data read")
|
||||
|
||||
service.search = forbidden
|
||||
service.store = SimpleNamespace(get_memories_by_ids=forbidden, get_timeline_event=forbidden)
|
||||
searched = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"}))
|
||||
evidence = asyncio.run(service.bridge_get_archive_evidence(["m1"], scope={"scope": "private"}))
|
||||
context = asyncio.run(
|
||||
service.bridge_get_archive_session_context(
|
||||
scope={"scope": "private"}, start_at="2026-01-01T00:00:00+00:00", end_at="2026-02-01T00:00:00+00:00"
|
||||
)
|
||||
)
|
||||
self.assertEqual(["p5_attestation_missing"], searched["warnings"])
|
||||
self.assertEqual(([], [], ["p5_attestation_missing"]), (evidence["memories"], evidence["excluded"], evidence["warnings"]))
|
||||
self.assertEqual(([], ["p5_attestation_missing"]), (context["items"], context["warnings"]))
|
||||
|
||||
def test_archive_default_off_reaches_legacy_search(self):
|
||||
service = _service()
|
||||
service.search_calls = 0
|
||||
|
||||
async def search(*_args, **_kwargs):
|
||||
service.search_calls += 1
|
||||
return []
|
||||
|
||||
service.search = search
|
||||
service.archive_visibility_policy = lambda: object()
|
||||
result = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"}))
|
||||
self.assertEqual([], result["memories"])
|
||||
self.assertEqual(1, service.search_calls)
|
||||
|
||||
def test_cross_user_gate_returns_empty_without_delegating_to_archive(self):
|
||||
service = _service(enable_p5_b2_cross_user_read_gate=True)
|
||||
|
||||
async def forbidden(*_args, **_kwargs):
|
||||
raise AssertionError("cross-user gate must stop before archive delegation")
|
||||
|
||||
service.bridge_search_archive_memories = forbidden
|
||||
result = asyncio.run(
|
||||
service.bridge_query_user_facts("needle", scope={"scope": "private", "user_id": "u1"})
|
||||
)
|
||||
self.assertEqual(([], [], ["p5_attestation_missing"]), (result["memories"], result["excluded"], result["warnings"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
229
tests/test_p5_provenance.py
普通文件
229
tests/test_p5_provenance.py
普通文件
|
|
@ -0,0 +1,229 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from core.p5_provenance import (
|
||||
COMPANION_ATTESTATION_ISSUER,
|
||||
P3_CONTRACT_FINGERPRINT,
|
||||
P3_CONTRACT_NAME,
|
||||
P3_CONTRACT_VERSION,
|
||||
PROVENANCE_CONTRACT_NAME,
|
||||
PROVENANCE_CONTRACT_VERSION,
|
||||
apply_planned_operation,
|
||||
contract_descriptor,
|
||||
contract_fingerprint,
|
||||
legacy_unresolved,
|
||||
observed_from_companion_snapshot,
|
||||
plan_legacy_migration,
|
||||
plan_owner_recovery,
|
||||
rollback_planned_operation,
|
||||
validate_provenance_record,
|
||||
)
|
||||
|
||||
|
||||
def _hash(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CompanionStyleSnapshot:
|
||||
schema_version: str
|
||||
contract_name: str
|
||||
contract_version: str
|
||||
contract_fingerprint: str
|
||||
issuer: str
|
||||
issuer_epoch: str
|
||||
authority_attestation_ref_hash: str
|
||||
request_hash: str
|
||||
session_hash: str
|
||||
source_kind: str
|
||||
source_trust: str
|
||||
firewall_status: str
|
||||
disposition: str
|
||||
reason_codes: tuple[str, ...]
|
||||
source_event_ref_hash: str
|
||||
derived_from_ref_hash: str
|
||||
provenance_state: str
|
||||
p3_contract_name: str
|
||||
p3_contract_version: str
|
||||
p3_contract_fingerprint: str
|
||||
sink: str
|
||||
|
||||
|
||||
def _snapshot(**overrides):
|
||||
value = {
|
||||
"schema_version": "ops.p5.attestation.v1",
|
||||
"contract_name": PROVENANCE_CONTRACT_NAME,
|
||||
"contract_version": PROVENANCE_CONTRACT_VERSION,
|
||||
"contract_fingerprint": contract_fingerprint(),
|
||||
"issuer": COMPANION_ATTESTATION_ISSUER,
|
||||
"issuer_epoch": _hash("issuer-epoch-1"),
|
||||
"p3_contract_name": P3_CONTRACT_NAME,
|
||||
"p3_contract_version": P3_CONTRACT_VERSION,
|
||||
"p3_contract_fingerprint": P3_CONTRACT_FINGERPRINT,
|
||||
"source_kind": "forwarded_text",
|
||||
"source_trust": "T3",
|
||||
"firewall_status": "sanitized",
|
||||
"disposition": "shadow_quarantine",
|
||||
"reason_codes": ("p5b_attested",),
|
||||
"source_event_ref_hash": _hash("source-event-1"),
|
||||
"authority_attestation_ref_hash": _hash("companion-attestation-1"),
|
||||
"request_hash": _hash("request-1"),
|
||||
"session_hash": _hash("session-1"),
|
||||
"derived_from_ref_hash": "",
|
||||
"provenance_state": "observed",
|
||||
"sink": "memory_recall",
|
||||
}
|
||||
value.update(overrides)
|
||||
return value
|
||||
|
||||
|
||||
class P5ProvenanceTests(unittest.TestCase):
|
||||
def test_descriptor_and_legacy_record_are_json_serializable(self):
|
||||
descriptor = contract_descriptor()
|
||||
expected = {
|
||||
"contract_name": "ops.p5.provenance.v1",
|
||||
"contract_version": "1.0",
|
||||
"record_fields": [
|
||||
"contract_name", "contract_version", "contract_fingerprint", "memory_id",
|
||||
"source_kind", "source_trust", "firewall_status", "source_event_ref_hash",
|
||||
"authority_attestation_ref_hash", "provenance_state", "migration_operation_ref",
|
||||
"recovery_operation_ref", "record_revision",
|
||||
],
|
||||
"provenance_states": ["observed", "legacy_unresolved", "owner_recovered", "invalid"],
|
||||
"trust_levels": ["T0", "T1", "T2", "T3", "T4"],
|
||||
"source_kinds": [
|
||||
"policy_config", "verified_authorization", "current_user_intent", "forwarded_text",
|
||||
"quoted_text", "vision_summary", "tool_output", "web_extract", "memory_recall",
|
||||
"derived_summary", "legacy_memory", "unknown",
|
||||
],
|
||||
"firewall_statuses": [
|
||||
"allowed", "sanitized", "blocked", "rejected", "quarantined", "unavailable", "unknown",
|
||||
],
|
||||
"dispositions": ["allow", "shadow_quarantine", "deny_high_risk"],
|
||||
"hash_format": "sha256_hex_lower",
|
||||
"attestation_issuer": "private_companion",
|
||||
"p3_contract_name": P3_CONTRACT_NAME,
|
||||
"p3_contract_version": P3_CONTRACT_VERSION,
|
||||
}
|
||||
self.assertEqual(expected, descriptor)
|
||||
canonical = json.dumps(expected, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
||||
self.assertEqual(hashlib.sha256(canonical.encode("utf-8")).hexdigest(), contract_fingerprint())
|
||||
record = legacy_unresolved("memory-42", record_revision=4)
|
||||
self.assertEqual(PROVENANCE_CONTRACT_NAME, descriptor["contract_name"])
|
||||
self.assertEqual(PROVENANCE_CONTRACT_VERSION, descriptor["contract_version"])
|
||||
self.assertEqual(64, len(contract_fingerprint()))
|
||||
self.assertEqual("legacy_unresolved", record["provenance_state"])
|
||||
self.assertEqual(("unknown", "T4", "unknown"), (record["source_kind"], record["source_trust"], record["firewall_status"]))
|
||||
self.assertTrue(validate_provenance_record(record)["ok"])
|
||||
json.dumps({"descriptor": descriptor, "record": record}, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
def test_only_attested_exact_p3_snapshot_can_be_observed(self):
|
||||
observed = observed_from_companion_snapshot(
|
||||
"memory-42",
|
||||
_snapshot(
|
||||
source_event_ref_hash=f"sha256:{_hash('source-event-1').upper()}",
|
||||
authority_attestation_ref_hash=f"sha256:{_hash('companion-attestation-1').upper()}",
|
||||
),
|
||||
record_revision=7,
|
||||
)
|
||||
self.assertEqual("observed", observed["provenance_state"])
|
||||
self.assertTrue(validate_provenance_record(observed)["ok"])
|
||||
self.assertEqual(_hash("source-event-1"), observed["source_event_ref_hash"])
|
||||
self.assertEqual(_hash("companion-attestation-1"), observed["authority_attestation_ref_hash"])
|
||||
self.assertNotIn("disposition", observed)
|
||||
noncanonical_record = copy.deepcopy(observed)
|
||||
noncanonical_record["source_event_ref_hash"] = f"sha256:{_hash('source-event-1')}"
|
||||
self.assertFalse(validate_provenance_record(noncanonical_record)["ok"])
|
||||
|
||||
companion_style = _CompanionStyleSnapshot(**_snapshot())
|
||||
from_dataclass = observed_from_companion_snapshot("memory-43", companion_style)
|
||||
self.assertEqual("observed", from_dataclass["provenance_state"])
|
||||
|
||||
for name, override in (
|
||||
("pseudo source plugin", {"source_kind": "source_plugin"}),
|
||||
("p5 descriptor mismatch", {"contract_fingerprint": _hash("wrong-p5")}),
|
||||
("p3 mismatch", {"p3_contract_fingerprint": _hash("wrong-p3")}),
|
||||
("missing attestation", {"authority_attestation_ref_hash": ""}),
|
||||
("invalid trust", {"source_trust": "T9"}),
|
||||
("conflicting source hashes", {"source_hash": _hash("other-source")}),
|
||||
("same roles hash", {"authority_attestation_ref_hash": _hash("source-event-1")}),
|
||||
("status used as disposition", {"disposition": "sanitized"}),
|
||||
("disposition used as status", {"firewall_status": "shadow_quarantine"}),
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
rejected = observed_from_companion_snapshot("memory-42", _snapshot(**override))
|
||||
self.assertEqual("invalid", rejected["provenance_state"])
|
||||
|
||||
def test_preview_is_zero_write_and_does_not_mutate_or_capture_content(self):
|
||||
source = [{"memory_id": "memory-42", "record_revision": 4}]
|
||||
before = copy.deepcopy(source)
|
||||
preview = plan_legacy_migration(source, operation_ref_hash=_hash("migration-42"))
|
||||
self.assertEqual(before, source)
|
||||
self.assertEqual(("preview", True, 0), (preview["mode"], preview["readonly"], preview["write_count"]))
|
||||
self.assertEqual(1, len(preview["operations"]))
|
||||
self.assertEqual([], preview["error_codes"])
|
||||
self.assertEqual("legacy_unresolved", preview["operations"][0]["after_record"]["provenance_state"])
|
||||
|
||||
hostile = [{"memory_id": "memory-43", "record_revision": 0, "content": "do not persist this"}]
|
||||
rejected = plan_legacy_migration(hostile, operation_ref_hash=_hash("migration-43"))
|
||||
rendered = json.dumps(rejected, ensure_ascii=False, sort_keys=True)
|
||||
self.assertNotIn("do not persist this", rendered)
|
||||
self.assertNotIn("memory-43", rendered)
|
||||
self.assertEqual([], rejected["operations"])
|
||||
|
||||
def test_apply_retry_and_rollback_are_revision_guarded(self):
|
||||
preview = plan_legacy_migration(
|
||||
[{"memory_id": "memory-42", "record_revision": 4}], operation_ref_hash=_hash("migration-42")
|
||||
)
|
||||
operation = preview["operations"][0]
|
||||
current = {"memory_id": "memory-42", "record_revision": 4}
|
||||
applied = apply_planned_operation(current, operation)
|
||||
self.assertEqual((True, "applied", True), (applied["ok"], applied["status"], applied["changed"]))
|
||||
retry = apply_planned_operation(applied["record"], operation)
|
||||
self.assertEqual((True, "idempotent", False), (retry["ok"], retry["status"], retry["changed"]))
|
||||
|
||||
concurrent = copy.deepcopy(applied["record"])
|
||||
concurrent["record_revision"] += 1
|
||||
rejected = rollback_planned_operation(concurrent, operation)
|
||||
self.assertEqual((False, "revision_conflict"), (rejected["ok"], rejected["status"]))
|
||||
rollback = rollback_planned_operation(applied["record"], operation)
|
||||
self.assertEqual((True, "rolled_back", True), (rollback["ok"], rollback["status"], rollback["removed"]))
|
||||
self.assertIsNone(rollback["record"])
|
||||
|
||||
def test_owner_recovery_requires_companion_evidence_and_keeps_operation_ref(self):
|
||||
legacy = legacy_unresolved("memory-42", record_revision=2)
|
||||
planned = plan_owner_recovery(
|
||||
legacy, _snapshot(), recovery_operation_ref_hash=_hash("recovery-42")
|
||||
)
|
||||
self.assertTrue(planned["ok"])
|
||||
after = planned["operation"]["after_record"]
|
||||
self.assertEqual("owner_recovered", after["provenance_state"])
|
||||
self.assertEqual(_hash("recovery-42"), after["recovery_operation_ref"])
|
||||
|
||||
rejected = plan_owner_recovery(
|
||||
legacy,
|
||||
_snapshot(p3_contract_fingerprint=_hash("wrong-p3")),
|
||||
recovery_operation_ref_hash=_hash("recovery-43"),
|
||||
)
|
||||
self.assertFalse(rejected["ok"])
|
||||
self.assertEqual(["companion_snapshot_invalid"], rejected["error_codes"])
|
||||
|
||||
def test_record_validation_rejects_raw_source_ids_and_bodies_without_echo(self):
|
||||
record = legacy_unresolved("memory-42")
|
||||
record["source_event_id"] = "raw-event-99"
|
||||
record["content"] = "hostile body"
|
||||
result = validate_provenance_record(record)
|
||||
self.assertFalse(result["ok"])
|
||||
rendered = json.dumps(result, ensure_ascii=False, sort_keys=True)
|
||||
self.assertNotIn("raw-event-99", rendered)
|
||||
self.assertNotIn("hostile body", rendered)
|
||||
self.assertIn("forbidden_raw_field", result["error_codes"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from core.bridge import MemoryCompanionBridge
|
||||
from core.models import MemoryRecord
|
||||
from core.p5_provenance_persistence import P5ProvenanceMigration
|
||||
from core.service import MemoryCompanionService
|
||||
from core.store import MemoryStore
|
||||
|
||||
|
||||
class _MigrationDouble:
|
||||
async def preview(self, *, limit: int):
|
||||
return {"mode": "preview", "ok": True, "limit": limit}
|
||||
|
||||
async def apply(self, *, operation_id: str, limit: int):
|
||||
return {"mode": "apply", "ok": True, "operation_id": operation_id, "limit": limit}
|
||||
|
||||
async def rollback(self, operation_id: str):
|
||||
return {"mode": "rollback", "ok": True, "operation_id": operation_id}
|
||||
|
||||
|
||||
class P5ProvenancePersistenceTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.store = MemoryStore(Path(self.temp.name) / "memory.db")
|
||||
self.store.initialize()
|
||||
self.migration = P5ProvenanceMigration(self.store)
|
||||
|
||||
def tearDown(self):
|
||||
self.store.close()
|
||||
self.temp.cleanup()
|
||||
|
||||
def _seed(self, identifier: str, *, source_plugin: str = "memory_companion") -> str:
|
||||
return asyncio.run(
|
||||
self.store.insert_memory(
|
||||
MemoryRecord(
|
||||
id=identifier,
|
||||
memory_type="observation",
|
||||
scope="private",
|
||||
session_id="private:test",
|
||||
content=f"memory {identifier}",
|
||||
evidence="test evidence",
|
||||
source_plugin=source_plugin,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def test_schema_is_additive_and_preview_is_zero_write(self):
|
||||
self._seed("memory-001")
|
||||
tables = {
|
||||
row["name"]
|
||||
for row in self.store._conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'p5_provenance_%'"
|
||||
).fetchall()
|
||||
}
|
||||
self.assertEqual(
|
||||
{
|
||||
"p5_provenance_records",
|
||||
"p5_provenance_migration_runs",
|
||||
"p5_provenance_migration_backups",
|
||||
},
|
||||
tables,
|
||||
)
|
||||
before_changes = self.store._conn.total_changes
|
||||
preview = asyncio.run(self.migration.preview(limit=50))
|
||||
self.assertEqual(("preview", True, True, 1, 0), (
|
||||
preview["mode"], preview["ok"], preview["readonly"], preview["count"], preview["write_count"],
|
||||
))
|
||||
self.assertEqual(before_changes, self.store._conn.total_changes)
|
||||
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
|
||||
|
||||
def test_apply_creates_legacy_records_and_backup_manifest_idempotently(self):
|
||||
self._seed("memory-001", source_plugin="source_plugin")
|
||||
self._seed("memory-002")
|
||||
result = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50))
|
||||
self.assertEqual(("apply", True, 2, False, 2), (
|
||||
result["mode"], result["ok"], result["changed"], result["deduplicated"], result["backup_count"],
|
||||
))
|
||||
backup_path = Path(result["backup_path"])
|
||||
self.assertTrue(backup_path.is_file())
|
||||
backup_connection = sqlite3.connect(backup_path)
|
||||
try:
|
||||
self.assertEqual(0, backup_connection.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
|
||||
finally:
|
||||
backup_connection.close()
|
||||
rows = self.store._conn.execute(
|
||||
"SELECT source_kind, source_trust, firewall_status, provenance_state, source_event_ref_hash, authority_attestation_ref_hash FROM p5_provenance_records ORDER BY memory_id"
|
||||
).fetchall()
|
||||
self.assertEqual(2, len(rows))
|
||||
for row in rows:
|
||||
self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved", "", ""), tuple(row))
|
||||
manifest = self.store._conn.execute(
|
||||
"SELECT before_exists, before_digest, before_revision, after_digest, after_revision, apply_state FROM p5_provenance_migration_backups ORDER BY memory_id"
|
||||
).fetchall()
|
||||
self.assertEqual(2, len(manifest))
|
||||
self.assertTrue(all(tuple(row)[0:3] == (0, "", 0) and row[4] == 1 and row[5] == "applied" for row in manifest))
|
||||
retry = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50))
|
||||
self.assertEqual((True, 0, True, 2), (retry["ok"], retry["changed"], retry["deduplicated"], retry["backup_count"]))
|
||||
self.assertEqual(str(backup_path), retry["backup_path"])
|
||||
|
||||
def test_invalid_or_failed_backup_leaves_no_run_or_provenance_mutation(self):
|
||||
self._seed("memory-001")
|
||||
backup_glob = "memory.backup.*.before_p5_provenance_b0.*.db"
|
||||
before_paths = set(Path(self.temp.name).glob(backup_glob))
|
||||
invalid = asyncio.run(self.migration.apply(operation_id="", limit=50))
|
||||
self.assertEqual("operation_id_required", invalid["error_code"])
|
||||
self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob)))
|
||||
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0])
|
||||
|
||||
original_backup = self.store.backup
|
||||
self.store.backup = lambda suffix="": (_ for _ in ()).throw(RuntimeError("backup unavailable"))
|
||||
try:
|
||||
failed = asyncio.run(self.migration.apply(operation_id="p5-b0-backup-fail", limit=50))
|
||||
finally:
|
||||
self.store.backup = original_backup
|
||||
self.assertEqual((False, "physical_backup_failed", 0), (failed["ok"], failed["error_code"], failed["changed"]))
|
||||
self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob)))
|
||||
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0])
|
||||
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
|
||||
|
||||
def test_rollback_deletes_only_untouched_operation_owned_records(self):
|
||||
self._seed("memory-001")
|
||||
asyncio.run(self.migration.apply(operation_id="p5-b0-rollback-1", limit=50))
|
||||
rollback = asyncio.run(self.migration.rollback("p5-b0-rollback-1"))
|
||||
self.assertEqual(("rollback", True, 1, 0), (rollback["mode"], rollback["ok"], rollback["restored"], rollback["conflicts"]))
|
||||
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
|
||||
duplicate = asyncio.run(self.migration.rollback("p5-b0-rollback-1"))
|
||||
self.assertTrue(duplicate["deduplicated"])
|
||||
|
||||
def test_rollback_cas_conflict_preserves_independent_newer_record(self):
|
||||
self._seed("memory-001")
|
||||
asyncio.run(self.migration.apply(operation_id="p5-b0-conflict-1", limit=50))
|
||||
self.store._conn.execute(
|
||||
"UPDATE p5_provenance_records SET record_revision=record_revision+1 WHERE memory_id='memory-001'"
|
||||
)
|
||||
self.store._conn.commit()
|
||||
rollback = asyncio.run(self.migration.rollback("p5-b0-conflict-1"))
|
||||
self.assertEqual((False, 0, 1), (rollback["ok"], rollback["restored"], rollback["conflicts"]))
|
||||
record = self.store._conn.execute(
|
||||
"SELECT record_revision, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'"
|
||||
).fetchone()
|
||||
self.assertEqual((2, "legacy_unresolved"), tuple(record))
|
||||
backup = self.store._conn.execute(
|
||||
"SELECT rollback_state FROM p5_provenance_migration_backups WHERE operation_id='p5-b0-conflict-1' AND memory_id='memory-001'"
|
||||
).fetchone()
|
||||
self.assertEqual("conflict", backup[0])
|
||||
|
||||
def test_forged_source_and_p3_metadata_never_upgrades_legacy_backfill(self):
|
||||
self._seed("memory-001", source_plugin="source_plugin")
|
||||
self.store._conn.execute(
|
||||
"UPDATE memories SET metadata=? WHERE id='memory-001'",
|
||||
('{"source_kind":"source_plugin","p3_contract_fingerprint":"forged"}',),
|
||||
)
|
||||
self.store._conn.commit()
|
||||
asyncio.run(self.migration.apply(operation_id="p5-b0-forged-1", limit=50))
|
||||
record = self.store._conn.execute(
|
||||
"SELECT source_kind, source_trust, firewall_status, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'"
|
||||
).fetchone()
|
||||
self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved"), tuple(record))
|
||||
|
||||
def test_service_and_bridge_migration_wrappers_and_unavailable_bridge(self):
|
||||
service = object.__new__(MemoryCompanionService)
|
||||
service.p5_provenance_migration = _MigrationDouble()
|
||||
self.assertEqual("preview", asyncio.run(service.preview_p5_provenance_migration(limit=7))["mode"])
|
||||
self.assertEqual("op-1", asyncio.run(service.apply_p5_provenance_migration(operation_id="op-1", limit=7))["operation_id"])
|
||||
self.assertEqual("rollback", asyncio.run(service.rollback_p5_provenance_migration("op-1"))["mode"])
|
||||
|
||||
unavailable = MemoryCompanionBridge(object())
|
||||
result = asyncio.run(unavailable.preview_p5_provenance_migration(limit=7))
|
||||
self.assertEqual((False, "bridge_method_unavailable"), (result["ok"], result["error_code"]))
|
||||
applied = asyncio.run(unavailable.apply_p5_provenance_migration(operation_id="op-1", limit=7))
|
||||
self.assertEqual((False, "bridge_method_unavailable", 0), (applied["ok"], applied["error_code"], applied["changed"]))
|
||||
rolled_back = asyncio.run(unavailable.rollback_p5_provenance_migration("op-1"))
|
||||
self.assertEqual((False, "bridge_method_unavailable", 0), (rolled_back["ok"], rolled_back["error_code"], rolled_back["restored"]))
|
||||
self.assertFalse(unavailable.probe_p5_provenance_migration_capabilities()["available"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
正在加载…
添加表格
在新工单中引用