From 274c9b95ea82f730e9f0564a3a46525c474417f1 Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Thu, 30 Jul 2026 01:12:07 +0800 Subject: [PATCH] feat(p5): add provenance isolation gates --- core/bridge.py | 69 ++ core/p5_b1_recall_gate.py | 139 ++++ core/p5_provenance.py | 799 ++++++++++++++++++++ core/p5_provenance_persistence.py | 468 ++++++++++++ core/retrieval.py | 33 +- core/service.py | 179 ++++- core/store.py | 203 ++++- tests/test_p5_b1_candidate_gate.py | 263 +++++++ tests/test_p5_b1_recall_gate.py | 337 +++++++++ tests/test_p5_b1_unverifiable_read_gates.py | 117 +++ tests/test_p5_provenance.py | 229 ++++++ tests/test_p5_provenance_persistence.py | 184 +++++ 12 files changed, 3008 insertions(+), 12 deletions(-) create mode 100644 core/p5_b1_recall_gate.py create mode 100644 core/p5_provenance.py create mode 100644 core/p5_provenance_persistence.py create mode 100644 tests/test_p5_b1_candidate_gate.py create mode 100644 tests/test_p5_b1_recall_gate.py create mode 100644 tests/test_p5_b1_unverifiable_read_gates.py create mode 100644 tests/test_p5_provenance.py create mode 100644 tests/test_p5_provenance_persistence.py diff --git a/core/bridge.py b/core/bridge.py index ec57442..a23cc44 100644 --- a/core/bridge.py +++ b/core/bridge.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime, timezone +import threading from typing import Any from zoneinfo import ZoneInfo @@ -15,6 +16,10 @@ from .unified_person_contract import ( ) from .person_memory_contract import capabilities as person_memory_capabilities, consume_person_memory_envelope from .context_orchestration import contract_descriptor, contract_fingerprint +from .p5_provenance import ( + contract_descriptor as p5_provenance_contract_descriptor, + contract_fingerprint as p5_provenance_contract_fingerprint, +) LOCAL_TZ = ZoneInfo("Asia/Shanghai") @@ -43,6 +48,28 @@ class MemoryCompanionBridge: def __init__(self, plugin: Any): self._plugin = plugin + self._p5_attestation_verifier: Any = None + self._p5_attestation_verifier_bound = False + self._p5_attestation_verifier_lock = threading.RLock() + + def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool: + """Pin the first in-process P5 verifier; replacement is forbidden.""" + + if not callable(consume_callable): + return False + with self._p5_attestation_verifier_lock: + if self._p5_attestation_verifier_bound: + return False + target_binder = getattr(self._plugin, "bind_p5_attestation_verifier", None) + if callable(target_binder): + try: + if target_binder(consume_callable) is not True: + return False + except Exception: + return False + self._p5_attestation_verifier = consume_callable + self._p5_attestation_verifier_bound = True + return True def _trusted_session_context(self, session_context: SessionContext | dict[str, Any] | None) -> SessionContext | dict[str, Any] | None: """Normalize a legacy dict at the public bridge boundary. @@ -164,6 +191,23 @@ class MemoryCompanionBridge: result["operations"] = list(methods); result["operation_availability"] = operation_availability return result + def probe_p5_provenance_migration_capabilities(self) -> dict[str, Any]: + methods = { + "preview": "preview_p5_provenance_migration", + "apply": "apply_p5_provenance_migration", + "rollback": "rollback_p5_provenance_migration", + } + availability = {name: callable(getattr(self._plugin, method, None)) for name, method in methods.items()} + return { + "available": all(availability.values()), + "mode": "migration_only", + "runtime_sink_behavior": "not_connected", + "contract": p5_provenance_contract_descriptor(), + "contract_fingerprint": p5_provenance_contract_fingerprint(), + "operations": list(methods), + "operation_availability": availability, + } + def consume_person_memory_envelope(self, envelope: dict[str, Any] | None, *, companion_available: bool = True) -> dict[str, Any]: return consume_person_memory_envelope(envelope, companion_available=companion_available) @@ -182,6 +226,27 @@ class MemoryCompanionBridge: if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"} return await getter(migration_id) + async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]: + getter = getattr(self._plugin, "preview_p5_provenance_migration", None) + if not callable(getter): + return {"mode": "preview", "ok": False, "readonly": True, "error_code": "bridge_method_unavailable", "count": 0} + result = await getter(limit=limit) + return result if isinstance(result, dict) else {"mode": "preview", "ok": False, "readonly": True, "error_code": "invalid_bridge_response", "count": 0} + + async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]: + getter = getattr(self._plugin, "apply_p5_provenance_migration", None) + if not callable(getter): + return {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "changed": 0} + result = await getter(operation_id=operation_id, limit=limit) + return result if isinstance(result, dict) else {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "changed": 0} + + async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]: + getter = getattr(self._plugin, "rollback_p5_provenance_migration", None) + if not callable(getter): + return {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "restored": 0, "conflicts": 0} + result = await getter(operation_id) + return result if isinstance(result, dict) else {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "restored": 0, "conflicts": 0} + async def correct_person_memory(self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord) -> dict[str, Any]: getter = getattr(self._plugin, "bridge_correct_person_memory", None) if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"} @@ -702,6 +767,7 @@ class MemoryCompanionBridge: max_chars: int | None = None, companion_bot_mood: str = "", companion_bot_energy: float = 0.0, + p5_b1_bridge_invocation: Any = None, ) -> str: return await self._plugin.bridge_compose_injection( query, @@ -710,6 +776,7 @@ class MemoryCompanionBridge: max_chars=max_chars, companion_bot_mood=companion_bot_mood, companion_bot_energy=companion_bot_energy, + p5_b1_bridge_invocation=p5_b1_bridge_invocation, ) def get_reply_delay( @@ -739,6 +806,7 @@ class MemoryCompanionBridge: companion_bot_mood: str = "", companion_bot_energy: float = 0.0, retrieval_profile: str = "", + p5_b1_bridge_invocation: Any = None, ) -> str: return await self._plugin.bridge_compose_context( query=query, @@ -748,6 +816,7 @@ class MemoryCompanionBridge: companion_bot_mood=companion_bot_mood, companion_bot_energy=companion_bot_energy, retrieval_profile=retrieval_profile, + p5_b1_bridge_invocation=p5_b1_bridge_invocation, ) async def create_archive_memory( diff --git a/core/p5_b1_recall_gate.py b/core/p5_b1_recall_gate.py new file mode 100644 index 0000000..0e6be9d --- /dev/null +++ b/core/p5_b1_recall_gate.py @@ -0,0 +1,139 @@ +"""P5-B1 attestation gates for normal recall and Bridge serialization. + +The handle carrier is deliberately untrusted and private to the current +request/event runtime objects. This module never serializes it, stores it, +or interprets a source trust label. Authenticity belongs solely to the +one-shot verifier pinned by ``MemoryCompanionBridge``. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Callable + + +P5_B1_ATTESTATION_HANDLES_ATTRIBUTE = "private_companion_p5_b1_attestation_handles" +MEMORY_RECALL_SINK = "memory_recall" +BRIDGE_SERIALIZATION_SINK = "bridge_serialization" +_BRIDGE_INVOCATION_ARGUMENTS_METHOD = "_p5_b1_bridge_attestation_arguments" + + +def _carrier_handle(req: Any, event: Any) -> Any: + """Read a carrier value without copying it onto another object.""" + + for carrier in (req, event): + handles = getattr(carrier, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, None) + if isinstance(handles, Mapping) and MEMORY_RECALL_SINK in handles: + return handles.get(MEMORY_RECALL_SINK) + return None + + +def _snapshot_field(snapshot: Any, name: str) -> Any: + if isinstance(snapshot, Mapping): + return snapshot.get(name) + return getattr(snapshot, name, None) + + +def evaluate_normal_recall_gate( + verifier: Callable[..., Any] | None, + *, + req: Any, + event: Any, + p3_state: Any, +) -> str: + """Return a static safe reason or ``""`` when normal recall may proceed. + + The pinned verifier receives the original object identities. It may + consume a Companion opaque handle only once; any error, missing carrier, + unavailable P3 anchor, or non-allow result fails closed without retaining + its error or snapshot payload. + """ + + if not callable(verifier): + return "p5_b1_verifier_unavailable" + if p3_state is None: + return "p5_b1_p3_unavailable" + handle = _carrier_handle(req, event) + if handle is None: + return "p5_b1_handle_missing" + try: + snapshot = verifier(handle, req, event, p3_state, MEMORY_RECALL_SINK) + except Exception: + return "p5_b1_verifier_rejected" + if snapshot is None: + return "p5_b1_verifier_rejected" + if _snapshot_field(snapshot, "sink") != MEMORY_RECALL_SINK: + return "p5_b1_sink_mismatch" + if _snapshot_field(snapshot, "disposition") != "allow": + return "p5_b1_disposition_denied" + return "" + + +def _bridge_invocation_arguments(invocation: Any) -> tuple[Any, Any, Any, Any] | None: + """Read live anchors from an opaque invocation without retaining them. + + A mapping, serialized reconstruction, or arbitrary payload cannot be an + invocation. Even an object that mimics the private method cannot create a + valid authorization because the pinned verifier still receives and checks + the opaque issuer handle plus every exact object identity. + """ + + if invocation is None or isinstance(invocation, Mapping): + return None + argument_getter = getattr(invocation, _BRIDGE_INVOCATION_ARGUMENTS_METHOD, None) + if not callable(argument_getter): + return None + try: + arguments = argument_getter() + except Exception: + return None + if type(arguments) is not tuple or len(arguments) != 4: + return None + return arguments + + +def evaluate_bridge_serialization_gate( + verifier: Callable[..., Any] | None, + *, + invocation: Any, +) -> str: + """Return a static reason unless an opaque Bridge invocation is allowed. + + The call is deliberately separate from normal recall. The actual pinned + verifier is always called by Memory with its fixed ``bridge_serialization`` + sink; the invocation never supplies a sink, a snapshot, a SessionContext, + or a persistence payload. + """ + + if not callable(verifier): + return "p5_b1_bridge_verifier_unavailable" + arguments = _bridge_invocation_arguments(invocation) + if arguments is None: + return "p5_b1_bridge_invocation_invalid" + handle, request, event, p3_state = arguments + if p3_state is None: + return "p5_b1_bridge_p3_unavailable" + if request is None or event is None: + return "p5_b1_bridge_carrier_missing" + if handle is None: + return "p5_b1_bridge_handle_missing" + try: + snapshot = verifier(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK) + except Exception: + return "p5_b1_bridge_verifier_rejected" + if snapshot is None: + return "p5_b1_bridge_verifier_rejected" + if _snapshot_field(snapshot, "sink") != BRIDGE_SERIALIZATION_SINK: + return "p5_b1_bridge_sink_mismatch" + if _snapshot_field(snapshot, "disposition") != "allow": + return "p5_b1_bridge_disposition_denied" + return "" + + +__all__ = [ + "BRIDGE_SERIALIZATION_SINK", + "MEMORY_RECALL_SINK", + "P5_B1_ATTESTATION_HANDLES_ATTRIBUTE", + "evaluate_bridge_serialization_gate", + "evaluate_normal_recall_gate", +] diff --git a/core/p5_provenance.py b/core/p5_provenance.py new file mode 100644 index 0000000..53092b2 --- /dev/null +++ b/core/p5_provenance.py @@ -0,0 +1,799 @@ +"""P5-B metadata-only provenance and migration planning contract. + +This module deliberately has no dependency on MemoryRecord, Store, Service, +Bridge, or a database. It is the small, deterministic contract that a later +integration may use to *plan* provenance backfill and guarded rollback. It +does not classify content, read a body, perform I/O, or grant a live recall or +tool permission. + +Only :func:`observed_from_companion_snapshot` can create an ``observed`` +record. It requires the frozen P3 carrier reference and a Companion +attestation reference. Historical records are represented explicitly as +``legacy_unresolved`` with ``unknown`` / ``T4`` provenance instead of being +silently upgraded. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from copy import deepcopy +from dataclasses import asdict, is_dataclass +import hashlib +import json +import re +from typing import Any + + +PROVENANCE_CONTRACT_NAME = "ops.p5.provenance.v1" +PROVENANCE_CONTRACT_VERSION = "1.0" + +# P3 remains owned by PrivateCompanion. This consumer duplicates only its +# already-frozen identity, never its state machine or source classifier. +P3_CONTRACT_NAME = "ops.context_orchestration.v1" +P3_CONTRACT_VERSION = "1.0" +P3_CONTRACT_FINGERPRINT = ( + "3bb7a12af05bb4d9a47cef9f31e78752cec512b90b1d50ae49feef54470974f8" +) +COMPANION_ATTESTATION_ISSUER = "private_companion" +ATTESTATION_SCHEMA_VERSION = "ops.p5.attestation.v1" + +TRUST_LEVELS = ("T0", "T1", "T2", "T3", "T4") +PROVENANCE_STATES = ("observed", "legacy_unresolved", "owner_recovered", "invalid") +# Ordered tuples are part of the shared cross-repository descriptor. Trust +# must remain explicit: source kind and firewall status never imply it. +SOURCE_KINDS = ( + "policy_config", + "verified_authorization", + "current_user_intent", + "forwarded_text", + "quoted_text", + "vision_summary", + "tool_output", + "web_extract", + "memory_recall", + "derived_summary", + "legacy_memory", + "unknown", +) +FIREWALL_STATUSES = ( + "allowed", + "sanitized", + "blocked", + "rejected", + "quarantined", + "unavailable", + "unknown", +) +DISPOSITIONS = ("allow", "shadow_quarantine", "deny_high_risk") +HASH_FORMAT = "sha256_hex_lower" +_SOURCE_KIND_SET = frozenset(SOURCE_KINDS) +_ATTESTATION_SINKS = frozenset( + { + "prompt_forward_quote", + "prompt_vision_summary", + "memory_recall", + "bridge_serialization", + "tool_retrieval", + "external_export", + "cross_user_read", + } +) + +RECORD_FIELDS = ( + "contract_name", + "contract_version", + "contract_fingerprint", + "memory_id", + "source_kind", + "source_trust", + "firewall_status", + "source_event_ref_hash", + "authority_attestation_ref_hash", + "provenance_state", + "migration_operation_ref", + "recovery_operation_ref", + "record_revision", +) +OPERATION_FIELDS = ( + "contract_name", + "contract_version", + "contract_fingerprint", + "operation_kind", + "operation_ref_hash", + "memory_id", + "expected_revision", + "before_record_digest", + "after_record_digest", + "before_record", + "after_record", +) + +_TOKEN_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_MEMORY_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$") +_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +_FORBIDDEN_RAW_FIELDS = frozenset( + { + "body", + "content", + "text", + "message", + "raw_id", + "source_id", + "source_event_id", + "event_id", + "attestation_id", + "authority_attestation_id", + "request_id", + "session_id", + } +) +_SNAPSHOT_FIELDS = frozenset( + { + "schema_version", + "contract_name", + "contract_version", + "contract_fingerprint", + "issuer", + "issuer_epoch", + "p3_contract_name", + "p3_contract_version", + "p3_contract_fingerprint", + "source_kind", + "source_trust", + "trust", + "firewall_status", + "source_event_ref_hash", + "source_hash", + "authority_attestation_ref_hash", + "attestation_ref_hash", + # The following are bounded, non-authoritative diagnostic metadata + # emitted by the companion registry. They never enter a P5 record. + "disposition", + "reason_codes", + "request_hash", + "session_hash", + "derived_hash", + "derived_from_ref_hash", + "provenance_state", + "sink", + } +) + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + +def _digest(value: Mapping[str, Any]) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def contract_descriptor() -> dict[str, Any]: + """Return the frozen, JSON-serializable P5 provenance descriptor.""" + + return { + "contract_name": PROVENANCE_CONTRACT_NAME, + "contract_version": PROVENANCE_CONTRACT_VERSION, + "record_fields": list(RECORD_FIELDS), + "provenance_states": list(PROVENANCE_STATES), + "trust_levels": list(TRUST_LEVELS), + "source_kinds": list(SOURCE_KINDS), + "firewall_statuses": list(FIREWALL_STATUSES), + "dispositions": list(DISPOSITIONS), + "hash_format": HASH_FORMAT, + "attestation_issuer": COMPANION_ATTESTATION_ISSUER, + "p3_contract_name": P3_CONTRACT_NAME, + "p3_contract_version": P3_CONTRACT_VERSION, + } + + +def contract_fingerprint() -> str: + """Return a stable descriptor fingerprint without importing Companion.""" + + return _digest(contract_descriptor()) + + +def provenance_record_digest(record: Mapping[str, Any] | None) -> str: + """Return a digest only for an already-valid, metadata-only P5 record.""" + + checked = validate_provenance_record(record) + if not checked["ok"]: + return "" + return _digest(checked["record"]) + + +def _safe_memory_id(value: Any) -> str: + if isinstance(value, str) and _MEMORY_ID_RE.fullmatch(value): + return value + return "" + + +def _safe_hash(value: Any) -> str: + """Accept only a canonical stored hash (no prefix and lowercase).""" + + if isinstance(value, str) and _SHA256_RE.fullmatch(value) and value == value.lower(): + return value + return "" + + +def _normalize_input_hash(value: Any) -> str: + """Normalize a boundary hash; persisted records remain canonical only.""" + + if not isinstance(value, str): + return "" + raw = value.removeprefix("sha256:") + if _SHA256_RE.fullmatch(raw): + return raw.lower() + return "" + + +def _safe_revision(value: Any) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _known_token(value: Any, known: frozenset[str] | tuple[str, ...]) -> bool: + return isinstance(value, str) and value in known + + +def _has_forbidden_raw_field(value: Mapping[str, Any]) -> bool: + return any(key in value for key in _FORBIDDEN_RAW_FIELDS) + + +def _field_set_is_exact(value: Mapping[str, Any], fields: tuple[str, ...]) -> bool: + return set(value) == set(fields) + + +def _error_result(*codes: str) -> dict[str, Any]: + # Error codes are static, so they neither echo a hostile value nor retain + # prose/raw IDs supplied by a caller. + return {"ok": False, "error_codes": sorted(set(codes))} + + +def _blank_record(memory_id: Any = "", *, revision: int = 0, state: str = "invalid") -> dict[str, Any]: + return { + "contract_name": PROVENANCE_CONTRACT_NAME, + "contract_version": PROVENANCE_CONTRACT_VERSION, + "contract_fingerprint": contract_fingerprint(), + "memory_id": _safe_memory_id(memory_id), + "source_kind": "unknown", + "source_trust": "T4", + "firewall_status": "unknown", + "source_event_ref_hash": "", + "authority_attestation_ref_hash": "", + "provenance_state": state, + "migration_operation_ref": "", + "recovery_operation_ref": "", + "record_revision": revision if _safe_revision(revision) is not None else 0, + } + + +def validate_provenance_record(record: Mapping[str, Any] | None) -> dict[str, Any]: + """Validate a persisted P5 record without coercion or content parsing. + + The result contains only static error codes. In particular it never + returns an invalid input field, a source event ID, or message text. + """ + + if not isinstance(record, Mapping): + return _error_result("record_not_mapping") + errors: list[str] = [] + if _has_forbidden_raw_field(record): + errors.append("forbidden_raw_field") + if not _field_set_is_exact(record, RECORD_FIELDS): + errors.append("record_fields_mismatch") + if record.get("contract_name") != PROVENANCE_CONTRACT_NAME: + errors.append("contract_name_mismatch") + if record.get("contract_version") != PROVENANCE_CONTRACT_VERSION: + errors.append("contract_version_mismatch") + if record.get("contract_fingerprint") != contract_fingerprint(): + errors.append("contract_fingerprint_mismatch") + if not _safe_memory_id(record.get("memory_id")): + errors.append("memory_id_invalid") + if not _known_token(record.get("source_kind"), SOURCE_KINDS): + errors.append("source_kind_invalid") + if not _known_token(record.get("source_trust"), TRUST_LEVELS): + errors.append("source_trust_invalid") + if not _known_token(record.get("firewall_status"), FIREWALL_STATUSES): + errors.append("firewall_status_invalid") + if not _known_token(record.get("provenance_state"), PROVENANCE_STATES): + errors.append("provenance_state_invalid") + if _safe_revision(record.get("record_revision")) is None: + errors.append("record_revision_invalid") + + source_hash_value = record.get("source_event_ref_hash") + authority_hash_value = record.get("authority_attestation_ref_hash") + source_hash = _safe_hash(source_hash_value) + authority_hash = _safe_hash(authority_hash_value) + if source_hash_value != "" and not source_hash: + errors.append("source_event_ref_hash_invalid") + if authority_hash_value != "" and not authority_hash: + errors.append("authority_attestation_ref_hash_invalid") + migration_ref = record.get("migration_operation_ref") + recovery_ref = record.get("recovery_operation_ref") + if migration_ref != "" and not _safe_hash(migration_ref): + errors.append("migration_operation_ref_invalid") + if recovery_ref != "" and not _safe_hash(recovery_ref): + errors.append("recovery_operation_ref_invalid") + + state = record.get("provenance_state") + if state == "legacy_unresolved": + if (record.get("source_kind"), record.get("source_trust"), record.get("firewall_status")) != ( + "unknown", + "T4", + "unknown", + ): + errors.append("legacy_defaults_required") + if source_hash_value != "" or authority_hash_value != "": + errors.append("legacy_hashes_forbidden") + if not isinstance(migration_ref, str) or not isinstance(recovery_ref, str): + errors.append("operation_ref_type_invalid") + elif state == "observed" or state == "owner_recovered": + if record.get("source_kind") == "unknown": + errors.append("observed_source_kind_required") + if not source_hash or not authority_hash: + errors.append("attestation_hashes_required") + if source_hash and authority_hash and source_hash == authority_hash: + errors.append("attestation_hashes_conflict") + if state == "owner_recovered" and not _safe_hash(recovery_ref): + errors.append("recovery_operation_ref_required") + elif state == "invalid": + # Invalid is an explicit quarantine representation, never an accepted + # migration target. Its safe shape is still useful for caller logs. + if source_hash or authority_hash: + errors.append("invalid_hashes_forbidden") + if errors: + return _error_result(*errors) + return {"ok": True, "record": deepcopy(dict(record))} + + +def legacy_unresolved(memory_id: str, *, record_revision: int = 0) -> dict[str, Any]: + """Represent a historical item without pretending it has known trust.""" + + record = _blank_record(memory_id, revision=record_revision, state="legacy_unresolved") + if not record["memory_id"]: + return _blank_record(state="invalid") + return record + + +def _snapshot_hash(snapshot: Mapping[str, Any], *keys: str) -> tuple[str, str]: + present = [(key, snapshot.get(key)) for key in keys if key in snapshot] + if not present: + return "missing", "" + values = [_normalize_input_hash(value) for _, value in present] + if not all(values): + return "invalid", "" + if len(set(values)) != 1: + return "conflict", "" + return "present", values[0] + + +def _snapshot_label( + snapshot: Mapping[str, Any], known: frozenset[str] | tuple[str, ...], *keys: str +) -> tuple[str, str]: + present = [(key, snapshot.get(key)) for key in keys if key in snapshot] + if not present: + return "missing", "" + values = [value for _, value in present] + if not all(_known_token(value, known) for value in values): + return "invalid", "" + if len(set(values)) != 1: + return "conflict", "" + return "present", values[0] + + +def _snapshot_diagnostics_valid(snapshot: Mapping[str, Any]) -> bool: + """Allow only bounded, non-prose diagnostics that are not persisted.""" + + # A disposition is an attestation-side decision label, distinct from the + # upstream P2/P3 firewall status that is persisted in the record. + if "disposition" not in snapshot: + return False + disposition = snapshot["disposition"] + if not _known_token(disposition, DISPOSITIONS): + return False + if "reason_codes" in snapshot: + codes = snapshot["reason_codes"] + if not isinstance(codes, (list, tuple)) or len(codes) > 16: + return False + if not all(isinstance(code, str) and _TOKEN_RE.fullmatch(code) for code in codes): + return False + for key in ("request_hash", "session_hash"): + if key in snapshot and not _normalize_input_hash(snapshot[key]): + return False + derived_values = [ + snapshot[key] for key in ("derived_from_ref_hash", "derived_hash") if key in snapshot + ] + if derived_values: + if any(value != "" and not _normalize_input_hash(value) for value in derived_values): + return False + normalized = [_normalize_input_hash(value) for value in derived_values if value != ""] + if len(set(normalized)) > 1: + return False + if snapshot.get("provenance_state") != "observed": + return False + epoch = snapshot.get("issuer_epoch") + if not _normalize_input_hash(epoch): + return False + sink = snapshot.get("sink") + if not _known_token(sink, _ATTESTATION_SINKS): + return False + return True + + +def _snapshot_mapping(snapshot: Any) -> Mapping[str, Any] | None: + """Accept the immutable Companion dataclass without importing Companion.""" + + if isinstance(snapshot, Mapping): + return snapshot + if is_dataclass(snapshot): + value = asdict(snapshot) + return value if isinstance(value, Mapping) else None + return None + + +def observed_from_companion_snapshot( + memory_id: str, snapshot: Any, *, record_revision: int = 0 +) -> dict[str, Any]: + """Convert only a valid Companion-attested snapshot into ``observed``. + + This function accepts hashes and bounded labels only. A raw source event + ID, text body, or arbitrary P3 fingerprint rejects conversion and yields a + safe ``invalid`` record rather than an observed one. + """ + + safe_id = _safe_memory_id(memory_id) + revision = _safe_revision(record_revision) + snapshot_mapping = _snapshot_mapping(snapshot) + if not safe_id or revision is None or snapshot_mapping is None: + return _blank_record(safe_id, revision=revision or 0) + if ( + _has_forbidden_raw_field(snapshot_mapping) + or not set(snapshot_mapping).issubset(_SNAPSHOT_FIELDS) + or not _snapshot_diagnostics_valid(snapshot_mapping) + ): + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("schema_version") != ATTESTATION_SCHEMA_VERSION: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("issuer") != COMPANION_ATTESTATION_ISSUER: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("contract_name") != PROVENANCE_CONTRACT_NAME: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("contract_version") != PROVENANCE_CONTRACT_VERSION: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("contract_fingerprint") != contract_fingerprint(): + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("p3_contract_name") != P3_CONTRACT_NAME: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("p3_contract_version") != P3_CONTRACT_VERSION: + return _blank_record(safe_id, revision=revision) + if snapshot_mapping.get("p3_contract_fingerprint") != P3_CONTRACT_FINGERPRINT: + return _blank_record(safe_id, revision=revision) + + source_status, source_hash = _snapshot_hash( + snapshot_mapping, "source_event_ref_hash", "source_hash" + ) + authority_status, authority_hash = _snapshot_hash( + snapshot_mapping, "authority_attestation_ref_hash", "attestation_ref_hash" + ) + trust_status, source_trust = _snapshot_label(snapshot_mapping, TRUST_LEVELS, "source_trust", "trust") + source_kind = snapshot_mapping.get("source_kind") + firewall_status = snapshot_mapping.get("firewall_status") + if ( + source_status != "present" + or authority_status != "present" + or trust_status != "present" + or source_hash == authority_hash + or not _known_token(source_kind, _SOURCE_KIND_SET - {"unknown"}) + or not _known_token(source_trust, TRUST_LEVELS) + or not _known_token(firewall_status, FIREWALL_STATUSES) + ): + return _blank_record(safe_id, revision=revision) + + record = _blank_record(safe_id, revision=revision, state="observed") + record.update( + { + "source_kind": source_kind, + "source_trust": source_trust, + "firewall_status": firewall_status, + "source_event_ref_hash": source_hash, + "authority_attestation_ref_hash": authority_hash, + } + ) + return record if validate_provenance_record(record)["ok"] else _blank_record(safe_id, revision=revision) + + +def _record_or_none(value: Any) -> dict[str, Any] | None: + if value is None: + return None + result = validate_provenance_record(value if isinstance(value, Mapping) else None) + return deepcopy(result["record"]) if result["ok"] else None + + +def _operation( + *, + operation_kind: str, + operation_ref_hash: str, + before_record: Mapping[str, Any] | None, + after_record: Mapping[str, Any], +) -> dict[str, Any] | None: + before = _record_or_none(before_record) + after = _record_or_none(after_record) + reference = _safe_hash(operation_ref_hash) + if not _known_token(operation_kind, ("migration", "recovery")) or not reference or after is None: + return None + if before is not None and before["memory_id"] != after["memory_id"]: + return None + expected_revision = before["record_revision"] if before is not None else after["record_revision"] - 1 + if expected_revision < 0 or after["record_revision"] != expected_revision + 1: + return None + operation = { + "contract_name": PROVENANCE_CONTRACT_NAME, + "contract_version": PROVENANCE_CONTRACT_VERSION, + "contract_fingerprint": contract_fingerprint(), + "operation_kind": operation_kind, + "operation_ref_hash": reference, + "memory_id": after["memory_id"], + "expected_revision": expected_revision, + "before_record_digest": _digest(before) if before is not None else "", + "after_record_digest": _digest(after), + "before_record": before, + "after_record": after, + } + return operation if validate_operation(operation)["ok"] else None + + +def validate_operation(operation: Mapping[str, Any] | None) -> dict[str, Any]: + """Validate an I/O-free migration/recovery operation and its CAS guards.""" + + if not isinstance(operation, Mapping): + return _error_result("operation_not_mapping") + errors: list[str] = [] + if _has_forbidden_raw_field(operation): + errors.append("forbidden_raw_field") + if not _field_set_is_exact(operation, OPERATION_FIELDS): + errors.append("operation_fields_mismatch") + if operation.get("contract_name") != PROVENANCE_CONTRACT_NAME: + errors.append("contract_name_mismatch") + if operation.get("contract_version") != PROVENANCE_CONTRACT_VERSION: + errors.append("contract_version_mismatch") + if operation.get("contract_fingerprint") != contract_fingerprint(): + errors.append("contract_fingerprint_mismatch") + kind = operation.get("operation_kind") + if not _known_token(kind, ("migration", "recovery")): + errors.append("operation_kind_invalid") + if not _safe_hash(operation.get("operation_ref_hash")): + errors.append("operation_ref_invalid") + if not _safe_memory_id(operation.get("memory_id")): + errors.append("memory_id_invalid") + expected_revision = _safe_revision(operation.get("expected_revision")) + if expected_revision is None: + errors.append("expected_revision_invalid") + + before = operation.get("before_record") + after = operation.get("after_record") + before_record = _record_or_none(before) + after_record = _record_or_none(after) + if before is not None and before_record is None: + errors.append("before_record_invalid") + if after_record is None: + errors.append("after_record_invalid") + if before_record is not None and operation.get("before_record_digest") != _digest(before_record): + errors.append("before_digest_mismatch") + if before is None and operation.get("before_record_digest") != "": + errors.append("before_digest_required_empty") + if after_record is not None and operation.get("after_record_digest") != _digest(after_record): + errors.append("after_digest_mismatch") + if after_record is not None: + if operation.get("memory_id") != after_record["memory_id"]: + errors.append("memory_id_mismatch") + if expected_revision is not None and after_record["record_revision"] != expected_revision + 1: + errors.append("revision_progression_invalid") + required_ref = "migration_operation_ref" if kind == "migration" else "recovery_operation_ref" + if after_record.get(required_ref) != operation.get("operation_ref_hash"): + errors.append("operation_ref_not_bound") + if before_record is not None and after_record is not None: + if before_record["memory_id"] != after_record["memory_id"]: + errors.append("before_after_memory_mismatch") + if expected_revision is not None and before_record["record_revision"] != expected_revision: + errors.append("before_revision_mismatch") + if errors: + return _error_result(*errors) + return {"ok": True, "operation": deepcopy(dict(operation))} + + +def plan_legacy_migration( + records: Iterable[Mapping[str, Any]], *, operation_ref_hash: str +) -> dict[str, Any]: + """Build a read-only P5 legacy-backfill preview; never write or mutate. + + ``records`` are intentionally tiny metadata snapshots containing only a + ``memory_id`` and ``record_revision``. A valid existing P5 record may be + supplied instead and is skipped; Store integration will decide how to + obtain those snapshots in a future phase. + """ + + reference = _safe_hash(operation_ref_hash) + result: dict[str, Any] = { + "mode": "preview", + "readonly": True, + "write_count": 0, + "operation_ref_hash": reference, + "operations": [], + "skipped": [], + "error_codes": [], + } + if not reference: + result["error_codes"].append("operation_ref_invalid") + return result + if isinstance(records, (str, bytes)) or not isinstance(records, Iterable): + result["error_codes"].append("records_not_iterable") + return result + + for candidate in records: + if not isinstance(candidate, Mapping) or _has_forbidden_raw_field(candidate): + result["error_codes"].append("candidate_invalid") + continue + existing = _record_or_none(candidate) + if existing is not None: + result["skipped"].append({"memory_id": existing["memory_id"], "reason_code": "already_provenanced"}) + continue + allowed_base_fields = {"memory_id", "record_revision"} + if set(candidate) != allowed_base_fields: + result["error_codes"].append("candidate_fields_mismatch") + continue + memory_id = _safe_memory_id(candidate.get("memory_id")) + revision = _safe_revision(candidate.get("record_revision")) + if not memory_id or revision is None: + result["error_codes"].append("candidate_invalid") + continue + after = legacy_unresolved(memory_id, record_revision=revision + 1) + after["migration_operation_ref"] = reference + operation = _operation( + operation_kind="migration", + operation_ref_hash=reference, + before_record=None, + after_record=after, + ) + if operation is None: + result["error_codes"].append("operation_invalid") + else: + result["operations"].append(operation) + result["error_codes"] = sorted(set(result["error_codes"])) + return result + + +def _minimal_current(current: Mapping[str, Any] | None) -> tuple[str, int | None, dict[str, Any] | None, str]: + """Extract a safe CAS view; never accept a raw current-record payload.""" + + if not isinstance(current, Mapping) or _has_forbidden_raw_field(current): + return "", None, None, "current_invalid" + p5 = _record_or_none(current) + if p5 is not None: + return p5["memory_id"], p5["record_revision"], p5, "" + if set(current) != {"memory_id", "record_revision"}: + return "", None, None, "current_fields_mismatch" + return _safe_memory_id(current.get("memory_id")), _safe_revision(current.get("record_revision")), None, "" + + +def apply_planned_operation( + current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None +) -> dict[str, Any]: + """Project a guarded apply result without touching storage. + + Applying the same operation twice is idempotent. A different record or + revision returns a conflict instead of overwriting a concurrent update. + """ + + checked = validate_operation(operation) + if not checked["ok"]: + return {"ok": False, "mode": "apply", "status": "invalid_operation", "error_codes": checked["error_codes"]} + op = checked["operation"] + memory_id, revision, current_record, error = _minimal_current(current) + if error or not memory_id or revision is None: + return {"ok": False, "mode": "apply", "status": "invalid_current", "error_codes": [error or "current_invalid"]} + after = op["after_record"] + if current_record is not None and _digest(current_record) == op["after_record_digest"]: + return {"ok": True, "mode": "apply", "status": "idempotent", "record": deepcopy(after), "changed": False} + if memory_id != op["memory_id"] or revision != op["expected_revision"]: + return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["revision_or_memory_mismatch"]} + expected_before = op["before_record"] + if expected_before is None: + if current_record is not None: + return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["unexpected_existing_provenance"]} + elif current_record is None or _digest(current_record) != op["before_record_digest"]: + return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["before_record_mismatch"]} + return {"ok": True, "mode": "apply", "status": "applied", "record": deepcopy(after), "changed": True} + + +def rollback_planned_operation( + current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None +) -> dict[str, Any]: + """Project a CAS-protected rollback without overwriting later changes.""" + + checked = validate_operation(operation) + if not checked["ok"]: + return {"ok": False, "mode": "rollback", "status": "invalid_operation", "error_codes": checked["error_codes"]} + op = checked["operation"] + memory_id, revision, current_record, error = _minimal_current(current) + if error or current_record is None or revision is None: + return {"ok": False, "mode": "rollback", "status": "invalid_current", "error_codes": [error or "current_invalid"]} + if ( + memory_id != op["memory_id"] + or revision != op["after_record"]["record_revision"] + or _digest(current_record) != op["after_record_digest"] + ): + return {"ok": False, "mode": "rollback", "status": "revision_conflict", "error_codes": ["concurrent_update_protected"]} + before = op["before_record"] + return { + "ok": True, + "mode": "rollback", + "status": "rolled_back", + "record": deepcopy(before) if before is not None else None, + "removed": before is None, + "changed": True, + } + + +def plan_owner_recovery( + current: Mapping[str, Any], + snapshot: Mapping[str, Any], + *, + recovery_operation_ref_hash: str, +) -> dict[str, Any]: + """Plan, but do not apply, a Companion-attested owner recovery. + + Recovery is intentionally limited to a valid ``legacy_unresolved`` record. + A later owner-authority workflow must supply the Companion snapshot; this + pure contract has no owner command, storage, or authorization side effect. + """ + + existing = _record_or_none(current) + reference = _safe_hash(recovery_operation_ref_hash) + if existing is None or existing["provenance_state"] != "legacy_unresolved" or not reference: + return {"ok": False, "mode": "preview", "error_codes": ["recovery_precondition_failed"]} + after = observed_from_companion_snapshot( + existing["memory_id"], snapshot, record_revision=existing["record_revision"] + 1 + ) + if after["provenance_state"] != "observed": + return {"ok": False, "mode": "preview", "error_codes": ["companion_snapshot_invalid"]} + after["provenance_state"] = "owner_recovered" + after["migration_operation_ref"] = existing["migration_operation_ref"] + after["recovery_operation_ref"] = reference + operation = _operation( + operation_kind="recovery", + operation_ref_hash=reference, + before_record=existing, + after_record=after, + ) + if operation is None: + return {"ok": False, "mode": "preview", "error_codes": ["operation_invalid"]} + return {"ok": True, "mode": "preview", "readonly": True, "write_count": 0, "operation": operation} + + +__all__ = [ + "COMPANION_ATTESTATION_ISSUER", + "DISPOSITIONS", + "FIREWALL_STATUSES", + "HASH_FORMAT", + "P3_CONTRACT_FINGERPRINT", + "P3_CONTRACT_NAME", + "P3_CONTRACT_VERSION", + "PROVENANCE_CONTRACT_NAME", + "PROVENANCE_CONTRACT_VERSION", + "PROVENANCE_STATES", + "SOURCE_KINDS", + "TRUST_LEVELS", + "apply_planned_operation", + "contract_descriptor", + "contract_fingerprint", + "legacy_unresolved", + "observed_from_companion_snapshot", + "plan_legacy_migration", + "plan_owner_recovery", + "provenance_record_digest", + "rollback_planned_operation", + "validate_operation", + "validate_provenance_record", +] diff --git a/core/p5_provenance_persistence.py b/core/p5_provenance_persistence.py new file mode 100644 index 0000000..c60f5e2 --- /dev/null +++ b/core/p5_provenance_persistence.py @@ -0,0 +1,468 @@ +"""P5-B0 additive SQLite persistence and recoverable legacy backfill. + +This is deliberately a storage/migration boundary. It does not consume a +Companion attestation, change memory capture, alter retrieval, or evaluate a +live sink. The only records it creates are explicit ``legacy_unresolved`` +records, so existing memory columns can never be used to infer source trust. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from collections.abc import Mapping +from copy import deepcopy +from typing import Any, TYPE_CHECKING + +from .models import utc_now +from .p5_provenance import ( + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + contract_fingerprint, + legacy_unresolved, + provenance_record_digest, + validate_provenance_record, +) + +if TYPE_CHECKING: + from .store import MemoryStore + + +P5_PERSISTENCE_SCHEMA_VERSION = 1 +_OPERATION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$") +_RECORD_COLUMNS = ( + "memory_id", + "contract_name", + "contract_version", + "contract_fingerprint", + "source_kind", + "source_trust", + "firewall_status", + "source_event_ref_hash", + "authority_attestation_ref_hash", + "provenance_state", + "migration_operation_ref", + "recovery_operation_ref", + "record_revision", + "record_digest", +) + + +def ensure_p5_provenance_persistence_schema(connection: sqlite3.Connection) -> None: + """Create only additive P5-B0 tables and indexes. + + This function neither scans memories nor creates provenance records; that + work is possible only through the explicit migration apply operation. + """ + + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS p5_provenance_records ( + memory_id TEXT PRIMARY KEY, + contract_name TEXT NOT NULL, + contract_version TEXT NOT NULL, + contract_fingerprint TEXT NOT NULL, + source_kind TEXT NOT NULL, + source_trust TEXT NOT NULL, + firewall_status TEXT NOT NULL, + source_event_ref_hash TEXT NOT NULL DEFAULT '', + authority_attestation_ref_hash TEXT NOT NULL DEFAULT '', + provenance_state TEXT NOT NULL, + migration_operation_ref TEXT NOT NULL DEFAULT '', + recovery_operation_ref TEXT NOT NULL DEFAULT '', + record_revision INTEGER NOT NULL, + record_digest TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + FOREIGN KEY(memory_id) REFERENCES memories(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_p5_provenance_state + ON p5_provenance_records(provenance_state, source_trust, firewall_status); + + CREATE TABLE IF NOT EXISTS p5_provenance_migration_runs ( + operation_id TEXT PRIMARY KEY, + operation_ref_hash TEXT NOT NULL, + contract_name TEXT NOT NULL, + contract_version TEXT NOT NULL, + contract_fingerprint TEXT NOT NULL, + state TEXT NOT NULL, + requested_limit INTEGER NOT NULL DEFAULT 0, + candidate_count INTEGER NOT NULL DEFAULT 0, + applied_count INTEGER NOT NULL DEFAULT 0, + rollback_conflict_count INTEGER NOT NULL DEFAULT 0, + backup_path TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT NOT NULL DEFAULT '', + rolled_back_at TEXT NOT NULL DEFAULT '' + ); + + CREATE TABLE IF NOT EXISTS p5_provenance_migration_backups ( + operation_id TEXT NOT NULL, + memory_id TEXT NOT NULL, + before_exists INTEGER NOT NULL, + before_record_json TEXT NOT NULL DEFAULT '', + before_digest TEXT NOT NULL DEFAULT '', + before_revision INTEGER NOT NULL DEFAULT 0, + after_record_json TEXT NOT NULL, + after_digest TEXT NOT NULL, + after_revision INTEGER NOT NULL, + apply_state TEXT NOT NULL DEFAULT 'prepared', + rollback_state TEXT NOT NULL DEFAULT 'pending', + created_at TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT '', + rolled_back_at TEXT NOT NULL DEFAULT '', + PRIMARY KEY(operation_id, memory_id), + FOREIGN KEY(operation_id) REFERENCES p5_provenance_migration_runs(operation_id) + ); + CREATE INDEX IF NOT EXISTS idx_p5_provenance_backup_operation + ON p5_provenance_migration_backups(operation_id, rollback_state, memory_id); + """ + ) + run_columns = { + str(row[1]) + for row in connection.execute("PRAGMA table_info(p5_provenance_migration_runs)").fetchall() + } + if "backup_path" not in run_columns: + connection.execute( + "ALTER TABLE p5_provenance_migration_runs ADD COLUMN backup_path TEXT NOT NULL DEFAULT ''" + ) + + +def _canonical_json(value: Mapping[str, Any]) -> str: + return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + +def _operation_ref(operation_id: str) -> str: + return hashlib.sha256( + f"{PROVENANCE_CONTRACT_NAME}:migration:{operation_id}".encode("utf-8") + ).hexdigest() + + +def _valid_operation_id(value: Any) -> str: + return value if isinstance(value, str) and _OPERATION_ID_RE.fullmatch(value) else "" + + +def _limit(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + return 0 + return min(value, 5000) + + +def _record_from_row(row: sqlite3.Row) -> dict[str, Any]: + return { + "contract_name": row["contract_name"], + "contract_version": row["contract_version"], + "contract_fingerprint": row["contract_fingerprint"], + "memory_id": row["memory_id"], + "source_kind": row["source_kind"], + "source_trust": row["source_trust"], + "firewall_status": row["firewall_status"], + "source_event_ref_hash": row["source_event_ref_hash"], + "authority_attestation_ref_hash": row["authority_attestation_ref_hash"], + "provenance_state": row["provenance_state"], + "migration_operation_ref": row["migration_operation_ref"], + "recovery_operation_ref": row["recovery_operation_ref"], + "record_revision": int(row["record_revision"]), + } + + +def _insert_record(connection: sqlite3.Connection, record: Mapping[str, Any], *, digest: str, now: str) -> None: + values = { + "memory_id": record["memory_id"], + "contract_name": record["contract_name"], + "contract_version": record["contract_version"], + "contract_fingerprint": record["contract_fingerprint"], + "source_kind": record["source_kind"], + "source_trust": record["source_trust"], + "firewall_status": record["firewall_status"], + "source_event_ref_hash": record["source_event_ref_hash"], + "authority_attestation_ref_hash": record["authority_attestation_ref_hash"], + "provenance_state": record["provenance_state"], + "migration_operation_ref": record["migration_operation_ref"], + "recovery_operation_ref": record["recovery_operation_ref"], + "record_revision": record["record_revision"], + "record_digest": digest, + "created_at": now, + "updated_at": now, + } + connection.execute( + """ + INSERT INTO p5_provenance_records( + memory_id, contract_name, contract_version, contract_fingerprint, + source_kind, source_trust, firewall_status, source_event_ref_hash, + authority_attestation_ref_hash, provenance_state, + migration_operation_ref, recovery_operation_ref, record_revision, + record_digest, created_at, updated_at + ) VALUES( + :memory_id, :contract_name, :contract_version, :contract_fingerprint, + :source_kind, :source_trust, :firewall_status, :source_event_ref_hash, + :authority_attestation_ref_hash, :provenance_state, + :migration_operation_ref, :recovery_operation_ref, :record_revision, + :record_digest, :created_at, :updated_at + ) + """, + values, + ) + + +class P5ProvenanceMigration: + """Recoverable B0 legacy provenance backfill over an existing MemoryStore.""" + + def __init__(self, store: MemoryStore): + self.store = store + + async def preview(self, *, limit: int = 5000) -> dict[str, Any]: + return await self.store._run_recoverable_database_operation(self._preview_sync, limit) + + def _preview_sync(self, limit: int) -> dict[str, Any]: + bounded_limit = _limit(limit) + if not bounded_limit: + return {"mode": "preview", "ok": False, "readonly": True, "error_code": "limit_invalid", "count": 0} + with self.store._lock: + rows = self.store._conn.execute( + """ + SELECT m.id + FROM memories AS m + LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id + WHERE p.memory_id IS NULL + ORDER BY m.id + LIMIT ? + """, + (bounded_limit,), + ).fetchall() + return { + "mode": "preview", + "ok": True, + "readonly": True, + "contract_name": PROVENANCE_CONTRACT_NAME, + "contract_version": PROVENANCE_CONTRACT_VERSION, + "contract_fingerprint": contract_fingerprint(), + "count": len(rows), + "write_count": 0, + } + + async def apply(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]: + return await self.store._run_recoverable_database_operation(self._apply_sync, operation_id, limit) + + def _apply_sync(self, operation_id: str, limit: int) -> dict[str, Any]: + clean_operation_id = _valid_operation_id(operation_id) + bounded_limit = _limit(limit) + if not clean_operation_id: + return {"mode": "apply", "ok": False, "operation_id": "", "error_code": "operation_id_required", "changed": 0} + if not bounded_limit: + return {"mode": "apply", "ok": False, "operation_id": clean_operation_id, "error_code": "limit_invalid", "changed": 0} + reference = _operation_ref(clean_operation_id) + with self.store._lock: + existing = self.store._conn.execute( + "SELECT state, applied_count, backup_path FROM p5_provenance_migration_runs WHERE operation_id=?", + (clean_operation_id,), + ).fetchone() + if existing is not None: + if existing["state"] == "completed": + return { + "mode": "apply", "ok": True, "operation_id": clean_operation_id, + "changed": 0, "deduplicated": True, "backup_count": int(existing["applied_count"] or 0), + "backup_path": str(existing["backup_path"] or ""), + } + return { + "mode": "apply", "ok": False, "operation_id": clean_operation_id, + "error_code": "operation_not_reapplicable", "changed": 0, + } + try: + # This commits only prior state and creates the physical SQLite + # image before the first B0 mutation transaction is opened. + backup_path = str(self.store.backup(f".before_p5_provenance_b0.{clean_operation_id}")) + except Exception: + return { + "mode": "apply", "ok": False, "operation_id": clean_operation_id, + "error_code": "physical_backup_failed", "changed": 0, "backup_path": "", + } + with self.store._transaction_sync(): + + candidates = self.store._conn.execute( + """ + SELECT m.id + FROM memories AS m + LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id + WHERE p.memory_id IS NULL + ORDER BY m.id + LIMIT ? + """, + (bounded_limit,), + ).fetchall() + now = utc_now() + self.store._conn.execute( + """ + INSERT INTO p5_provenance_migration_runs( + operation_id, operation_ref_hash, contract_name, contract_version, + contract_fingerprint, state, requested_limit, candidate_count, + applied_count, rollback_conflict_count, backup_path, created_at, updated_at + ) VALUES(?,?,?,?,?,'running',?,?,0,0,?,?,?) + """, + ( + clean_operation_id, reference, PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, contract_fingerprint(), bounded_limit, + len(candidates), backup_path, now, now, + ), + ) + changed = 0 + for row in candidates: + record = legacy_unresolved(row["id"], record_revision=1) + record["migration_operation_ref"] = reference + checked = validate_provenance_record(record) + digest = provenance_record_digest(record) + if not checked["ok"] or not digest: + raise RuntimeError("p5_provenance_legacy_record_invalid") + record_json = _canonical_json(record) + # The backup/manifest row is deliberately inserted before + # the matching provenance record mutation in this transaction. + self.store._conn.execute( + """ + INSERT INTO p5_provenance_migration_backups( + operation_id, memory_id, before_exists, before_record_json, + before_digest, before_revision, after_record_json, after_digest, + after_revision, apply_state, rollback_state, created_at + ) VALUES(?,?,0,'','',0,?,?,?,'prepared','pending',?) + """, + (clean_operation_id, record["memory_id"], record_json, digest, record["record_revision"], now), + ) + _insert_record(self.store._conn, record, digest=digest, now=now) + self.store._conn.execute( + """ + UPDATE p5_provenance_migration_backups + SET apply_state='applied', applied_at=? + WHERE operation_id=? AND memory_id=? + """, + (now, clean_operation_id, record["memory_id"]), + ) + changed += 1 + self.store._conn.execute( + """ + UPDATE p5_provenance_migration_runs + SET state='completed', applied_count=?, updated_at=?, completed_at=? + WHERE operation_id=? + """, + (changed, now, now, clean_operation_id), + ) + return { + "mode": "apply", "ok": True, "operation_id": clean_operation_id, + "changed": changed, "deduplicated": False, "backup_count": changed, + "backup_path": backup_path, + } + + async def rollback(self, operation_id: str) -> dict[str, Any]: + return await self.store._run_recoverable_database_operation(self._rollback_sync, operation_id) + + def _rollback_sync(self, operation_id: str) -> dict[str, Any]: + clean_operation_id = _valid_operation_id(operation_id) + if not clean_operation_id: + return {"mode": "rollback", "ok": False, "operation_id": "", "error_code": "operation_id_required", "restored": 0, "conflicts": 0} + with self.store._lock: + with self.store._transaction_sync(): + run = self.store._conn.execute( + "SELECT state, operation_ref_hash FROM p5_provenance_migration_runs WHERE operation_id=?", + (clean_operation_id,), + ).fetchone() + if run is None: + return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_found", "restored": 0, "conflicts": 0} + if run["state"] == "rolled_back": + return {"mode": "rollback", "ok": True, "operation_id": clean_operation_id, "restored": 0, "conflicts": 0, "deduplicated": True} + if run["state"] not in {"completed", "rollback_conflicted"}: + return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_rollbackable", "restored": 0, "conflicts": 0} + + backups = self.store._conn.execute( + """ + SELECT * FROM p5_provenance_migration_backups + WHERE operation_id=? AND rollback_state='pending' + ORDER BY memory_id + """, + (clean_operation_id,), + ).fetchall() + restored = 0 + conflicts = 0 + now = utc_now() + for backup in backups: + current = self.store._conn.execute( + "SELECT * FROM p5_provenance_records WHERE memory_id=?", + (backup["memory_id"],), + ).fetchone() + expected = ( + current is not None + and current["record_digest"] == backup["after_digest"] + and int(current["record_revision"]) == int(backup["after_revision"]) + and current["migration_operation_ref"] == run["operation_ref_hash"] + ) + if not expected: + self.store._conn.execute( + """ + UPDATE p5_provenance_migration_backups + SET rollback_state='conflict' + WHERE operation_id=? AND memory_id=? + """, + (clean_operation_id, backup["memory_id"]), + ) + conflicts += 1 + continue + if int(backup["before_exists"]): + try: + before = json.loads(backup["before_record_json"]) + except (TypeError, ValueError): + before = None + before_digest = provenance_record_digest(before if isinstance(before, Mapping) else None) + if not before_digest or before_digest != backup["before_digest"]: + self.store._conn.execute( + "UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?", + (clean_operation_id, backup["memory_id"]), + ) + conflicts += 1 + continue + self.store._conn.execute("DELETE FROM p5_provenance_records WHERE memory_id=?", (backup["memory_id"],)) + _insert_record(self.store._conn, before, digest=before_digest, now=now) + else: + deleted = self.store._conn.execute( + "DELETE FROM p5_provenance_records WHERE memory_id=? AND record_digest=? AND record_revision=? AND migration_operation_ref=?", + (backup["memory_id"], backup["after_digest"], backup["after_revision"], run["operation_ref_hash"]), + ) + if deleted.rowcount != 1: + # The earlier SELECT predicate should make this + # unreachable under the store lock, but retain a + # conservative conflict if an adapter changes it. + self.store._conn.execute( + "UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?", + (clean_operation_id, backup["memory_id"]), + ) + conflicts += 1 + continue + self.store._conn.execute( + """ + UPDATE p5_provenance_migration_backups + SET rollback_state='restored', rolled_back_at=? + WHERE operation_id=? AND memory_id=? + """, + (now, clean_operation_id, backup["memory_id"]), + ) + restored += 1 + state = "rolled_back" if conflicts == 0 else "rollback_conflicted" + self.store._conn.execute( + """ + UPDATE p5_provenance_migration_runs + SET state=?, rollback_conflict_count=rollback_conflict_count + ?, + updated_at=?, rolled_back_at=? + WHERE operation_id=? + """, + (state, conflicts, now, now if conflicts == 0 else "", clean_operation_id), + ) + return { + "mode": "rollback", "ok": conflicts == 0, "operation_id": clean_operation_id, + "restored": restored, "conflicts": conflicts, "deduplicated": False, + } + + +__all__ = [ + "P5ProvenanceMigration", + "P5_PERSISTENCE_SCHEMA_VERSION", + "ensure_p5_provenance_persistence_schema", +] diff --git a/core/retrieval.py b/core/retrieval.py index 70ebcf0..cfed62a 100644 --- a/core/retrieval.py +++ b/core/retrieval.py @@ -41,6 +41,7 @@ class RetrievalEngine: keyword_fallback_min_fts_candidates: int = 80, knowledge_graph_enabled: bool = True, knowledge_graph_expansion_limit: int = 12, + p5_recall_provenance_required: bool = False, usage_recorder: Any | None = None, ): self.store = store @@ -69,6 +70,7 @@ class RetrievalEngine: ) self.knowledge_graph_enabled = bool(knowledge_graph_enabled) self.knowledge_graph_expansion_limit = max(0, int(knowledge_graph_expansion_limit or 0)) + self.p5_recall_provenance_required = bool(p5_recall_provenance_required) self.usage_recorder = usage_recorder if callable(usage_recorder) else None self._rank_path_info: dict[str, Any] = {} self.last_path_info: dict[str, Any] = { @@ -102,7 +104,11 @@ class RetrievalEngine: clean_text(getattr(getattr(item, "memory", None), "id", ""), 120) for item in results or [] ] - current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id]) + active_ids = [memory_id for memory_id in memory_ids if memory_id] + if self.p5_recall_provenance_required: + current = await self.store.p5_recall_eligible_memories_by_ids(active_ids) + else: + current = await self.store.get_memories_by_ids(active_ids) acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state() validated: list[SearchResult] = [] for item, memory_id in zip(results or [], memory_ids): @@ -126,7 +132,11 @@ class RetrievalEngine: ) -> list[dict[str, str]]: """Drop cached diagnostic snippets that are no longer readable.""" memory_ids = [clean_text(item.get("id"), 120) for item in blocked if isinstance(item, dict)] - current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id]) + active_ids = [memory_id for memory_id in memory_ids if memory_id] + if self.p5_recall_provenance_required: + current = await self.store.p5_recall_eligible_memories_by_ids(active_ids) + else: + current = await self.store.get_memories_by_ids(active_ids) acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state() validated: list[dict[str, str]] = [] for item in blocked or []: @@ -866,7 +876,11 @@ class RetrievalEngine: query = clean_text(query, 1000) terms = self._terms(query) graph_terms = [] - if self.knowledge_graph_enabled and self.knowledge_graph_expansion_limit > 0: + if ( + not self.p5_recall_provenance_required + and self.knowledge_graph_enabled + and self.knowledge_graph_expansion_limit > 0 + ): graph_labels = await self.store.related_knowledge_terms( terms, scope=ctx.scope, @@ -881,7 +895,11 @@ class RetrievalEngine: ) expanded_terms = self._merge_terms(terms, graph_terms) include_pending = not self.policy.hide_pending_review - ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending) + ranked_candidates = await self.store.list_candidate_memories( + limit=2000, + include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, + ) current_window_candidates = await self.store.list_current_window_candidate_memories( scope=ctx.scope, session_id=ctx.session_id, @@ -889,6 +907,7 @@ class RetrievalEngine: group_id=ctx.group_id, limit=self.current_window_candidate_limit, include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, ) time_window_candidates: list[MemoryRecord] = [] if time_intent is not None and time_intent.active: @@ -897,12 +916,14 @@ class RetrievalEngine: time_intent.end_at, limit=1600, include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, ) keyword_terms = self._keyword_candidate_terms(query, expanded_terms) fts_candidates = await self.store.list_fts_candidate_memories( keyword_terms, limit=1200, include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, ) use_keyword_fallback = len(fts_candidates) < self.keyword_fallback_min_fts_candidates keyword_candidates = ( @@ -910,6 +931,7 @@ class RetrievalEngine: keyword_terms, limit=1200, include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, ) if use_keyword_fallback else [] @@ -917,6 +939,7 @@ class RetrievalEngine: vector_candidates, vector_scores, embedding_info = await self._embedding_candidate_memories( query, include_pending=include_pending, + require_p5_recall_provenance=self.p5_recall_provenance_required, ) self._rank_path_info = embedding_info self._rank_path_info.update( @@ -1010,6 +1033,7 @@ class RetrievalEngine: query: str, *, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> tuple[list[MemoryRecord], dict[str, float], dict[str, Any]]: info: dict[str, Any] = { "embedding_enabled": bool(self.embedding_enabled), @@ -1046,6 +1070,7 @@ class RetrievalEngine: provider_id=self.embedding_provider_id, limit=self.embedding_candidate_limit, include_pending=include_pending, + require_p5_recall_provenance=require_p5_recall_provenance, ) except Exception as error: info["embedding_reason"] = f"embedding_store_error:{clean_text(error, 120)}" diff --git a/core/service.py b/core/service.py index 3046d6b..3e61adf 100644 --- a/core/service.py +++ b/core/service.py @@ -9,6 +9,7 @@ import json import hashlib import inspect import re +import threading from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -92,9 +93,13 @@ from .user_memory_boundary import UserMemoryBoundaryError, validate_user_memory_ from .user_memory_migration import UserMemoryMigration from .unified_person_persistence import UnifiedPersonPersistence, UnifiedPersonReferenceMigration from .person_memory import PersonMemoryFacts, PersonMemoryMigration +from .p5_provenance_persistence import P5ProvenanceMigration +from .p5_b1_recall_gate import evaluate_bridge_serialization_gate, evaluate_normal_recall_gate from .visibility import VisibilityPolicy +_P5_ATTESTATION_BIND_FALLBACK_LOCK = threading.RLock() + _RECENT_FACT_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( ("meal", re.compile(r"吃过|吃完|吃了|没吃|还没吃|正在吃|吃饭|早饭|早餐|午饭|午餐|晚饭|晚餐|夜宵")), ("bath", re.compile(r"洗澡|洗完澡|冲澡|沐浴")), @@ -198,10 +203,14 @@ class MemoryCompanionService: self._retrieval_result_cache_stats: dict[str, int] = {"hits": 0, "misses": 0, "evictions": 0} self._RETRIEVAL_RESULT_CACHE_TTL: float = 45.0 self._RETRIEVAL_RESULT_CACHE_MAX: int = 128 + self._p5_attestation_verifier: Any = None + self._p5_attestation_verifier_bound = False + self._p5_attestation_verifier_lock = threading.RLock() self.user_memory_migration = UserMemoryMigration(self.store) self.unified_person_persistence = UnifiedPersonPersistence(self.store) self.unified_person_reference_migration = UnifiedPersonReferenceMigration(self.store) self.person_memory_migration = PersonMemoryMigration(self.store) + self.p5_provenance_migration = P5ProvenanceMigration(self.store) self.person_memory_facts = PersonMemoryFacts(self.store) self.portable_archive = PortableMemoryArchive(self.store, self.data_dir) self.chat_importer = HistoricalChatImporter(self) @@ -281,6 +290,28 @@ class MemoryCompanionService: async def rollback_person_memory_migration(self, migration_id: str) -> dict[str, Any]: return await self.person_memory_migration.rollback(migration_id) + async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]: + return await self.p5_provenance_migration.preview(limit=limit) + + async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]: + return await self.p5_provenance_migration.apply(operation_id=operation_id, limit=limit) + + async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]: + return await self.p5_provenance_migration.rollback(operation_id) + + def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool: + """Pin the first Bridge-provided in-process B1 verifier.""" + + if not callable(consume_callable): + return False + bind_lock = getattr(self, "_p5_attestation_verifier_lock", _P5_ATTESTATION_BIND_FALLBACK_LOCK) + with bind_lock: + if getattr(self, "_p5_attestation_verifier_bound", False): + return False + self._p5_attestation_verifier = consume_callable + self._p5_attestation_verifier_bound = True + return True + async def bridge_correct_person_memory( self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord, ) -> dict[str, Any]: @@ -500,6 +531,84 @@ class MemoryCompanionService: ctx.context_orchestration = project_context_state(state) or {} ctx.context_orchestration_status = "accepted" + def _p5_b1_record_normal_recall_audit(self, *, event: Any, req: Any, reason: str) -> None: + """Expose only a static gate reason; never retain a handle/snapshot.""" + + for carrier in (req, event): + if carrier is None: + continue + try: + setattr(carrier, "memory_companion_p5_b1_recall_gate_reason", reason) + except Exception: + pass + + async def _p5_b1_normal_recall_injection( + self, + ctx: SessionContext, + req: Any, + *, + event: Any, + p3_state: Any, + ) -> tuple[str | None, str]: + """Gate only the normal request-time memory injection seam. + + With the default switch off this calls the historical seam unchanged. + With it on, the pinned verifier must consume the opaque handle using + these exact request/event/P3 object identities before recall proceeds. + """ + + if not self.config.bool("enable_p5_b1_recall_gate", False): + return await self.inject_memories(ctx, req, event=event), "" + reason = evaluate_normal_recall_gate( + getattr(self, "_p5_attestation_verifier", None), + req=req, + event=event, + p3_state=p3_state, + ) + if reason: + self._p5_b1_record_normal_recall_audit(event=event, req=req, reason=reason) + return None, reason + return await self.inject_memories(ctx, req, event=event), "" + + def _p5_b1_bridge_gate_reason(self, invocation: Any) -> str: + """Consume an explicitly carried B1b invocation before Bridge recall. + + ``None`` means a legacy/background Bridge caller which is outside the + current-request B1b scope. A protected Companion path always carries + an opaque invocation, including an intentionally unusable one when + minting or verifier binding failed, so it fails closed here. + """ + + if invocation is None: + return "" + if not self.config.bool("enable_p5_b1_bridge_gate", False): + return "" + return evaluate_bridge_serialization_gate( + getattr(self, "_p5_attestation_verifier", None), + invocation=invocation, + ) + + def _p5_b1_tool_recall_gate_reason(self) -> str: + """Fail closed because tool callbacks do not carry a request invocation.""" + + config = getattr(self, "config", None) + getter = getattr(config, "bool", None) + return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b1_tool_recall_gate", False) else "" + + def _p5_b2_archive_read_gate_reason(self) -> str: + """Reject scope-dict archive reads until a trusted identity contract exists.""" + + config = getattr(self, "config", None) + getter = getattr(config, "bool", None) + return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_archive_read_gate", False) else "" + + def _p5_b2_cross_user_read_gate_reason(self) -> str: + """Reject scope-dict cross-user reads until a trusted identity contract exists.""" + + config = getattr(self, "config", None) + getter = getattr(config, "bool", None) + return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_cross_user_read_gate", False) else "" + async def handle_llm_request(self, event: Any, req: Any) -> None: ctx = await self.identity.resolve_event_context(event) self._sanitize_session_context_message_text(ctx) @@ -533,8 +642,13 @@ class MemoryCompanionService: # this memory plugin no longer advances relationship authority. try: - injection = await self.inject_memories(ctx, req, event=event) - if injection is not None: + injection, p5_b1_reason = await self._p5_b1_normal_recall_injection( + ctx, req, event=event, p3_state=p3_state + ) + if p5_b1_reason: + self._mark_memory_companion_injection_state(event, req, injected=False, conversation_memory=False, slot_map={}) + self._p3_complete_memory(p3_state, ctx=ctx, injection="", event=event, req=req) + elif injection is not None: self._p3_complete_memory(p3_state, ctx=ctx, injection=injection, event=event, req=req) except Exception as exc: logger.warning( @@ -1354,6 +1468,16 @@ class MemoryCompanionService: time_range: dict[str, str] | None = None, ) -> dict[str, Any]: payload = dict(scope or {}) + p5_b2_reason = self._p5_b2_cross_user_read_gate_reason() + if p5_b2_reason: + return { + **user_fact_query_capabilities(available=True, warnings=[p5_b2_reason]), + "query": clean_text(query, 1400), + "scope": payload, + "memories": [], + "excluded": [], + "warnings": [p5_b2_reason], + } requested_scope = clean_text(payload.get("scope"), 40).lower() user_id = clean_text(payload.get("user_id"), 120) group_id = clean_text(payload.get("group_id"), 120) @@ -1445,6 +1569,15 @@ class MemoryCompanionService: time_range: dict[str, str] | None = None, include_timeline: bool = True, ) -> dict[str, Any]: + p5_b2_reason = self._p5_b2_archive_read_gate_reason() + if p5_b2_reason: + return { + "archive_api_version": ARCHIVE_API_VERSION, + "scope": await self.bridge_get_archive_scope(scope=scope), + "memories": [], + "excluded": [], + "warnings": [p5_b2_reason], + } ctx = self._archive_context(scope) query_text = clean_text(query, 1400) gate_reason = self._p3_gate_reason( @@ -1515,6 +1648,13 @@ class MemoryCompanionService: scope: dict[str, Any], include_timeline: bool = True, ) -> dict[str, Any]: + p5_b2_reason = self._p5_b2_archive_read_gate_reason() + if p5_b2_reason: + return { + "archive_api_version": "1.0", + "scope": await self.bridge_get_archive_scope(scope=scope), + "memories": [], "excluded": [], "warnings": [p5_b2_reason], + } ctx = self._archive_context(scope) gate_reason = self._p3_gate_reason( ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None, @@ -1628,6 +1768,14 @@ class MemoryCompanionService: limit: int = 30, include_timeline: bool = True, ) -> dict[str, Any]: + p5_b2_reason = self._p5_b2_archive_read_gate_reason() + if p5_b2_reason: + return { + "archive_api_version": "1.0", + "scope": await self.bridge_get_archive_scope(scope=scope), + "start_at": clean_text(start_at, 80), "end_at": clean_text(end_at, 80), + "items": [], "warnings": [p5_b2_reason], + } ctx = self._archive_context(scope) gate_reason = self._p3_gate_reason( ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None, @@ -1694,7 +1842,10 @@ class MemoryCompanionService: max_chars: int | None = None, companion_bot_mood: str = "", companion_bot_energy: float = 0.0, + p5_b1_bridge_invocation: Any = None, ) -> str: + if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation): + return "" ctx = self.session_context_from_bridge(session_context) if self._p3_bridge_gate_reason(ctx): return "" @@ -1722,6 +1873,7 @@ class MemoryCompanionService: companion_bot_mood: str = "", companion_bot_energy: float = 0.0, retrieval_profile: str = "", + p5_b1_bridge_invocation: Any = None, ) -> str: profile = clean_text(retrieval_profile, 40).lower() fast_profile_enabled = { @@ -1743,6 +1895,7 @@ class MemoryCompanionService: companion_bot_mood=companion_bot_mood, companion_bot_energy=companion_bot_energy, context_kind="outfit" if profile == "outfit_fast" else "schedule", + p5_b1_bridge_invocation=p5_b1_bridge_invocation, ) return await self.bridge_compose_injection( query, @@ -1751,6 +1904,7 @@ class MemoryCompanionService: max_chars=max_chars, companion_bot_mood=companion_bot_mood, companion_bot_energy=companion_bot_energy, + p5_b1_bridge_invocation=p5_b1_bridge_invocation, ) async def _bridge_compose_schedule_fast_context( @@ -1763,7 +1917,10 @@ class MemoryCompanionService: companion_bot_mood: str, companion_bot_energy: float, context_kind: str = "schedule", + p5_b1_bridge_invocation: Any = None, ) -> str: + if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation): + return "" started = time.monotonic() ctx = self._normalized_session_context(self.session_context_from_bridge(session_context)) if self._p3_bridge_gate_reason(ctx): @@ -1781,6 +1938,7 @@ class MemoryCompanionService: bot_id=ctx.bot_id, limit=max(72, limit * 12) if outfit_focus else max(24, limit * 6), strict_session_only=ctx.strict_session_only, + require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False), ) policy = self.visibility_policy() visible = [record for record in records if policy.is_visible(record, ctx)[0]] @@ -2453,6 +2611,7 @@ class MemoryCompanionService: retrieval_mode="basic", embedding_enabled=False, knowledge_graph_enabled=False, + p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False), ) async def _retrieval_engine(self, ctx: SessionContext, *, admin_read_all: bool = False) -> RetrievalEngine: @@ -2494,6 +2653,7 @@ class MemoryCompanionService: ), knowledge_graph_enabled=self.config.bool("knowledge_graph.retrieval_expansion_enabled", True), knowledge_graph_expansion_limit=self.config.int("knowledge_graph.expansion_limit", 12), + p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False), usage_recorder=self._record_token_usage, ) @@ -2986,6 +3146,7 @@ class MemoryCompanionService: provider_id=provider_id, limit=self.config.int("retrieval.embedding_backfill_batch_size", 50), include_pending=self.config.bool("retrieval.embedding_index_pending", False), + require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False), ) for record in records: if await self._embed_memory_record(provider, provider_id, record): @@ -3004,12 +3165,23 @@ class MemoryCompanionService: return False if record.review_status == "pending" and not self.config.bool("retrieval.embedding_index_pending", False): return False + p5_recall_provenance_required = self.config.bool("enable_p5_b1_recall_gate", False) + if p5_recall_provenance_required: + eligible = await self.store.p5_recall_eligible_memory_ids([record.id]) + if record.id not in eligible: + return False text = self._memory_embedding_text(record) text_hash = self._memory_embedding_text_hash(record) if not text or not text_hash: return False async with self._embedding_background_semaphore: try: + # Revalidate immediately before external egress. This closes + # the interval while waiting for the shared provider slot. + if p5_recall_provenance_required: + eligible = await self.store.p5_recall_eligible_memory_ids([record.id]) + if record.id not in eligible: + return False vector = await self._embed_text_with_provider(provider, text, provider_id=provider_id) vector = self._normalize_embedding_vector(vector) if not vector: @@ -4060,6 +4232,9 @@ class MemoryCompanionService: reason = self._p3_gate_reason(ctx, event=event, state=state) if reason: return {"ok": False, "error": reason, "memories": []} + p5_b1_reason = self._p5_b1_tool_recall_gate_reason() + if p5_b1_reason: + return {"ok": False, "error": p5_b1_reason, "memories": []} query = clean_text(query, 1000) if not query: return {"ok": False, "error": "empty query", "memories": []} diff --git a/core/store.py b/core/store.py index af595eb..aa5a298 100644 --- a/core/store.py +++ b/core/store.py @@ -14,6 +14,66 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads, from .unified_person_persistence import ensure_unified_person_persistence_schema from .person_memory import apply_envelope, ensure_person_memory_schema, indexable, normalize_record from .person_memory_contract import consume_person_memory_envelope +from .p5_provenance_persistence import ensure_p5_provenance_persistence_schema +from .p5_provenance import ( + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + SOURCE_KINDS as P5_SOURCE_KINDS, + contract_fingerprint as p5_provenance_contract_fingerprint, +) + + +def _p5_recall_eligibility_predicate(memory_alias: str = "") -> tuple[str, list[str]]: + """Return the B1e SQL predicate for complete, valid recall records only.""" + + prefix = f"{memory_alias}." if memory_alias else "" + observed_source_kinds = tuple(kind for kind in P5_SOURCE_KINDS if kind != "unknown") + source_kind_placeholders = ", ".join("?" for _ in observed_source_kinds) + return ( + f""" + EXISTS( + SELECT 1 + FROM p5_provenance_records AS p5 + WHERE p5.memory_id={prefix}id + AND p5.contract_name=? + AND p5.contract_version=? + AND p5.contract_fingerprint=? + AND p5.provenance_state='observed' + AND p5.source_kind IN ({source_kind_placeholders}) + AND p5.source_trust IN ('T0', 'T1', 'T2') + AND p5.firewall_status IN ('allowed', 'sanitized') + AND length(p5.source_event_ref_hash)=64 + AND p5.source_event_ref_hash NOT GLOB '*[^0-9a-f]*' + AND length(p5.authority_attestation_ref_hash)=64 + AND p5.authority_attestation_ref_hash NOT GLOB '*[^0-9a-f]*' + AND p5.source_event_ref_hash != p5.authority_attestation_ref_hash + AND typeof(p5.record_revision)='integer' + AND p5.record_revision >= 0 + AND length(p5.record_digest)=64 + AND p5.record_digest NOT GLOB '*[^0-9a-f]*' + AND ( + p5.migration_operation_ref='' + OR ( + length(p5.migration_operation_ref)=64 + AND p5.migration_operation_ref NOT GLOB '*[^0-9a-f]*' + ) + ) + AND ( + p5.recovery_operation_ref='' + OR ( + length(p5.recovery_operation_ref)=64 + AND p5.recovery_operation_ref NOT GLOB '*[^0-9a-f]*' + ) + ) + ) + """, + [ + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + p5_provenance_contract_fingerprint(), + *observed_source_kinds, + ], + ) class MemoryStore: @@ -495,6 +555,7 @@ class MemoryStore: self._ensure_memory_columns_sync() ensure_unified_person_persistence_schema(self._conn) ensure_person_memory_schema(self._conn) + ensure_p5_provenance_persistence_schema(self._conn) self._conn.execute( "CREATE INDEX IF NOT EXISTS idx_memories_domain ON memories(memory_domain, lifecycle, scope, group_id, subject_id)" ) @@ -667,6 +728,22 @@ class MemoryStore: BEGIN UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1; END; + + CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_insert + AFTER INSERT ON p5_provenance_records + BEGIN + UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1; + END; + CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_update + AFTER UPDATE ON p5_provenance_records + BEGIN + UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1; + END; + CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_delete + AFTER DELETE ON p5_provenance_records + BEGIN + UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1; + END; """ ) @@ -3492,14 +3569,34 @@ class MemoryStore: result.append(item) return result - async def list_candidate_memories(self, limit: int = 500, include_pending: bool = False) -> list[MemoryRecord]: - return await self._run_recoverable_database_operation(self._list_candidate_memories_sync, limit, include_pending) + async def list_candidate_memories( + self, + limit: int = 500, + include_pending: bool = False, + *, + require_p5_recall_provenance: bool = False, + ) -> list[MemoryRecord]: + return await self._run_recoverable_database_operation( + self._list_candidate_memories_sync, + limit, + include_pending, + require_p5_recall_provenance, + ) - def _list_candidate_memories_sync(self, limit: int, include_pending: bool) -> list[MemoryRecord]: + def _list_candidate_memories_sync( + self, + limit: int, + include_pending: bool, + require_p5_recall_provenance: bool = False, + ) -> list[MemoryRecord]: where = "lifecycle != 'archived'" params: list[Any] = [] if not include_pending: where += " AND review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate() + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( f""" @@ -3520,6 +3617,7 @@ class MemoryStore: bot_id: str = "", limit: int = 36, strict_session_only: bool = False, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: """Read a small schedule-focused snapshot without waiting on the shared writer lock.""" return await asyncio.to_thread( @@ -3529,6 +3627,7 @@ class MemoryStore: bot_id, limit, strict_session_only, + require_p5_recall_provenance, ) def _list_schedule_context_memories_sync( @@ -3538,6 +3637,7 @@ class MemoryStore: bot_id: str, limit: int, strict_session_only: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: session_id = clean_text(session_id, 200) user_id = clean_text(user_id, 120) @@ -3623,6 +3723,10 @@ class MemoryStore: return [] where += " AND session_id=?" params.append(session_id) + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate("memories") + where += f" AND {predicate}" + params.extend(predicate_params) uri = self.db_path.resolve().as_uri() + "?mode=ro" with closing(sqlite3.connect(uri, uri=True, timeout=0.35, check_same_thread=False)) as connection: connection.row_factory = sqlite3.Row @@ -3664,6 +3768,7 @@ class MemoryStore: group_id: str = "", limit: int = 600, include_pending: bool = False, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: return await self._run_recoverable_database_operation( self._list_current_window_candidate_memories_sync, @@ -3673,6 +3778,7 @@ class MemoryStore: group_id, limit, include_pending, + require_p5_recall_provenance, ) def _list_current_window_candidate_memories_sync( @@ -3683,6 +3789,7 @@ class MemoryStore: group_id: str, limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: scope = clean_text(scope, 40).lower() session_id = clean_text(session_id, 200) @@ -3704,6 +3811,10 @@ class MemoryStore: where = "lifecycle != 'archived' AND (" + " OR ".join(clauses) + ")" if not include_pending: where += " AND review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate() + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( f""" @@ -3724,6 +3835,7 @@ class MemoryStore: end_at: str, limit: int = 1200, include_pending: bool = False, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: return await self._run_recoverable_database_operation( self._list_time_window_candidate_memories_sync, @@ -3731,6 +3843,7 @@ class MemoryStore: end_at, limit, include_pending, + require_p5_recall_provenance, ) def _list_time_window_candidate_memories_sync( @@ -3739,15 +3852,21 @@ class MemoryStore: end_at: str, limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: start = clean_text(start_at, 80) end = clean_text(end_at, 80) if not start or not end: return [] where = "lifecycle != 'archived'" - params: list[Any] = [start, end, start, end, start, end] + time_params: list[Any] = [start, end, start, end, start, end] + params: list[Any] = [] if not include_pending: where += " AND review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate() + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( f""" @@ -3764,7 +3883,7 @@ class MemoryStore: importance DESC LIMIT ? """, - params + [max(1, int(limit or 1))], + params + time_params + [max(1, int(limit or 1))], ).fetchall() return [MemoryRecord.from_row(row) for row in rows] @@ -3773,12 +3892,15 @@ class MemoryStore: terms: list[str], limit: int = 800, include_pending: bool = False, + *, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: return await self._run_recoverable_database_operation( self._list_fts_candidate_memories_sync, terms, limit, include_pending, + require_p5_recall_provenance, ) def _list_fts_candidate_memories_sync( @@ -3786,6 +3908,7 @@ class MemoryStore: terms: list[str], limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: if not self._fts_enabled: return [] @@ -3796,6 +3919,10 @@ class MemoryStore: params: list[Any] = [query] if not include_pending: where += " AND m.review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate("m") + where += f" AND {predicate}" + params.extend(predicate_params) try: with self._lock: rows = self._conn.execute( @@ -3845,12 +3972,15 @@ class MemoryStore: terms: list[str], limit: int = 800, include_pending: bool = False, + *, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: return await self._run_recoverable_database_operation( self._list_keyword_candidate_memories_sync, terms, limit, include_pending, + require_p5_recall_provenance, ) def _list_keyword_candidate_memories_sync( @@ -3858,6 +3988,7 @@ class MemoryStore: terms: list[str], limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: cleaned_terms = [] for term in terms or []: @@ -3894,6 +4025,10 @@ class MemoryStore: ) params.extend([like] * len(columns)) where += " AND (" + " OR ".join(term_clauses) + ")" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate() + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( @@ -4585,6 +4720,48 @@ class MemoryStore: async def get_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]: return await self._run_recoverable_database_operation(self._get_memories_by_ids_sync, memory_ids) + async def p5_recall_eligible_memory_ids(self, memory_ids: list[str]) -> set[str]: + """Return only IDs that still satisfy the B1e SQL provenance rule.""" + + return await self._run_recoverable_database_operation( + self._p5_recall_eligible_memory_ids_sync, + memory_ids, + ) + + def _p5_recall_eligible_memory_ids_sync(self, memory_ids: list[str]) -> set[str]: + ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120))) + if not ids: + return set() + placeholders = ",".join("?" for _ in ids) + predicate, predicate_params = _p5_recall_eligibility_predicate("m") + with self._lock: + rows = self._conn.execute( + f"SELECT m.id FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}", + [*ids, *predicate_params], + ).fetchall() + return {clean_text(row["id"], 120) for row in rows if clean_text(row["id"], 120)} + + async def p5_recall_eligible_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]: + """Atomically re-read cached bodies under the B1e provenance predicate.""" + + return await self._run_recoverable_database_operation( + self._p5_recall_eligible_memories_by_ids_sync, + memory_ids, + ) + + def _p5_recall_eligible_memories_by_ids_sync(self, memory_ids: list[str]) -> dict[str, MemoryRecord]: + ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120))) + if not ids: + return {} + placeholders = ",".join("?" for _ in ids) + predicate, predicate_params = _p5_recall_eligibility_predicate("m") + with self._lock: + rows = self._conn.execute( + f"SELECT m.* FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}", + [*ids, *predicate_params], + ).fetchall() + return {row["id"]: MemoryRecord.from_row(row) for row in rows} + async def get_timeline_event(self, event_id: str) -> dict[str, Any] | None: return await self._run_recoverable_database_operation(self._get_timeline_event_sync, event_id) @@ -5252,12 +5429,14 @@ class MemoryStore: provider_id: str, limit: int = 3000, include_pending: bool = False, + require_p5_recall_provenance: bool = False, ) -> list[tuple[MemoryRecord, list[float], str]]: return await self._run_recoverable_database_operation( self._list_embedding_candidate_rows_sync, provider_id, limit, include_pending, + require_p5_recall_provenance, ) def _list_embedding_candidate_rows_sync( @@ -5265,6 +5444,7 @@ class MemoryStore: provider_id: str, limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[tuple[MemoryRecord, list[float], str]]: provider_id = clean_text(provider_id, 160) if not provider_id: @@ -5275,7 +5455,7 @@ class MemoryStore: if revision != self._embedding_candidate_cache_revision: self._embedding_candidate_cache.clear() self._embedding_candidate_cache_revision = revision - cache_key = (provider_id, bool(include_pending), safe_limit) + cache_key = (provider_id, bool(include_pending), safe_limit, bool(require_p5_recall_provenance)) cached = self._embedding_candidate_cache.get(cache_key) if cached is not None: return deepcopy(cached) @@ -5283,6 +5463,10 @@ class MemoryStore: params: list[Any] = [provider_id] if not include_pending: where += " AND m.review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate("m") + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( f""" @@ -5323,12 +5507,14 @@ class MemoryStore: provider_id: str, limit: int = 80, include_pending: bool = False, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: return await self._run_recoverable_database_operation( self._list_memories_missing_embeddings_sync, provider_id, limit, include_pending, + require_p5_recall_provenance, ) def _list_memories_missing_embeddings_sync( @@ -5336,6 +5522,7 @@ class MemoryStore: provider_id: str, limit: int, include_pending: bool, + require_p5_recall_provenance: bool = False, ) -> list[MemoryRecord]: provider_id = clean_text(provider_id, 160) if not provider_id: @@ -5344,6 +5531,10 @@ class MemoryStore: params: list[Any] = [provider_id] if not include_pending: where += " AND m.review_status != 'pending'" + if require_p5_recall_provenance: + predicate, predicate_params = _p5_recall_eligibility_predicate("m") + where += f" AND {predicate}" + params.extend(predicate_params) with self._lock: rows = self._conn.execute( f""" diff --git a/tests/test_p5_b1_candidate_gate.py b/tests/test_p5_b1_candidate_gate.py new file mode 100644 index 0000000..efa0cae --- /dev/null +++ b/tests/test_p5_b1_candidate_gate.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import asyncio +import tempfile +import unittest +from pathlib import Path + +from core.models import EntityRef, MemoryRecord, SearchResult, SessionContext +from core.p5_provenance import ( + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + contract_fingerprint, +) +from core.retrieval import RetrievalEngine +from core.service import MemoryCompanionService +from core.store import MemoryStore +from core.visibility import VisibilityPolicy + + +_HASH = "a" * 64 +_OTHER_HASH = "b" * 64 + + +class _RerankProvider: + model = "test-reranker" + + def __init__(self) -> None: + self.documents: list[str] = [] + + async def rerank(self, **kwargs): + self.documents = list(kwargs.get("documents") or []) + return [{"index": index, "relevance_score": 1.0} for index in range(len(self.documents))] + + +class _ServiceConfig: + def __init__(self, p5_required: bool) -> None: + self.p5_required = p5_required + + def bool(self, key: str, default: bool = False) -> bool: + if key == "enable_p5_b1_recall_gate": + return self.p5_required + return default + + def int(self, _key: str, default: int = 0) -> int: + return default + + def float(self, _key: str, default: float = 0.0) -> float: + return default + + def get(self, _key: str, default=None): + return default + + +class P5B1CandidateGateTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.store = MemoryStore(Path(self.temp.name) / "memory.db") + self.store.initialize() + for identifier in ("eligible", "legacy", "t3", "t4", "missing", "invalid", "bad_kind", "same_hash"): + await self.store.insert_memory( + MemoryRecord( + id=identifier, + memory_domain="user_memory", + subject=EntityRef(kind="user", id="u1"), + scope="private", + session_id="s1", + content=f"needle {identifier} private body", + evidence="test", + lifecycle="stable_memory", + visibility="shareable", + importance=0.8, + ) + ) + self._put_provenance("eligible") + self._put_provenance("legacy", state="legacy_unresolved", trust="T4", firewall="unknown", reference="") + self._put_provenance("t3", trust="T3") + self._put_provenance("t4", trust="T4") + self._put_provenance("invalid", reference=_HASH.upper()) + self._put_provenance("bad_kind", source_kind="message") + self._put_provenance("same_hash", source_event_hash=_HASH) + + async def asyncTearDown(self) -> None: + self.store.close() + self.temp.cleanup() + + def _put_provenance( + self, + memory_id: str, + *, + state: str = "observed", + trust: str = "T1", + firewall: str = "allowed", + reference: str = _HASH, + source_kind: str = "current_user_intent", + source_event_hash: str = _OTHER_HASH, + ) -> None: + self.store._conn.execute( + """ + INSERT INTO p5_provenance_records( + memory_id, contract_name, contract_version, contract_fingerprint, + source_kind, source_trust, firewall_status, source_event_ref_hash, + authority_attestation_ref_hash, provenance_state, + migration_operation_ref, recovery_operation_ref, record_revision, + record_digest, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', '', 1, ?, + '2026-07-29T00:00:00+00:00', '2026-07-29T00:00:00+00:00') + """, + ( + memory_id, + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + contract_fingerprint(), + source_kind, + trust, + firewall, + source_event_hash, + reference, + state, + _HASH, + ), + ) + self.store._conn.commit() + + @staticmethod + def _ids(rows) -> set[str]: + return {row.id if hasattr(row, "id") else row[0].id for row in rows} + + async def test_store_sql_fts_keyword_vector_and_windows_fail_closed(self) -> None: + self.assertGreaterEqual(len(await self.store.list_candidate_memories(limit=20)), 6) + self.assertEqual( + {"eligible"}, self._ids(await self.store.list_candidate_memories(limit=20, require_p5_recall_provenance=True)) + ) + self.assertEqual( + {"eligible"}, + self._ids( + await self.store.list_current_window_candidate_memories( + scope="private", session_id="s1", user_id="u1", limit=20, + require_p5_recall_provenance=True, + ) + ), + ) + self.assertEqual( + {"eligible"}, + self._ids( + await self.store.list_time_window_candidate_memories( + "2026-01-01T00:00:00+00:00", "2027-01-01T00:00:00+00:00", limit=20, + require_p5_recall_provenance=True, + ) + ), + ) + self.assertEqual( + {"eligible"}, + self._ids(await self.store.list_fts_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)), + ) + self.assertEqual( + {"eligible"}, + self._ids(await self.store.list_keyword_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)), + ) + self.assertEqual( + {"eligible"}, + self._ids( + await self.store.list_memories_missing_embeddings( + provider_id="embed", limit=20, require_p5_recall_provenance=True + ) + ), + ) + await self.store.upsert_memory_embedding(memory_id="eligible", provider_id="embed", text_hash="e", vector=[1.0, 0.0]) + await self.store.upsert_memory_embedding(memory_id="legacy", provider_id="embed", text_hash="l", vector=[1.0, 0.0]) + self.assertEqual( + {"eligible"}, + self._ids( + await self.store.list_embedding_candidate_rows( + provider_id="embed", limit=20, require_p5_recall_provenance=True + ) + ), + ) + self.store._conn.execute("UPDATE p5_provenance_records SET source_trust='T3' WHERE memory_id='eligible'") + self.store._conn.commit() + self.assertEqual( + set(), + self._ids( + await self.store.list_embedding_candidate_rows( + provider_id="embed", limit=20, require_p5_recall_provenance=True + ) + ), + ) + + async def test_engine_excludes_unattested_body_before_rerank_and_revalidates_cache(self) -> None: + provider = _RerankProvider() + engine = RetrievalEngine( + self.store, + VisibilityPolicy(enable_acl_rules=False), + retrieval_mode="rerank", + rerank_provider=provider, + rerank_provider_id="test", + embedding_enabled=False, + knowledge_graph_enabled=True, + p5_recall_provenance_required=True, + ) + ctx = SessionContext(session_id="s1", scope="private", user_id="u1") + results = await engine.search("needle", ctx, 10) + self.assertEqual({"eligible"}, {item.memory.id for item in results}) + self.assertEqual(1, len(provider.documents)) + self.assertIn("needle eligible private body", provider.documents[0]) + self.assertNotIn("needle legacy private body", provider.documents[0]) + cached = await engine.revalidate_cached_results( + [SearchResult(memory=(await self.store.get_memory("legacy")), score=1.0)], ctx + ) + self.assertEqual([], cached) + + default_off = RetrievalEngine( + self.store, + VisibilityPolicy(enable_acl_rules=False), + retrieval_mode="basic", + embedding_enabled=False, + knowledge_graph_enabled=False, + p5_recall_provenance_required=False, + ) + legacy_visible = await default_off.search("needle legacy", ctx, 10) + self.assertIn("legacy", {item.memory.id for item in legacy_visible}) + + async def test_service_propagates_recall_switch_to_retrieval_and_cache_validation_engines(self) -> None: + async def no_rerank_provider(*_args, **_kwargs): + return None, "" + + for enabled in (False, True): + service = object.__new__(MemoryCompanionService) + service.store = self.store + service.config = _ServiceConfig(enabled) + service.visibility_policy = lambda **_kwargs: VisibilityPolicy(enable_acl_rules=False) + service._resolve_rerank_provider = no_rerank_provider + service._record_token_usage = lambda *_args, **_kwargs: None + service._embedding_timeout_ms = lambda: 1200 + engine = await service._retrieval_engine(SessionContext()) + validation = service._retrieval_validation_engine() + self.assertEqual(enabled, engine.p5_recall_provenance_required) + self.assertEqual(enabled, validation.p5_recall_provenance_required) + + async def test_embedding_provider_rechecks_provenance_before_body_egress(self) -> None: + calls: list[str] = [] + + async def embed(_provider, text: str, *, provider_id: str): + calls.append(text) + return [1.0, 0.0] + + def build_service(enabled: bool): + service = object.__new__(MemoryCompanionService) + service.store = self.store + service.config = _ServiceConfig(enabled) + service._embedding_background_semaphore = asyncio.Semaphore(1) + service._embed_text_with_provider = embed + return service + + legacy = await self.store.get_memory("legacy") + self.assertFalse(await build_service(True)._embed_memory_record(object(), "embed-guard", legacy)) + self.assertEqual([], calls) + + self.assertTrue(await build_service(False)._embed_memory_record(object(), "embed-default-off", legacy)) + self.assertEqual(1, len(calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p5_b1_recall_gate.py b/tests/test_p5_b1_recall_gate.py new file mode 100644 index 0000000..89a058c --- /dev/null +++ b/tests/test_p5_b1_recall_gate.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +import threading +import unittest + +from core.bridge import MemoryCompanionBridge +from core.p5_b1_recall_gate import ( + BRIDGE_SERIALIZATION_SINK, + MEMORY_RECALL_SINK, + P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, +) +from core.service import MemoryCompanionService + + +class _Config: + def __init__(self, enabled: bool, *, bridge_enabled: bool | None = None): + self.enabled = enabled + self.bridge_enabled = enabled if bridge_enabled is None else bridge_enabled + + def bool(self, key: str, default: bool = False) -> bool: + if key == "enable_p5_b1_recall_gate": + return self.enabled + if key == "enable_p5_b1_bridge_gate": + return self.bridge_enabled + return default + + def int(self, key: str, default: int = 0) -> int: + return default + + +class _Verifier: + def __init__(self, *, disposition: str = "allow", sink: str = MEMORY_RECALL_SINK): + self.disposition = disposition + self.sink = sink + self.calls: list[tuple[object, object, object, object, str]] = [] + self.consumed: set[int] = set() + + def __call__(self, handle, request, event, p3_state, sink): + self.calls.append((handle, request, event, p3_state, sink)) + marker = id(handle) + if marker in self.consumed: + return None + self.consumed.add(marker) + return SimpleNamespace(disposition=self.disposition, sink=self.sink, secret_snapshot_prose="never retained") + + +def _service(*, enabled: bool, verifier=None): + service = object.__new__(MemoryCompanionService) + service.config = _Config(enabled) + service._p5_attestation_verifier = verifier + service._p5_attestation_verifier_bound = verifier is not None + service.inject_calls: list[tuple[object, object, object]] = [] + + async def inject(ctx, req, *, event): + service.inject_calls.append((ctx, req, event)) + return "normal-memory-injection" + + service.inject_memories = inject + return service + + +class _BridgeInvocation: + def __init__(self, handle, request, event, p3_state): + self.values = (handle, request, event, p3_state) + + def _p5_b1_bridge_attestation_arguments(self): + return self.values + + +def _carriers(handle=object()): + request = SimpleNamespace() + event = SimpleNamespace() + setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {MEMORY_RECALL_SINK: handle}) + return request, event, handle + + +class P5B1NormalRecallGateTests(unittest.TestCase): + def test_first_bind_pins_service_and_rejects_rebind_or_replacement(self): + service = _service(enabled=False) + bridge = MemoryCompanionBridge(service) + first = _Verifier() + second = _Verifier() + self.assertTrue(bridge.bind_p5_attestation_verifier(first)) + self.assertIs(first, service._p5_attestation_verifier) + self.assertFalse(bridge.bind_p5_attestation_verifier(first)) + self.assertFalse(bridge.bind_p5_attestation_verifier(second)) + self.assertIs(first, service._p5_attestation_verifier) + self.assertFalse(service.bind_p5_attestation_verifier(second)) + + def test_concurrent_first_bind_has_one_winner_and_never_replaces_it(self): + service = _service(enabled=False) + bridge = MemoryCompanionBridge(service) + first = _Verifier() + second = _Verifier() + start = threading.Barrier(3) + results: list[tuple[object, bool]] = [] + result_lock = threading.Lock() + + def bind(verifier): + start.wait() + value = bridge.bind_p5_attestation_verifier(verifier) + with result_lock: + results.append((verifier, value)) + + first_thread = threading.Thread(target=bind, args=(first,)) + second_thread = threading.Thread(target=bind, args=(second,)) + first_thread.start() + second_thread.start() + start.wait() + first_thread.join(timeout=5) + second_thread.join(timeout=5) + self.assertFalse(first_thread.is_alive()) + self.assertFalse(second_thread.is_alive()) + winners = [verifier for verifier, accepted in results if accepted] + self.assertEqual(1, len(winners)) + self.assertIs(winners[0], bridge._p5_attestation_verifier) + self.assertIs(winners[0], service._p5_attestation_verifier) + + direct_service = _service(enabled=False) + direct_start = threading.Barrier(3) + direct_results: list[tuple[object, bool]] = [] + + def direct_bind(verifier): + direct_start.wait() + with result_lock: + direct_results.append((verifier, direct_service.bind_p5_attestation_verifier(verifier))) + + direct_first = threading.Thread(target=direct_bind, args=(first,)) + direct_second = threading.Thread(target=direct_bind, args=(second,)) + direct_first.start() + direct_second.start() + direct_start.wait() + direct_first.join(timeout=5) + direct_second.join(timeout=5) + direct_winners = [verifier for verifier, accepted in direct_results if accepted] + self.assertEqual(1, len(direct_winners)) + self.assertIs(direct_winners[0], direct_service._p5_attestation_verifier) + + def test_default_off_preserves_normal_injection_without_verifier_or_handle(self): + service = _service(enabled=False) + request, event, _ = _carriers() + setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {}) + injection, reason = asyncio.run( + service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=None) + ) + self.assertEqual(("normal-memory-injection", ""), (injection, reason)) + self.assertEqual(1, len(service.inject_calls)) + + def test_valid_allow_uses_exact_runtime_identities_and_reaches_normal_seam(self): + verifier = _Verifier() + service = _service(enabled=True, verifier=verifier) + request, event, handle = _carriers() + p3_state = object() + ctx = object() + injection, reason = asyncio.run( + service._p5_b1_normal_recall_injection(ctx, request, event=event, p3_state=p3_state) + ) + self.assertEqual(("normal-memory-injection", ""), (injection, reason)) + self.assertEqual([(handle, request, event, p3_state, MEMORY_RECALL_SINK)], verifier.calls) + self.assertEqual([(ctx, request, event)], service.inject_calls) + self.assertFalse(hasattr(request, "memory_companion_p5_b1_recall_gate_reason")) + + def test_missing_fake_replay_wrong_identity_sink_and_nonallow_fail_closed(self): + cases = [] + + missing_service = _service(enabled=True, verifier=_Verifier()) + missing_request, missing_event, _ = _carriers() + setattr(missing_request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {}) + cases.append((missing_service, missing_request, missing_event, object(), "p5_b1_handle_missing")) + + fake_service = _service(enabled=True, verifier=lambda *args: None) + fake_request, fake_event, _ = _carriers() + cases.append((fake_service, fake_request, fake_event, object(), "p5_b1_verifier_rejected")) + + sink_service = _service(enabled=True, verifier=_Verifier(sink="bridge_serialization")) + sink_request, sink_event, _ = _carriers() + cases.append((sink_service, sink_request, sink_event, object(), "p5_b1_sink_mismatch")) + + denied_service = _service(enabled=True, verifier=_Verifier(disposition="shadow_quarantine")) + denied_request, denied_event, _ = _carriers() + cases.append((denied_service, denied_request, denied_event, object(), "p5_b1_disposition_denied")) + + p3_service = _service(enabled=True, verifier=_Verifier()) + p3_request, p3_event, _ = _carriers() + cases.append((p3_service, p3_request, p3_event, None, "p5_b1_p3_unavailable")) + + for service, request, event, p3_state, expected in cases: + with self.subTest(expected=expected): + injection, reason = asyncio.run( + service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=p3_state) + ) + self.assertIsNone(injection) + self.assertEqual(expected, reason) + self.assertEqual([], service.inject_calls) + self.assertEqual(expected, request.memory_companion_p5_b1_recall_gate_reason) + rendered = repr(vars(request)) + repr(vars(event)) + self.assertNotIn("secret_snapshot_prose", rendered) + + replay_verifier = _Verifier() + replay_service = _service(enabled=True, verifier=replay_verifier) + replay_request, replay_event, _ = _carriers() + first, first_reason = asyncio.run( + replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object()) + ) + second, second_reason = asyncio.run( + replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object()) + ) + self.assertEqual(("normal-memory-injection", ""), (first, first_reason)) + self.assertEqual((None, "p5_b1_verifier_rejected"), (second, second_reason)) + + def test_rejected_identity_verifier_receives_no_fallback_or_snapshot(self): + expected_request, expected_event, _ = _carriers() + + def exact_only(handle, request, event, p3_state, sink): + if request is not expected_request or event is not expected_event or sink != MEMORY_RECALL_SINK: + return None + return SimpleNamespace(disposition="allow", sink=MEMORY_RECALL_SINK) + + service = _service(enabled=True, verifier=exact_only) + replacement_request, replacement_event, _ = _carriers() + injection, reason = asyncio.run( + service._p5_b1_normal_recall_injection( + object(), replacement_request, event=replacement_event, p3_state=object() + ) + ) + self.assertEqual((None, "p5_b1_verifier_rejected"), (injection, reason)) + self.assertEqual([], service.inject_calls) + + def test_bridge_gate_is_default_off_and_enabled_denials_stop_before_retrieval(self): + request = SimpleNamespace() + event = SimpleNamespace() + p3_state = object() + handle = object() + + def build_bridge_service(*, bridge_enabled: bool, verifier): + service = object.__new__(MemoryCompanionService) + service.config = _Config(False, bridge_enabled=bridge_enabled) + service._p5_attestation_verifier = verifier + service._p5_attestation_verifier_bound = verifier is not None + service.retrieval_calls = 0 + service.session_context_from_bridge = lambda _value: SimpleNamespace(message_text="query") + service._p3_bridge_gate_reason = lambda _ctx: "" + + async def compose(*_args, **_kwargs): + service.retrieval_calls += 1 + return "bridge-memory" + + service._compose_memory_injection = compose + return service + + allowed = _Verifier(sink=BRIDGE_SERIALIZATION_SINK) + enabled_service = build_bridge_service( + bridge_enabled=True, + verifier=allowed, + ) + result = asyncio.run( + enabled_service.bridge_compose_injection( + "query", + session_context={}, + p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state), + ) + ) + self.assertEqual("bridge-memory", result) + self.assertEqual(1, enabled_service.retrieval_calls) + self.assertEqual( + [(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK)], + allowed.calls, + ) + + replay = build_bridge_service(bridge_enabled=True, verifier=allowed) + replay_invocation = _BridgeInvocation(handle, request, event, p3_state) + self.assertEqual( + "", + asyncio.run( + replay.bridge_compose_injection( + "query", session_context={}, p5_b1_bridge_invocation=replay_invocation + ) + ), + ) + self.assertEqual(0, replay.retrieval_calls) + + for verifier in ( + _Verifier(disposition="shadow_quarantine", sink=BRIDGE_SERIALIZATION_SINK), + _Verifier(sink=MEMORY_RECALL_SINK), + ): + with self.subTest(verifier=verifier.disposition + verifier.sink): + denied = build_bridge_service(bridge_enabled=True, verifier=verifier) + self.assertEqual( + "", + asyncio.run( + denied.bridge_compose_injection( + "query", + session_context={}, + p5_b1_bridge_invocation=_BridgeInvocation(object(), request, event, p3_state), + ) + ), + ) + self.assertEqual(0, denied.retrieval_calls) + + for invocation in ( + None, + {}, + _BridgeInvocation(object(), SimpleNamespace(), event, p3_state), + _BridgeInvocation(object(), request, event, p3_state), + ): + with self.subTest(invocation=type(invocation).__name__): + denied = build_bridge_service(bridge_enabled=True, verifier=lambda *args: None) + blocked = asyncio.run( + denied.bridge_compose_injection( + "query", + session_context={}, + p5_b1_bridge_invocation=invocation, + ) + ) + if invocation is None: + self.assertEqual("bridge-memory", blocked) + self.assertEqual(1, denied.retrieval_calls) + else: + self.assertEqual("", blocked) + self.assertEqual(0, denied.retrieval_calls) + + default_off = build_bridge_service(bridge_enabled=False, verifier=None) + legacy = asyncio.run( + default_off.bridge_compose_injection( + "query", + session_context={}, + p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state), + ) + ) + self.assertEqual("bridge-memory", legacy) + self.assertEqual(1, default_off.retrieval_calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p5_b1_unverifiable_read_gates.py b/tests/test_p5_b1_unverifiable_read_gates.py new file mode 100644 index 0000000..9c81ae7 --- /dev/null +++ b/tests/test_p5_b1_unverifiable_read_gates.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +import unittest + +from core.service import MemoryCompanionService + + +class _Config: + def __init__(self, **flags: bool) -> None: + self.flags = flags + + def bool(self, key: str, default: bool = False) -> bool: + return bool(self.flags.get(key, default)) + + def int(self, _key: str, default: int = 0) -> int: + return default + + +class _Identity: + async def resolve_event_context(self, _event): + return SimpleNamespace() + + +def _service(**flags: bool): + service = object.__new__(MemoryCompanionService) + service.config = _Config(**flags) + service.identity = _Identity() + service._p3_prepare_context = lambda _ctx, **_kwargs: object() + service._p3_gate_reason = lambda *_args, **_kwargs: "" + service._archive_context = lambda _scope: SimpleNamespace( + session_id="s1", scope="private", platform="test", user_id="u1", group_id="", bot_id="b1", + message_id="m1", strict_session_only=True, context_orchestration=None, + ) + + async def archive_scope(*, scope=None): + return {"scope": dict(scope or {})} + + service.bridge_get_archive_scope = archive_scope + return service + + +class P5B1UnverifiableReadGateTests(unittest.TestCase): + def test_tool_recall_default_off_preserves_search_and_enabled_stops_before_search(self): + default_off = _service() + default_off.search_calls = 0 + + async def search(*_args, **_kwargs): + default_off.search_calls += 1 + return [] + + default_off.search = search + self.assertEqual( + {"ok": True, "memories": []}, asyncio.run(default_off.tool_recall(object(), "needle")) + ) + self.assertEqual(1, default_off.search_calls) + + enabled = _service(enable_p5_b1_tool_recall_gate=True) + + async def forbidden_search(*_args, **_kwargs): + raise AssertionError("tool gate must stop before search") + + enabled.search = forbidden_search + self.assertEqual( + {"ok": False, "error": "p5_attestation_missing", "memories": []}, + asyncio.run(enabled.tool_recall(object(), "needle")), + ) + + def test_archive_gate_stops_search_evidence_and_timeline_before_data_reads(self): + service = _service(enable_p5_b2_archive_read_gate=True) + + async def forbidden(*_args, **_kwargs): + raise AssertionError("archive gate must stop before a data read") + + service.search = forbidden + service.store = SimpleNamespace(get_memories_by_ids=forbidden, get_timeline_event=forbidden) + searched = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"})) + evidence = asyncio.run(service.bridge_get_archive_evidence(["m1"], scope={"scope": "private"})) + context = asyncio.run( + service.bridge_get_archive_session_context( + scope={"scope": "private"}, start_at="2026-01-01T00:00:00+00:00", end_at="2026-02-01T00:00:00+00:00" + ) + ) + self.assertEqual(["p5_attestation_missing"], searched["warnings"]) + self.assertEqual(([], [], ["p5_attestation_missing"]), (evidence["memories"], evidence["excluded"], evidence["warnings"])) + self.assertEqual(([], ["p5_attestation_missing"]), (context["items"], context["warnings"])) + + def test_archive_default_off_reaches_legacy_search(self): + service = _service() + service.search_calls = 0 + + async def search(*_args, **_kwargs): + service.search_calls += 1 + return [] + + service.search = search + service.archive_visibility_policy = lambda: object() + result = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"})) + self.assertEqual([], result["memories"]) + self.assertEqual(1, service.search_calls) + + def test_cross_user_gate_returns_empty_without_delegating_to_archive(self): + service = _service(enable_p5_b2_cross_user_read_gate=True) + + async def forbidden(*_args, **_kwargs): + raise AssertionError("cross-user gate must stop before archive delegation") + + service.bridge_search_archive_memories = forbidden + result = asyncio.run( + service.bridge_query_user_facts("needle", scope={"scope": "private", "user_id": "u1"}) + ) + self.assertEqual(([], [], ["p5_attestation_missing"]), (result["memories"], result["excluded"], result["warnings"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p5_provenance.py b/tests/test_p5_provenance.py new file mode 100644 index 0000000..f58a8d5 --- /dev/null +++ b/tests/test_p5_provenance.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import copy +from dataclasses import dataclass +import hashlib +import json +import unittest + +from core.p5_provenance import ( + COMPANION_ATTESTATION_ISSUER, + P3_CONTRACT_FINGERPRINT, + P3_CONTRACT_NAME, + P3_CONTRACT_VERSION, + PROVENANCE_CONTRACT_NAME, + PROVENANCE_CONTRACT_VERSION, + apply_planned_operation, + contract_descriptor, + contract_fingerprint, + legacy_unresolved, + observed_from_companion_snapshot, + plan_legacy_migration, + plan_owner_recovery, + rollback_planned_operation, + validate_provenance_record, +) + + +def _hash(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class _CompanionStyleSnapshot: + schema_version: str + contract_name: str + contract_version: str + contract_fingerprint: str + issuer: str + issuer_epoch: str + authority_attestation_ref_hash: str + request_hash: str + session_hash: str + source_kind: str + source_trust: str + firewall_status: str + disposition: str + reason_codes: tuple[str, ...] + source_event_ref_hash: str + derived_from_ref_hash: str + provenance_state: str + p3_contract_name: str + p3_contract_version: str + p3_contract_fingerprint: str + sink: str + + +def _snapshot(**overrides): + value = { + "schema_version": "ops.p5.attestation.v1", + "contract_name": PROVENANCE_CONTRACT_NAME, + "contract_version": PROVENANCE_CONTRACT_VERSION, + "contract_fingerprint": contract_fingerprint(), + "issuer": COMPANION_ATTESTATION_ISSUER, + "issuer_epoch": _hash("issuer-epoch-1"), + "p3_contract_name": P3_CONTRACT_NAME, + "p3_contract_version": P3_CONTRACT_VERSION, + "p3_contract_fingerprint": P3_CONTRACT_FINGERPRINT, + "source_kind": "forwarded_text", + "source_trust": "T3", + "firewall_status": "sanitized", + "disposition": "shadow_quarantine", + "reason_codes": ("p5b_attested",), + "source_event_ref_hash": _hash("source-event-1"), + "authority_attestation_ref_hash": _hash("companion-attestation-1"), + "request_hash": _hash("request-1"), + "session_hash": _hash("session-1"), + "derived_from_ref_hash": "", + "provenance_state": "observed", + "sink": "memory_recall", + } + value.update(overrides) + return value + + +class P5ProvenanceTests(unittest.TestCase): + def test_descriptor_and_legacy_record_are_json_serializable(self): + descriptor = contract_descriptor() + expected = { + "contract_name": "ops.p5.provenance.v1", + "contract_version": "1.0", + "record_fields": [ + "contract_name", "contract_version", "contract_fingerprint", "memory_id", + "source_kind", "source_trust", "firewall_status", "source_event_ref_hash", + "authority_attestation_ref_hash", "provenance_state", "migration_operation_ref", + "recovery_operation_ref", "record_revision", + ], + "provenance_states": ["observed", "legacy_unresolved", "owner_recovered", "invalid"], + "trust_levels": ["T0", "T1", "T2", "T3", "T4"], + "source_kinds": [ + "policy_config", "verified_authorization", "current_user_intent", "forwarded_text", + "quoted_text", "vision_summary", "tool_output", "web_extract", "memory_recall", + "derived_summary", "legacy_memory", "unknown", + ], + "firewall_statuses": [ + "allowed", "sanitized", "blocked", "rejected", "quarantined", "unavailable", "unknown", + ], + "dispositions": ["allow", "shadow_quarantine", "deny_high_risk"], + "hash_format": "sha256_hex_lower", + "attestation_issuer": "private_companion", + "p3_contract_name": P3_CONTRACT_NAME, + "p3_contract_version": P3_CONTRACT_VERSION, + } + self.assertEqual(expected, descriptor) + canonical = json.dumps(expected, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + self.assertEqual(hashlib.sha256(canonical.encode("utf-8")).hexdigest(), contract_fingerprint()) + record = legacy_unresolved("memory-42", record_revision=4) + self.assertEqual(PROVENANCE_CONTRACT_NAME, descriptor["contract_name"]) + self.assertEqual(PROVENANCE_CONTRACT_VERSION, descriptor["contract_version"]) + self.assertEqual(64, len(contract_fingerprint())) + self.assertEqual("legacy_unresolved", record["provenance_state"]) + self.assertEqual(("unknown", "T4", "unknown"), (record["source_kind"], record["source_trust"], record["firewall_status"])) + self.assertTrue(validate_provenance_record(record)["ok"]) + json.dumps({"descriptor": descriptor, "record": record}, ensure_ascii=False, sort_keys=True) + + def test_only_attested_exact_p3_snapshot_can_be_observed(self): + observed = observed_from_companion_snapshot( + "memory-42", + _snapshot( + source_event_ref_hash=f"sha256:{_hash('source-event-1').upper()}", + authority_attestation_ref_hash=f"sha256:{_hash('companion-attestation-1').upper()}", + ), + record_revision=7, + ) + self.assertEqual("observed", observed["provenance_state"]) + self.assertTrue(validate_provenance_record(observed)["ok"]) + self.assertEqual(_hash("source-event-1"), observed["source_event_ref_hash"]) + self.assertEqual(_hash("companion-attestation-1"), observed["authority_attestation_ref_hash"]) + self.assertNotIn("disposition", observed) + noncanonical_record = copy.deepcopy(observed) + noncanonical_record["source_event_ref_hash"] = f"sha256:{_hash('source-event-1')}" + self.assertFalse(validate_provenance_record(noncanonical_record)["ok"]) + + companion_style = _CompanionStyleSnapshot(**_snapshot()) + from_dataclass = observed_from_companion_snapshot("memory-43", companion_style) + self.assertEqual("observed", from_dataclass["provenance_state"]) + + for name, override in ( + ("pseudo source plugin", {"source_kind": "source_plugin"}), + ("p5 descriptor mismatch", {"contract_fingerprint": _hash("wrong-p5")}), + ("p3 mismatch", {"p3_contract_fingerprint": _hash("wrong-p3")}), + ("missing attestation", {"authority_attestation_ref_hash": ""}), + ("invalid trust", {"source_trust": "T9"}), + ("conflicting source hashes", {"source_hash": _hash("other-source")}), + ("same roles hash", {"authority_attestation_ref_hash": _hash("source-event-1")}), + ("status used as disposition", {"disposition": "sanitized"}), + ("disposition used as status", {"firewall_status": "shadow_quarantine"}), + ): + with self.subTest(name=name): + rejected = observed_from_companion_snapshot("memory-42", _snapshot(**override)) + self.assertEqual("invalid", rejected["provenance_state"]) + + def test_preview_is_zero_write_and_does_not_mutate_or_capture_content(self): + source = [{"memory_id": "memory-42", "record_revision": 4}] + before = copy.deepcopy(source) + preview = plan_legacy_migration(source, operation_ref_hash=_hash("migration-42")) + self.assertEqual(before, source) + self.assertEqual(("preview", True, 0), (preview["mode"], preview["readonly"], preview["write_count"])) + self.assertEqual(1, len(preview["operations"])) + self.assertEqual([], preview["error_codes"]) + self.assertEqual("legacy_unresolved", preview["operations"][0]["after_record"]["provenance_state"]) + + hostile = [{"memory_id": "memory-43", "record_revision": 0, "content": "do not persist this"}] + rejected = plan_legacy_migration(hostile, operation_ref_hash=_hash("migration-43")) + rendered = json.dumps(rejected, ensure_ascii=False, sort_keys=True) + self.assertNotIn("do not persist this", rendered) + self.assertNotIn("memory-43", rendered) + self.assertEqual([], rejected["operations"]) + + def test_apply_retry_and_rollback_are_revision_guarded(self): + preview = plan_legacy_migration( + [{"memory_id": "memory-42", "record_revision": 4}], operation_ref_hash=_hash("migration-42") + ) + operation = preview["operations"][0] + current = {"memory_id": "memory-42", "record_revision": 4} + applied = apply_planned_operation(current, operation) + self.assertEqual((True, "applied", True), (applied["ok"], applied["status"], applied["changed"])) + retry = apply_planned_operation(applied["record"], operation) + self.assertEqual((True, "idempotent", False), (retry["ok"], retry["status"], retry["changed"])) + + concurrent = copy.deepcopy(applied["record"]) + concurrent["record_revision"] += 1 + rejected = rollback_planned_operation(concurrent, operation) + self.assertEqual((False, "revision_conflict"), (rejected["ok"], rejected["status"])) + rollback = rollback_planned_operation(applied["record"], operation) + self.assertEqual((True, "rolled_back", True), (rollback["ok"], rollback["status"], rollback["removed"])) + self.assertIsNone(rollback["record"]) + + def test_owner_recovery_requires_companion_evidence_and_keeps_operation_ref(self): + legacy = legacy_unresolved("memory-42", record_revision=2) + planned = plan_owner_recovery( + legacy, _snapshot(), recovery_operation_ref_hash=_hash("recovery-42") + ) + self.assertTrue(planned["ok"]) + after = planned["operation"]["after_record"] + self.assertEqual("owner_recovered", after["provenance_state"]) + self.assertEqual(_hash("recovery-42"), after["recovery_operation_ref"]) + + rejected = plan_owner_recovery( + legacy, + _snapshot(p3_contract_fingerprint=_hash("wrong-p3")), + recovery_operation_ref_hash=_hash("recovery-43"), + ) + self.assertFalse(rejected["ok"]) + self.assertEqual(["companion_snapshot_invalid"], rejected["error_codes"]) + + def test_record_validation_rejects_raw_source_ids_and_bodies_without_echo(self): + record = legacy_unresolved("memory-42") + record["source_event_id"] = "raw-event-99" + record["content"] = "hostile body" + result = validate_provenance_record(record) + self.assertFalse(result["ok"]) + rendered = json.dumps(result, ensure_ascii=False, sort_keys=True) + self.assertNotIn("raw-event-99", rendered) + self.assertNotIn("hostile body", rendered) + self.assertIn("forbidden_raw_field", result["error_codes"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_p5_provenance_persistence.py b/tests/test_p5_provenance_persistence.py new file mode 100644 index 0000000..ae60d57 --- /dev/null +++ b/tests/test_p5_provenance_persistence.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import asyncio +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from core.bridge import MemoryCompanionBridge +from core.models import MemoryRecord +from core.p5_provenance_persistence import P5ProvenanceMigration +from core.service import MemoryCompanionService +from core.store import MemoryStore + + +class _MigrationDouble: + async def preview(self, *, limit: int): + return {"mode": "preview", "ok": True, "limit": limit} + + async def apply(self, *, operation_id: str, limit: int): + return {"mode": "apply", "ok": True, "operation_id": operation_id, "limit": limit} + + async def rollback(self, operation_id: str): + return {"mode": "rollback", "ok": True, "operation_id": operation_id} + + +class P5ProvenancePersistenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.store = MemoryStore(Path(self.temp.name) / "memory.db") + self.store.initialize() + self.migration = P5ProvenanceMigration(self.store) + + def tearDown(self): + self.store.close() + self.temp.cleanup() + + def _seed(self, identifier: str, *, source_plugin: str = "memory_companion") -> str: + return asyncio.run( + self.store.insert_memory( + MemoryRecord( + id=identifier, + memory_type="observation", + scope="private", + session_id="private:test", + content=f"memory {identifier}", + evidence="test evidence", + source_plugin=source_plugin, + ) + ) + ) + + def test_schema_is_additive_and_preview_is_zero_write(self): + self._seed("memory-001") + tables = { + row["name"] + for row in self.store._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'p5_provenance_%'" + ).fetchall() + } + self.assertEqual( + { + "p5_provenance_records", + "p5_provenance_migration_runs", + "p5_provenance_migration_backups", + }, + tables, + ) + before_changes = self.store._conn.total_changes + preview = asyncio.run(self.migration.preview(limit=50)) + self.assertEqual(("preview", True, True, 1, 0), ( + preview["mode"], preview["ok"], preview["readonly"], preview["count"], preview["write_count"], + )) + self.assertEqual(before_changes, self.store._conn.total_changes) + self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0]) + + def test_apply_creates_legacy_records_and_backup_manifest_idempotently(self): + self._seed("memory-001", source_plugin="source_plugin") + self._seed("memory-002") + result = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50)) + self.assertEqual(("apply", True, 2, False, 2), ( + result["mode"], result["ok"], result["changed"], result["deduplicated"], result["backup_count"], + )) + backup_path = Path(result["backup_path"]) + self.assertTrue(backup_path.is_file()) + backup_connection = sqlite3.connect(backup_path) + try: + self.assertEqual(0, backup_connection.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0]) + finally: + backup_connection.close() + rows = self.store._conn.execute( + "SELECT source_kind, source_trust, firewall_status, provenance_state, source_event_ref_hash, authority_attestation_ref_hash FROM p5_provenance_records ORDER BY memory_id" + ).fetchall() + self.assertEqual(2, len(rows)) + for row in rows: + self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved", "", ""), tuple(row)) + manifest = self.store._conn.execute( + "SELECT before_exists, before_digest, before_revision, after_digest, after_revision, apply_state FROM p5_provenance_migration_backups ORDER BY memory_id" + ).fetchall() + self.assertEqual(2, len(manifest)) + self.assertTrue(all(tuple(row)[0:3] == (0, "", 0) and row[4] == 1 and row[5] == "applied" for row in manifest)) + retry = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50)) + self.assertEqual((True, 0, True, 2), (retry["ok"], retry["changed"], retry["deduplicated"], retry["backup_count"])) + self.assertEqual(str(backup_path), retry["backup_path"]) + + def test_invalid_or_failed_backup_leaves_no_run_or_provenance_mutation(self): + self._seed("memory-001") + backup_glob = "memory.backup.*.before_p5_provenance_b0.*.db" + before_paths = set(Path(self.temp.name).glob(backup_glob)) + invalid = asyncio.run(self.migration.apply(operation_id="", limit=50)) + self.assertEqual("operation_id_required", invalid["error_code"]) + self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob))) + self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0]) + + original_backup = self.store.backup + self.store.backup = lambda suffix="": (_ for _ in ()).throw(RuntimeError("backup unavailable")) + try: + failed = asyncio.run(self.migration.apply(operation_id="p5-b0-backup-fail", limit=50)) + finally: + self.store.backup = original_backup + self.assertEqual((False, "physical_backup_failed", 0), (failed["ok"], failed["error_code"], failed["changed"])) + self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob))) + self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0]) + self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0]) + + def test_rollback_deletes_only_untouched_operation_owned_records(self): + self._seed("memory-001") + asyncio.run(self.migration.apply(operation_id="p5-b0-rollback-1", limit=50)) + rollback = asyncio.run(self.migration.rollback("p5-b0-rollback-1")) + self.assertEqual(("rollback", True, 1, 0), (rollback["mode"], rollback["ok"], rollback["restored"], rollback["conflicts"])) + self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0]) + duplicate = asyncio.run(self.migration.rollback("p5-b0-rollback-1")) + self.assertTrue(duplicate["deduplicated"]) + + def test_rollback_cas_conflict_preserves_independent_newer_record(self): + self._seed("memory-001") + asyncio.run(self.migration.apply(operation_id="p5-b0-conflict-1", limit=50)) + self.store._conn.execute( + "UPDATE p5_provenance_records SET record_revision=record_revision+1 WHERE memory_id='memory-001'" + ) + self.store._conn.commit() + rollback = asyncio.run(self.migration.rollback("p5-b0-conflict-1")) + self.assertEqual((False, 0, 1), (rollback["ok"], rollback["restored"], rollback["conflicts"])) + record = self.store._conn.execute( + "SELECT record_revision, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'" + ).fetchone() + self.assertEqual((2, "legacy_unresolved"), tuple(record)) + backup = self.store._conn.execute( + "SELECT rollback_state FROM p5_provenance_migration_backups WHERE operation_id='p5-b0-conflict-1' AND memory_id='memory-001'" + ).fetchone() + self.assertEqual("conflict", backup[0]) + + def test_forged_source_and_p3_metadata_never_upgrades_legacy_backfill(self): + self._seed("memory-001", source_plugin="source_plugin") + self.store._conn.execute( + "UPDATE memories SET metadata=? WHERE id='memory-001'", + ('{"source_kind":"source_plugin","p3_contract_fingerprint":"forged"}',), + ) + self.store._conn.commit() + asyncio.run(self.migration.apply(operation_id="p5-b0-forged-1", limit=50)) + record = self.store._conn.execute( + "SELECT source_kind, source_trust, firewall_status, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'" + ).fetchone() + self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved"), tuple(record)) + + def test_service_and_bridge_migration_wrappers_and_unavailable_bridge(self): + service = object.__new__(MemoryCompanionService) + service.p5_provenance_migration = _MigrationDouble() + self.assertEqual("preview", asyncio.run(service.preview_p5_provenance_migration(limit=7))["mode"]) + self.assertEqual("op-1", asyncio.run(service.apply_p5_provenance_migration(operation_id="op-1", limit=7))["operation_id"]) + self.assertEqual("rollback", asyncio.run(service.rollback_p5_provenance_migration("op-1"))["mode"]) + + unavailable = MemoryCompanionBridge(object()) + result = asyncio.run(unavailable.preview_p5_provenance_migration(limit=7)) + self.assertEqual((False, "bridge_method_unavailable"), (result["ok"], result["error_code"])) + applied = asyncio.run(unavailable.apply_p5_provenance_migration(operation_id="op-1", limit=7)) + self.assertEqual((False, "bridge_method_unavailable", 0), (applied["ok"], applied["error_code"], applied["changed"])) + rolled_back = asyncio.run(unavailable.rollback_p5_provenance_migration("op-1")) + self.assertEqual((False, "bridge_method_unavailable", 0), (rolled_back["ok"], rolled_back["error_code"], rolled_back["restored"])) + self.assertFalse(unavailable.probe_p5_provenance_migration_capabilities()["available"]) + + +if __name__ == "__main__": + unittest.main()