feat(p5): add provenance isolation gates

这个提交存在于:
YokiiroBW 2026-07-30 01:12:07 +08:00
父节点 6db395beb9
当前提交 274c9b95ea
共有 12 个文件被更改,包括 3008 次插入 和 12 次删除

查看文件

@ -1,6 +1,7 @@
from __future__ import annotations
from datetime import datetime, timezone
import threading
from typing import Any
from zoneinfo import ZoneInfo
@ -15,6 +16,10 @@ from .unified_person_contract import (
)
from .person_memory_contract import capabilities as person_memory_capabilities, consume_person_memory_envelope
from .context_orchestration import contract_descriptor, contract_fingerprint
from .p5_provenance import (
contract_descriptor as p5_provenance_contract_descriptor,
contract_fingerprint as p5_provenance_contract_fingerprint,
)
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
@ -43,6 +48,28 @@ class MemoryCompanionBridge:
def __init__(self, plugin: Any):
self._plugin = plugin
self._p5_attestation_verifier: Any = None
self._p5_attestation_verifier_bound = False
self._p5_attestation_verifier_lock = threading.RLock()
def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool:
"""Pin the first in-process P5 verifier; replacement is forbidden."""
if not callable(consume_callable):
return False
with self._p5_attestation_verifier_lock:
if self._p5_attestation_verifier_bound:
return False
target_binder = getattr(self._plugin, "bind_p5_attestation_verifier", None)
if callable(target_binder):
try:
if target_binder(consume_callable) is not True:
return False
except Exception:
return False
self._p5_attestation_verifier = consume_callable
self._p5_attestation_verifier_bound = True
return True
def _trusted_session_context(self, session_context: SessionContext | dict[str, Any] | None) -> SessionContext | dict[str, Any] | None:
"""Normalize a legacy dict at the public bridge boundary.
@ -164,6 +191,23 @@ class MemoryCompanionBridge:
result["operations"] = list(methods); result["operation_availability"] = operation_availability
return result
def probe_p5_provenance_migration_capabilities(self) -> dict[str, Any]:
methods = {
"preview": "preview_p5_provenance_migration",
"apply": "apply_p5_provenance_migration",
"rollback": "rollback_p5_provenance_migration",
}
availability = {name: callable(getattr(self._plugin, method, None)) for name, method in methods.items()}
return {
"available": all(availability.values()),
"mode": "migration_only",
"runtime_sink_behavior": "not_connected",
"contract": p5_provenance_contract_descriptor(),
"contract_fingerprint": p5_provenance_contract_fingerprint(),
"operations": list(methods),
"operation_availability": availability,
}
def consume_person_memory_envelope(self, envelope: dict[str, Any] | None, *, companion_available: bool = True) -> dict[str, Any]:
return consume_person_memory_envelope(envelope, companion_available=companion_available)
@ -182,6 +226,27 @@ class MemoryCompanionBridge:
if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"}
return await getter(migration_id)
async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]:
getter = getattr(self._plugin, "preview_p5_provenance_migration", None)
if not callable(getter):
return {"mode": "preview", "ok": False, "readonly": True, "error_code": "bridge_method_unavailable", "count": 0}
result = await getter(limit=limit)
return result if isinstance(result, dict) else {"mode": "preview", "ok": False, "readonly": True, "error_code": "invalid_bridge_response", "count": 0}
async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
getter = getattr(self._plugin, "apply_p5_provenance_migration", None)
if not callable(getter):
return {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "changed": 0}
result = await getter(operation_id=operation_id, limit=limit)
return result if isinstance(result, dict) else {"mode": "apply", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "changed": 0}
async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]:
getter = getattr(self._plugin, "rollback_p5_provenance_migration", None)
if not callable(getter):
return {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "bridge_method_unavailable", "restored": 0, "conflicts": 0}
result = await getter(operation_id)
return result if isinstance(result, dict) else {"mode": "rollback", "ok": False, "operation_id": operation_id, "error_code": "invalid_bridge_response", "restored": 0, "conflicts": 0}
async def correct_person_memory(self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord) -> dict[str, Any]:
getter = getattr(self._plugin, "bridge_correct_person_memory", None)
if not callable(getter): return {"ok": False, "error_code": "bridge_method_unavailable"}
@ -702,6 +767,7 @@ class MemoryCompanionBridge:
max_chars: int | None = None,
companion_bot_mood: str = "",
companion_bot_energy: float = 0.0,
p5_b1_bridge_invocation: Any = None,
) -> str:
return await self._plugin.bridge_compose_injection(
query,
@ -710,6 +776,7 @@ class MemoryCompanionBridge:
max_chars=max_chars,
companion_bot_mood=companion_bot_mood,
companion_bot_energy=companion_bot_energy,
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
)
def get_reply_delay(
@ -739,6 +806,7 @@ class MemoryCompanionBridge:
companion_bot_mood: str = "",
companion_bot_energy: float = 0.0,
retrieval_profile: str = "",
p5_b1_bridge_invocation: Any = None,
) -> str:
return await self._plugin.bridge_compose_context(
query=query,
@ -748,6 +816,7 @@ class MemoryCompanionBridge:
companion_bot_mood=companion_bot_mood,
companion_bot_energy=companion_bot_energy,
retrieval_profile=retrieval_profile,
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
)
async def create_archive_memory(

139
core/p5_b1_recall_gate.py 普通文件
查看文件

@ -0,0 +1,139 @@
"""P5-B1 attestation gates for normal recall and Bridge serialization.
The handle carrier is deliberately untrusted and private to the current
request/event runtime objects. This module never serializes it, stores it,
or interprets a source trust label. Authenticity belongs solely to the
one-shot verifier pinned by ``MemoryCompanionBridge``.
"""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any, Callable
P5_B1_ATTESTATION_HANDLES_ATTRIBUTE = "private_companion_p5_b1_attestation_handles"
MEMORY_RECALL_SINK = "memory_recall"
BRIDGE_SERIALIZATION_SINK = "bridge_serialization"
_BRIDGE_INVOCATION_ARGUMENTS_METHOD = "_p5_b1_bridge_attestation_arguments"
def _carrier_handle(req: Any, event: Any) -> Any:
"""Read a carrier value without copying it onto another object."""
for carrier in (req, event):
handles = getattr(carrier, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, None)
if isinstance(handles, Mapping) and MEMORY_RECALL_SINK in handles:
return handles.get(MEMORY_RECALL_SINK)
return None
def _snapshot_field(snapshot: Any, name: str) -> Any:
if isinstance(snapshot, Mapping):
return snapshot.get(name)
return getattr(snapshot, name, None)
def evaluate_normal_recall_gate(
verifier: Callable[..., Any] | None,
*,
req: Any,
event: Any,
p3_state: Any,
) -> str:
"""Return a static safe reason or ``""`` when normal recall may proceed.
The pinned verifier receives the original object identities. It may
consume a Companion opaque handle only once; any error, missing carrier,
unavailable P3 anchor, or non-allow result fails closed without retaining
its error or snapshot payload.
"""
if not callable(verifier):
return "p5_b1_verifier_unavailable"
if p3_state is None:
return "p5_b1_p3_unavailable"
handle = _carrier_handle(req, event)
if handle is None:
return "p5_b1_handle_missing"
try:
snapshot = verifier(handle, req, event, p3_state, MEMORY_RECALL_SINK)
except Exception:
return "p5_b1_verifier_rejected"
if snapshot is None:
return "p5_b1_verifier_rejected"
if _snapshot_field(snapshot, "sink") != MEMORY_RECALL_SINK:
return "p5_b1_sink_mismatch"
if _snapshot_field(snapshot, "disposition") != "allow":
return "p5_b1_disposition_denied"
return ""
def _bridge_invocation_arguments(invocation: Any) -> tuple[Any, Any, Any, Any] | None:
"""Read live anchors from an opaque invocation without retaining them.
A mapping, serialized reconstruction, or arbitrary payload cannot be an
invocation. Even an object that mimics the private method cannot create a
valid authorization because the pinned verifier still receives and checks
the opaque issuer handle plus every exact object identity.
"""
if invocation is None or isinstance(invocation, Mapping):
return None
argument_getter = getattr(invocation, _BRIDGE_INVOCATION_ARGUMENTS_METHOD, None)
if not callable(argument_getter):
return None
try:
arguments = argument_getter()
except Exception:
return None
if type(arguments) is not tuple or len(arguments) != 4:
return None
return arguments
def evaluate_bridge_serialization_gate(
verifier: Callable[..., Any] | None,
*,
invocation: Any,
) -> str:
"""Return a static reason unless an opaque Bridge invocation is allowed.
The call is deliberately separate from normal recall. The actual pinned
verifier is always called by Memory with its fixed ``bridge_serialization``
sink; the invocation never supplies a sink, a snapshot, a SessionContext,
or a persistence payload.
"""
if not callable(verifier):
return "p5_b1_bridge_verifier_unavailable"
arguments = _bridge_invocation_arguments(invocation)
if arguments is None:
return "p5_b1_bridge_invocation_invalid"
handle, request, event, p3_state = arguments
if p3_state is None:
return "p5_b1_bridge_p3_unavailable"
if request is None or event is None:
return "p5_b1_bridge_carrier_missing"
if handle is None:
return "p5_b1_bridge_handle_missing"
try:
snapshot = verifier(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK)
except Exception:
return "p5_b1_bridge_verifier_rejected"
if snapshot is None:
return "p5_b1_bridge_verifier_rejected"
if _snapshot_field(snapshot, "sink") != BRIDGE_SERIALIZATION_SINK:
return "p5_b1_bridge_sink_mismatch"
if _snapshot_field(snapshot, "disposition") != "allow":
return "p5_b1_bridge_disposition_denied"
return ""
__all__ = [
"BRIDGE_SERIALIZATION_SINK",
"MEMORY_RECALL_SINK",
"P5_B1_ATTESTATION_HANDLES_ATTRIBUTE",
"evaluate_bridge_serialization_gate",
"evaluate_normal_recall_gate",
]

799
core/p5_provenance.py 普通文件
查看文件

@ -0,0 +1,799 @@
"""P5-B metadata-only provenance and migration planning contract.
This module deliberately has no dependency on MemoryRecord, Store, Service,
Bridge, or a database. It is the small, deterministic contract that a later
integration may use to *plan* provenance backfill and guarded rollback. It
does not classify content, read a body, perform I/O, or grant a live recall or
tool permission.
Only :func:`observed_from_companion_snapshot` can create an ``observed``
record. It requires the frozen P3 carrier reference and a Companion
attestation reference. Historical records are represented explicitly as
``legacy_unresolved`` with ``unknown`` / ``T4`` provenance instead of being
silently upgraded.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from copy import deepcopy
from dataclasses import asdict, is_dataclass
import hashlib
import json
import re
from typing import Any
PROVENANCE_CONTRACT_NAME = "ops.p5.provenance.v1"
PROVENANCE_CONTRACT_VERSION = "1.0"
# P3 remains owned by PrivateCompanion. This consumer duplicates only its
# already-frozen identity, never its state machine or source classifier.
P3_CONTRACT_NAME = "ops.context_orchestration.v1"
P3_CONTRACT_VERSION = "1.0"
P3_CONTRACT_FINGERPRINT = (
"3bb7a12af05bb4d9a47cef9f31e78752cec512b90b1d50ae49feef54470974f8"
)
COMPANION_ATTESTATION_ISSUER = "private_companion"
ATTESTATION_SCHEMA_VERSION = "ops.p5.attestation.v1"
TRUST_LEVELS = ("T0", "T1", "T2", "T3", "T4")
PROVENANCE_STATES = ("observed", "legacy_unresolved", "owner_recovered", "invalid")
# Ordered tuples are part of the shared cross-repository descriptor. Trust
# must remain explicit: source kind and firewall status never imply it.
SOURCE_KINDS = (
"policy_config",
"verified_authorization",
"current_user_intent",
"forwarded_text",
"quoted_text",
"vision_summary",
"tool_output",
"web_extract",
"memory_recall",
"derived_summary",
"legacy_memory",
"unknown",
)
FIREWALL_STATUSES = (
"allowed",
"sanitized",
"blocked",
"rejected",
"quarantined",
"unavailable",
"unknown",
)
DISPOSITIONS = ("allow", "shadow_quarantine", "deny_high_risk")
HASH_FORMAT = "sha256_hex_lower"
_SOURCE_KIND_SET = frozenset(SOURCE_KINDS)
_ATTESTATION_SINKS = frozenset(
{
"prompt_forward_quote",
"prompt_vision_summary",
"memory_recall",
"bridge_serialization",
"tool_retrieval",
"external_export",
"cross_user_read",
}
)
RECORD_FIELDS = (
"contract_name",
"contract_version",
"contract_fingerprint",
"memory_id",
"source_kind",
"source_trust",
"firewall_status",
"source_event_ref_hash",
"authority_attestation_ref_hash",
"provenance_state",
"migration_operation_ref",
"recovery_operation_ref",
"record_revision",
)
OPERATION_FIELDS = (
"contract_name",
"contract_version",
"contract_fingerprint",
"operation_kind",
"operation_ref_hash",
"memory_id",
"expected_revision",
"before_record_digest",
"after_record_digest",
"before_record",
"after_record",
)
_TOKEN_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
_MEMORY_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$")
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
_FORBIDDEN_RAW_FIELDS = frozenset(
{
"body",
"content",
"text",
"message",
"raw_id",
"source_id",
"source_event_id",
"event_id",
"attestation_id",
"authority_attestation_id",
"request_id",
"session_id",
}
)
_SNAPSHOT_FIELDS = frozenset(
{
"schema_version",
"contract_name",
"contract_version",
"contract_fingerprint",
"issuer",
"issuer_epoch",
"p3_contract_name",
"p3_contract_version",
"p3_contract_fingerprint",
"source_kind",
"source_trust",
"trust",
"firewall_status",
"source_event_ref_hash",
"source_hash",
"authority_attestation_ref_hash",
"attestation_ref_hash",
# The following are bounded, non-authoritative diagnostic metadata
# emitted by the companion registry. They never enter a P5 record.
"disposition",
"reason_codes",
"request_hash",
"session_hash",
"derived_hash",
"derived_from_ref_hash",
"provenance_state",
"sink",
}
)
def _canonical_json(value: Any) -> str:
return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
def _digest(value: Mapping[str, Any]) -> str:
return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest()
def contract_descriptor() -> dict[str, Any]:
"""Return the frozen, JSON-serializable P5 provenance descriptor."""
return {
"contract_name": PROVENANCE_CONTRACT_NAME,
"contract_version": PROVENANCE_CONTRACT_VERSION,
"record_fields": list(RECORD_FIELDS),
"provenance_states": list(PROVENANCE_STATES),
"trust_levels": list(TRUST_LEVELS),
"source_kinds": list(SOURCE_KINDS),
"firewall_statuses": list(FIREWALL_STATUSES),
"dispositions": list(DISPOSITIONS),
"hash_format": HASH_FORMAT,
"attestation_issuer": COMPANION_ATTESTATION_ISSUER,
"p3_contract_name": P3_CONTRACT_NAME,
"p3_contract_version": P3_CONTRACT_VERSION,
}
def contract_fingerprint() -> str:
"""Return a stable descriptor fingerprint without importing Companion."""
return _digest(contract_descriptor())
def provenance_record_digest(record: Mapping[str, Any] | None) -> str:
"""Return a digest only for an already-valid, metadata-only P5 record."""
checked = validate_provenance_record(record)
if not checked["ok"]:
return ""
return _digest(checked["record"])
def _safe_memory_id(value: Any) -> str:
if isinstance(value, str) and _MEMORY_ID_RE.fullmatch(value):
return value
return ""
def _safe_hash(value: Any) -> str:
"""Accept only a canonical stored hash (no prefix and lowercase)."""
if isinstance(value, str) and _SHA256_RE.fullmatch(value) and value == value.lower():
return value
return ""
def _normalize_input_hash(value: Any) -> str:
"""Normalize a boundary hash; persisted records remain canonical only."""
if not isinstance(value, str):
return ""
raw = value.removeprefix("sha256:")
if _SHA256_RE.fullmatch(raw):
return raw.lower()
return ""
def _safe_revision(value: Any) -> int | None:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def _known_token(value: Any, known: frozenset[str] | tuple[str, ...]) -> bool:
return isinstance(value, str) and value in known
def _has_forbidden_raw_field(value: Mapping[str, Any]) -> bool:
return any(key in value for key in _FORBIDDEN_RAW_FIELDS)
def _field_set_is_exact(value: Mapping[str, Any], fields: tuple[str, ...]) -> bool:
return set(value) == set(fields)
def _error_result(*codes: str) -> dict[str, Any]:
# Error codes are static, so they neither echo a hostile value nor retain
# prose/raw IDs supplied by a caller.
return {"ok": False, "error_codes": sorted(set(codes))}
def _blank_record(memory_id: Any = "", *, revision: int = 0, state: str = "invalid") -> dict[str, Any]:
return {
"contract_name": PROVENANCE_CONTRACT_NAME,
"contract_version": PROVENANCE_CONTRACT_VERSION,
"contract_fingerprint": contract_fingerprint(),
"memory_id": _safe_memory_id(memory_id),
"source_kind": "unknown",
"source_trust": "T4",
"firewall_status": "unknown",
"source_event_ref_hash": "",
"authority_attestation_ref_hash": "",
"provenance_state": state,
"migration_operation_ref": "",
"recovery_operation_ref": "",
"record_revision": revision if _safe_revision(revision) is not None else 0,
}
def validate_provenance_record(record: Mapping[str, Any] | None) -> dict[str, Any]:
"""Validate a persisted P5 record without coercion or content parsing.
The result contains only static error codes. In particular it never
returns an invalid input field, a source event ID, or message text.
"""
if not isinstance(record, Mapping):
return _error_result("record_not_mapping")
errors: list[str] = []
if _has_forbidden_raw_field(record):
errors.append("forbidden_raw_field")
if not _field_set_is_exact(record, RECORD_FIELDS):
errors.append("record_fields_mismatch")
if record.get("contract_name") != PROVENANCE_CONTRACT_NAME:
errors.append("contract_name_mismatch")
if record.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
errors.append("contract_version_mismatch")
if record.get("contract_fingerprint") != contract_fingerprint():
errors.append("contract_fingerprint_mismatch")
if not _safe_memory_id(record.get("memory_id")):
errors.append("memory_id_invalid")
if not _known_token(record.get("source_kind"), SOURCE_KINDS):
errors.append("source_kind_invalid")
if not _known_token(record.get("source_trust"), TRUST_LEVELS):
errors.append("source_trust_invalid")
if not _known_token(record.get("firewall_status"), FIREWALL_STATUSES):
errors.append("firewall_status_invalid")
if not _known_token(record.get("provenance_state"), PROVENANCE_STATES):
errors.append("provenance_state_invalid")
if _safe_revision(record.get("record_revision")) is None:
errors.append("record_revision_invalid")
source_hash_value = record.get("source_event_ref_hash")
authority_hash_value = record.get("authority_attestation_ref_hash")
source_hash = _safe_hash(source_hash_value)
authority_hash = _safe_hash(authority_hash_value)
if source_hash_value != "" and not source_hash:
errors.append("source_event_ref_hash_invalid")
if authority_hash_value != "" and not authority_hash:
errors.append("authority_attestation_ref_hash_invalid")
migration_ref = record.get("migration_operation_ref")
recovery_ref = record.get("recovery_operation_ref")
if migration_ref != "" and not _safe_hash(migration_ref):
errors.append("migration_operation_ref_invalid")
if recovery_ref != "" and not _safe_hash(recovery_ref):
errors.append("recovery_operation_ref_invalid")
state = record.get("provenance_state")
if state == "legacy_unresolved":
if (record.get("source_kind"), record.get("source_trust"), record.get("firewall_status")) != (
"unknown",
"T4",
"unknown",
):
errors.append("legacy_defaults_required")
if source_hash_value != "" or authority_hash_value != "":
errors.append("legacy_hashes_forbidden")
if not isinstance(migration_ref, str) or not isinstance(recovery_ref, str):
errors.append("operation_ref_type_invalid")
elif state == "observed" or state == "owner_recovered":
if record.get("source_kind") == "unknown":
errors.append("observed_source_kind_required")
if not source_hash or not authority_hash:
errors.append("attestation_hashes_required")
if source_hash and authority_hash and source_hash == authority_hash:
errors.append("attestation_hashes_conflict")
if state == "owner_recovered" and not _safe_hash(recovery_ref):
errors.append("recovery_operation_ref_required")
elif state == "invalid":
# Invalid is an explicit quarantine representation, never an accepted
# migration target. Its safe shape is still useful for caller logs.
if source_hash or authority_hash:
errors.append("invalid_hashes_forbidden")
if errors:
return _error_result(*errors)
return {"ok": True, "record": deepcopy(dict(record))}
def legacy_unresolved(memory_id: str, *, record_revision: int = 0) -> dict[str, Any]:
"""Represent a historical item without pretending it has known trust."""
record = _blank_record(memory_id, revision=record_revision, state="legacy_unresolved")
if not record["memory_id"]:
return _blank_record(state="invalid")
return record
def _snapshot_hash(snapshot: Mapping[str, Any], *keys: str) -> tuple[str, str]:
present = [(key, snapshot.get(key)) for key in keys if key in snapshot]
if not present:
return "missing", ""
values = [_normalize_input_hash(value) for _, value in present]
if not all(values):
return "invalid", ""
if len(set(values)) != 1:
return "conflict", ""
return "present", values[0]
def _snapshot_label(
snapshot: Mapping[str, Any], known: frozenset[str] | tuple[str, ...], *keys: str
) -> tuple[str, str]:
present = [(key, snapshot.get(key)) for key in keys if key in snapshot]
if not present:
return "missing", ""
values = [value for _, value in present]
if not all(_known_token(value, known) for value in values):
return "invalid", ""
if len(set(values)) != 1:
return "conflict", ""
return "present", values[0]
def _snapshot_diagnostics_valid(snapshot: Mapping[str, Any]) -> bool:
"""Allow only bounded, non-prose diagnostics that are not persisted."""
# A disposition is an attestation-side decision label, distinct from the
# upstream P2/P3 firewall status that is persisted in the record.
if "disposition" not in snapshot:
return False
disposition = snapshot["disposition"]
if not _known_token(disposition, DISPOSITIONS):
return False
if "reason_codes" in snapshot:
codes = snapshot["reason_codes"]
if not isinstance(codes, (list, tuple)) or len(codes) > 16:
return False
if not all(isinstance(code, str) and _TOKEN_RE.fullmatch(code) for code in codes):
return False
for key in ("request_hash", "session_hash"):
if key in snapshot and not _normalize_input_hash(snapshot[key]):
return False
derived_values = [
snapshot[key] for key in ("derived_from_ref_hash", "derived_hash") if key in snapshot
]
if derived_values:
if any(value != "" and not _normalize_input_hash(value) for value in derived_values):
return False
normalized = [_normalize_input_hash(value) for value in derived_values if value != ""]
if len(set(normalized)) > 1:
return False
if snapshot.get("provenance_state") != "observed":
return False
epoch = snapshot.get("issuer_epoch")
if not _normalize_input_hash(epoch):
return False
sink = snapshot.get("sink")
if not _known_token(sink, _ATTESTATION_SINKS):
return False
return True
def _snapshot_mapping(snapshot: Any) -> Mapping[str, Any] | None:
"""Accept the immutable Companion dataclass without importing Companion."""
if isinstance(snapshot, Mapping):
return snapshot
if is_dataclass(snapshot):
value = asdict(snapshot)
return value if isinstance(value, Mapping) else None
return None
def observed_from_companion_snapshot(
memory_id: str, snapshot: Any, *, record_revision: int = 0
) -> dict[str, Any]:
"""Convert only a valid Companion-attested snapshot into ``observed``.
This function accepts hashes and bounded labels only. A raw source event
ID, text body, or arbitrary P3 fingerprint rejects conversion and yields a
safe ``invalid`` record rather than an observed one.
"""
safe_id = _safe_memory_id(memory_id)
revision = _safe_revision(record_revision)
snapshot_mapping = _snapshot_mapping(snapshot)
if not safe_id or revision is None or snapshot_mapping is None:
return _blank_record(safe_id, revision=revision or 0)
if (
_has_forbidden_raw_field(snapshot_mapping)
or not set(snapshot_mapping).issubset(_SNAPSHOT_FIELDS)
or not _snapshot_diagnostics_valid(snapshot_mapping)
):
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("schema_version") != ATTESTATION_SCHEMA_VERSION:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("issuer") != COMPANION_ATTESTATION_ISSUER:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("contract_name") != PROVENANCE_CONTRACT_NAME:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("contract_fingerprint") != contract_fingerprint():
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("p3_contract_name") != P3_CONTRACT_NAME:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("p3_contract_version") != P3_CONTRACT_VERSION:
return _blank_record(safe_id, revision=revision)
if snapshot_mapping.get("p3_contract_fingerprint") != P3_CONTRACT_FINGERPRINT:
return _blank_record(safe_id, revision=revision)
source_status, source_hash = _snapshot_hash(
snapshot_mapping, "source_event_ref_hash", "source_hash"
)
authority_status, authority_hash = _snapshot_hash(
snapshot_mapping, "authority_attestation_ref_hash", "attestation_ref_hash"
)
trust_status, source_trust = _snapshot_label(snapshot_mapping, TRUST_LEVELS, "source_trust", "trust")
source_kind = snapshot_mapping.get("source_kind")
firewall_status = snapshot_mapping.get("firewall_status")
if (
source_status != "present"
or authority_status != "present"
or trust_status != "present"
or source_hash == authority_hash
or not _known_token(source_kind, _SOURCE_KIND_SET - {"unknown"})
or not _known_token(source_trust, TRUST_LEVELS)
or not _known_token(firewall_status, FIREWALL_STATUSES)
):
return _blank_record(safe_id, revision=revision)
record = _blank_record(safe_id, revision=revision, state="observed")
record.update(
{
"source_kind": source_kind,
"source_trust": source_trust,
"firewall_status": firewall_status,
"source_event_ref_hash": source_hash,
"authority_attestation_ref_hash": authority_hash,
}
)
return record if validate_provenance_record(record)["ok"] else _blank_record(safe_id, revision=revision)
def _record_or_none(value: Any) -> dict[str, Any] | None:
if value is None:
return None
result = validate_provenance_record(value if isinstance(value, Mapping) else None)
return deepcopy(result["record"]) if result["ok"] else None
def _operation(
*,
operation_kind: str,
operation_ref_hash: str,
before_record: Mapping[str, Any] | None,
after_record: Mapping[str, Any],
) -> dict[str, Any] | None:
before = _record_or_none(before_record)
after = _record_or_none(after_record)
reference = _safe_hash(operation_ref_hash)
if not _known_token(operation_kind, ("migration", "recovery")) or not reference or after is None:
return None
if before is not None and before["memory_id"] != after["memory_id"]:
return None
expected_revision = before["record_revision"] if before is not None else after["record_revision"] - 1
if expected_revision < 0 or after["record_revision"] != expected_revision + 1:
return None
operation = {
"contract_name": PROVENANCE_CONTRACT_NAME,
"contract_version": PROVENANCE_CONTRACT_VERSION,
"contract_fingerprint": contract_fingerprint(),
"operation_kind": operation_kind,
"operation_ref_hash": reference,
"memory_id": after["memory_id"],
"expected_revision": expected_revision,
"before_record_digest": _digest(before) if before is not None else "",
"after_record_digest": _digest(after),
"before_record": before,
"after_record": after,
}
return operation if validate_operation(operation)["ok"] else None
def validate_operation(operation: Mapping[str, Any] | None) -> dict[str, Any]:
"""Validate an I/O-free migration/recovery operation and its CAS guards."""
if not isinstance(operation, Mapping):
return _error_result("operation_not_mapping")
errors: list[str] = []
if _has_forbidden_raw_field(operation):
errors.append("forbidden_raw_field")
if not _field_set_is_exact(operation, OPERATION_FIELDS):
errors.append("operation_fields_mismatch")
if operation.get("contract_name") != PROVENANCE_CONTRACT_NAME:
errors.append("contract_name_mismatch")
if operation.get("contract_version") != PROVENANCE_CONTRACT_VERSION:
errors.append("contract_version_mismatch")
if operation.get("contract_fingerprint") != contract_fingerprint():
errors.append("contract_fingerprint_mismatch")
kind = operation.get("operation_kind")
if not _known_token(kind, ("migration", "recovery")):
errors.append("operation_kind_invalid")
if not _safe_hash(operation.get("operation_ref_hash")):
errors.append("operation_ref_invalid")
if not _safe_memory_id(operation.get("memory_id")):
errors.append("memory_id_invalid")
expected_revision = _safe_revision(operation.get("expected_revision"))
if expected_revision is None:
errors.append("expected_revision_invalid")
before = operation.get("before_record")
after = operation.get("after_record")
before_record = _record_or_none(before)
after_record = _record_or_none(after)
if before is not None and before_record is None:
errors.append("before_record_invalid")
if after_record is None:
errors.append("after_record_invalid")
if before_record is not None and operation.get("before_record_digest") != _digest(before_record):
errors.append("before_digest_mismatch")
if before is None and operation.get("before_record_digest") != "":
errors.append("before_digest_required_empty")
if after_record is not None and operation.get("after_record_digest") != _digest(after_record):
errors.append("after_digest_mismatch")
if after_record is not None:
if operation.get("memory_id") != after_record["memory_id"]:
errors.append("memory_id_mismatch")
if expected_revision is not None and after_record["record_revision"] != expected_revision + 1:
errors.append("revision_progression_invalid")
required_ref = "migration_operation_ref" if kind == "migration" else "recovery_operation_ref"
if after_record.get(required_ref) != operation.get("operation_ref_hash"):
errors.append("operation_ref_not_bound")
if before_record is not None and after_record is not None:
if before_record["memory_id"] != after_record["memory_id"]:
errors.append("before_after_memory_mismatch")
if expected_revision is not None and before_record["record_revision"] != expected_revision:
errors.append("before_revision_mismatch")
if errors:
return _error_result(*errors)
return {"ok": True, "operation": deepcopy(dict(operation))}
def plan_legacy_migration(
records: Iterable[Mapping[str, Any]], *, operation_ref_hash: str
) -> dict[str, Any]:
"""Build a read-only P5 legacy-backfill preview; never write or mutate.
``records`` are intentionally tiny metadata snapshots containing only a
``memory_id`` and ``record_revision``. A valid existing P5 record may be
supplied instead and is skipped; Store integration will decide how to
obtain those snapshots in a future phase.
"""
reference = _safe_hash(operation_ref_hash)
result: dict[str, Any] = {
"mode": "preview",
"readonly": True,
"write_count": 0,
"operation_ref_hash": reference,
"operations": [],
"skipped": [],
"error_codes": [],
}
if not reference:
result["error_codes"].append("operation_ref_invalid")
return result
if isinstance(records, (str, bytes)) or not isinstance(records, Iterable):
result["error_codes"].append("records_not_iterable")
return result
for candidate in records:
if not isinstance(candidate, Mapping) or _has_forbidden_raw_field(candidate):
result["error_codes"].append("candidate_invalid")
continue
existing = _record_or_none(candidate)
if existing is not None:
result["skipped"].append({"memory_id": existing["memory_id"], "reason_code": "already_provenanced"})
continue
allowed_base_fields = {"memory_id", "record_revision"}
if set(candidate) != allowed_base_fields:
result["error_codes"].append("candidate_fields_mismatch")
continue
memory_id = _safe_memory_id(candidate.get("memory_id"))
revision = _safe_revision(candidate.get("record_revision"))
if not memory_id or revision is None:
result["error_codes"].append("candidate_invalid")
continue
after = legacy_unresolved(memory_id, record_revision=revision + 1)
after["migration_operation_ref"] = reference
operation = _operation(
operation_kind="migration",
operation_ref_hash=reference,
before_record=None,
after_record=after,
)
if operation is None:
result["error_codes"].append("operation_invalid")
else:
result["operations"].append(operation)
result["error_codes"] = sorted(set(result["error_codes"]))
return result
def _minimal_current(current: Mapping[str, Any] | None) -> tuple[str, int | None, dict[str, Any] | None, str]:
"""Extract a safe CAS view; never accept a raw current-record payload."""
if not isinstance(current, Mapping) or _has_forbidden_raw_field(current):
return "", None, None, "current_invalid"
p5 = _record_or_none(current)
if p5 is not None:
return p5["memory_id"], p5["record_revision"], p5, ""
if set(current) != {"memory_id", "record_revision"}:
return "", None, None, "current_fields_mismatch"
return _safe_memory_id(current.get("memory_id")), _safe_revision(current.get("record_revision")), None, ""
def apply_planned_operation(
current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None
) -> dict[str, Any]:
"""Project a guarded apply result without touching storage.
Applying the same operation twice is idempotent. A different record or
revision returns a conflict instead of overwriting a concurrent update.
"""
checked = validate_operation(operation)
if not checked["ok"]:
return {"ok": False, "mode": "apply", "status": "invalid_operation", "error_codes": checked["error_codes"]}
op = checked["operation"]
memory_id, revision, current_record, error = _minimal_current(current)
if error or not memory_id or revision is None:
return {"ok": False, "mode": "apply", "status": "invalid_current", "error_codes": [error or "current_invalid"]}
after = op["after_record"]
if current_record is not None and _digest(current_record) == op["after_record_digest"]:
return {"ok": True, "mode": "apply", "status": "idempotent", "record": deepcopy(after), "changed": False}
if memory_id != op["memory_id"] or revision != op["expected_revision"]:
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["revision_or_memory_mismatch"]}
expected_before = op["before_record"]
if expected_before is None:
if current_record is not None:
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["unexpected_existing_provenance"]}
elif current_record is None or _digest(current_record) != op["before_record_digest"]:
return {"ok": False, "mode": "apply", "status": "revision_conflict", "error_codes": ["before_record_mismatch"]}
return {"ok": True, "mode": "apply", "status": "applied", "record": deepcopy(after), "changed": True}
def rollback_planned_operation(
current: Mapping[str, Any] | None, operation: Mapping[str, Any] | None
) -> dict[str, Any]:
"""Project a CAS-protected rollback without overwriting later changes."""
checked = validate_operation(operation)
if not checked["ok"]:
return {"ok": False, "mode": "rollback", "status": "invalid_operation", "error_codes": checked["error_codes"]}
op = checked["operation"]
memory_id, revision, current_record, error = _minimal_current(current)
if error or current_record is None or revision is None:
return {"ok": False, "mode": "rollback", "status": "invalid_current", "error_codes": [error or "current_invalid"]}
if (
memory_id != op["memory_id"]
or revision != op["after_record"]["record_revision"]
or _digest(current_record) != op["after_record_digest"]
):
return {"ok": False, "mode": "rollback", "status": "revision_conflict", "error_codes": ["concurrent_update_protected"]}
before = op["before_record"]
return {
"ok": True,
"mode": "rollback",
"status": "rolled_back",
"record": deepcopy(before) if before is not None else None,
"removed": before is None,
"changed": True,
}
def plan_owner_recovery(
current: Mapping[str, Any],
snapshot: Mapping[str, Any],
*,
recovery_operation_ref_hash: str,
) -> dict[str, Any]:
"""Plan, but do not apply, a Companion-attested owner recovery.
Recovery is intentionally limited to a valid ``legacy_unresolved`` record.
A later owner-authority workflow must supply the Companion snapshot; this
pure contract has no owner command, storage, or authorization side effect.
"""
existing = _record_or_none(current)
reference = _safe_hash(recovery_operation_ref_hash)
if existing is None or existing["provenance_state"] != "legacy_unresolved" or not reference:
return {"ok": False, "mode": "preview", "error_codes": ["recovery_precondition_failed"]}
after = observed_from_companion_snapshot(
existing["memory_id"], snapshot, record_revision=existing["record_revision"] + 1
)
if after["provenance_state"] != "observed":
return {"ok": False, "mode": "preview", "error_codes": ["companion_snapshot_invalid"]}
after["provenance_state"] = "owner_recovered"
after["migration_operation_ref"] = existing["migration_operation_ref"]
after["recovery_operation_ref"] = reference
operation = _operation(
operation_kind="recovery",
operation_ref_hash=reference,
before_record=existing,
after_record=after,
)
if operation is None:
return {"ok": False, "mode": "preview", "error_codes": ["operation_invalid"]}
return {"ok": True, "mode": "preview", "readonly": True, "write_count": 0, "operation": operation}
__all__ = [
"COMPANION_ATTESTATION_ISSUER",
"DISPOSITIONS",
"FIREWALL_STATUSES",
"HASH_FORMAT",
"P3_CONTRACT_FINGERPRINT",
"P3_CONTRACT_NAME",
"P3_CONTRACT_VERSION",
"PROVENANCE_CONTRACT_NAME",
"PROVENANCE_CONTRACT_VERSION",
"PROVENANCE_STATES",
"SOURCE_KINDS",
"TRUST_LEVELS",
"apply_planned_operation",
"contract_descriptor",
"contract_fingerprint",
"legacy_unresolved",
"observed_from_companion_snapshot",
"plan_legacy_migration",
"plan_owner_recovery",
"provenance_record_digest",
"rollback_planned_operation",
"validate_operation",
"validate_provenance_record",
]

查看文件

@ -0,0 +1,468 @@
"""P5-B0 additive SQLite persistence and recoverable legacy backfill.
This is deliberately a storage/migration boundary. It does not consume a
Companion attestation, change memory capture, alter retrieval, or evaluate a
live sink. The only records it creates are explicit ``legacy_unresolved``
records, so existing memory columns can never be used to infer source trust.
"""
from __future__ import annotations
import hashlib
import json
import re
import sqlite3
from collections.abc import Mapping
from copy import deepcopy
from typing import Any, TYPE_CHECKING
from .models import utc_now
from .p5_provenance import (
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
contract_fingerprint,
legacy_unresolved,
provenance_record_digest,
validate_provenance_record,
)
if TYPE_CHECKING:
from .store import MemoryStore
P5_PERSISTENCE_SCHEMA_VERSION = 1
_OPERATION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$")
_RECORD_COLUMNS = (
"memory_id",
"contract_name",
"contract_version",
"contract_fingerprint",
"source_kind",
"source_trust",
"firewall_status",
"source_event_ref_hash",
"authority_attestation_ref_hash",
"provenance_state",
"migration_operation_ref",
"recovery_operation_ref",
"record_revision",
"record_digest",
)
def ensure_p5_provenance_persistence_schema(connection: sqlite3.Connection) -> None:
"""Create only additive P5-B0 tables and indexes.
This function neither scans memories nor creates provenance records; that
work is possible only through the explicit migration apply operation.
"""
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS p5_provenance_records (
memory_id TEXT PRIMARY KEY,
contract_name TEXT NOT NULL,
contract_version TEXT NOT NULL,
contract_fingerprint TEXT NOT NULL,
source_kind TEXT NOT NULL,
source_trust TEXT NOT NULL,
firewall_status TEXT NOT NULL,
source_event_ref_hash TEXT NOT NULL DEFAULT '',
authority_attestation_ref_hash TEXT NOT NULL DEFAULT '',
provenance_state TEXT NOT NULL,
migration_operation_ref TEXT NOT NULL DEFAULT '',
recovery_operation_ref TEXT NOT NULL DEFAULT '',
record_revision INTEGER NOT NULL,
record_digest TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
FOREIGN KEY(memory_id) REFERENCES memories(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_p5_provenance_state
ON p5_provenance_records(provenance_state, source_trust, firewall_status);
CREATE TABLE IF NOT EXISTS p5_provenance_migration_runs (
operation_id TEXT PRIMARY KEY,
operation_ref_hash TEXT NOT NULL,
contract_name TEXT NOT NULL,
contract_version TEXT NOT NULL,
contract_fingerprint TEXT NOT NULL,
state TEXT NOT NULL,
requested_limit INTEGER NOT NULL DEFAULT 0,
candidate_count INTEGER NOT NULL DEFAULT 0,
applied_count INTEGER NOT NULL DEFAULT 0,
rollback_conflict_count INTEGER NOT NULL DEFAULT 0,
backup_path TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
completed_at TEXT NOT NULL DEFAULT '',
rolled_back_at TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS p5_provenance_migration_backups (
operation_id TEXT NOT NULL,
memory_id TEXT NOT NULL,
before_exists INTEGER NOT NULL,
before_record_json TEXT NOT NULL DEFAULT '',
before_digest TEXT NOT NULL DEFAULT '',
before_revision INTEGER NOT NULL DEFAULT 0,
after_record_json TEXT NOT NULL,
after_digest TEXT NOT NULL,
after_revision INTEGER NOT NULL,
apply_state TEXT NOT NULL DEFAULT 'prepared',
rollback_state TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT '',
rolled_back_at TEXT NOT NULL DEFAULT '',
PRIMARY KEY(operation_id, memory_id),
FOREIGN KEY(operation_id) REFERENCES p5_provenance_migration_runs(operation_id)
);
CREATE INDEX IF NOT EXISTS idx_p5_provenance_backup_operation
ON p5_provenance_migration_backups(operation_id, rollback_state, memory_id);
"""
)
run_columns = {
str(row[1])
for row in connection.execute("PRAGMA table_info(p5_provenance_migration_runs)").fetchall()
}
if "backup_path" not in run_columns:
connection.execute(
"ALTER TABLE p5_provenance_migration_runs ADD COLUMN backup_path TEXT NOT NULL DEFAULT ''"
)
def _canonical_json(value: Mapping[str, Any]) -> str:
return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
def _operation_ref(operation_id: str) -> str:
return hashlib.sha256(
f"{PROVENANCE_CONTRACT_NAME}:migration:{operation_id}".encode("utf-8")
).hexdigest()
def _valid_operation_id(value: Any) -> str:
return value if isinstance(value, str) and _OPERATION_ID_RE.fullmatch(value) else ""
def _limit(value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
return 0
return min(value, 5000)
def _record_from_row(row: sqlite3.Row) -> dict[str, Any]:
return {
"contract_name": row["contract_name"],
"contract_version": row["contract_version"],
"contract_fingerprint": row["contract_fingerprint"],
"memory_id": row["memory_id"],
"source_kind": row["source_kind"],
"source_trust": row["source_trust"],
"firewall_status": row["firewall_status"],
"source_event_ref_hash": row["source_event_ref_hash"],
"authority_attestation_ref_hash": row["authority_attestation_ref_hash"],
"provenance_state": row["provenance_state"],
"migration_operation_ref": row["migration_operation_ref"],
"recovery_operation_ref": row["recovery_operation_ref"],
"record_revision": int(row["record_revision"]),
}
def _insert_record(connection: sqlite3.Connection, record: Mapping[str, Any], *, digest: str, now: str) -> None:
values = {
"memory_id": record["memory_id"],
"contract_name": record["contract_name"],
"contract_version": record["contract_version"],
"contract_fingerprint": record["contract_fingerprint"],
"source_kind": record["source_kind"],
"source_trust": record["source_trust"],
"firewall_status": record["firewall_status"],
"source_event_ref_hash": record["source_event_ref_hash"],
"authority_attestation_ref_hash": record["authority_attestation_ref_hash"],
"provenance_state": record["provenance_state"],
"migration_operation_ref": record["migration_operation_ref"],
"recovery_operation_ref": record["recovery_operation_ref"],
"record_revision": record["record_revision"],
"record_digest": digest,
"created_at": now,
"updated_at": now,
}
connection.execute(
"""
INSERT INTO p5_provenance_records(
memory_id, contract_name, contract_version, contract_fingerprint,
source_kind, source_trust, firewall_status, source_event_ref_hash,
authority_attestation_ref_hash, provenance_state,
migration_operation_ref, recovery_operation_ref, record_revision,
record_digest, created_at, updated_at
) VALUES(
:memory_id, :contract_name, :contract_version, :contract_fingerprint,
:source_kind, :source_trust, :firewall_status, :source_event_ref_hash,
:authority_attestation_ref_hash, :provenance_state,
:migration_operation_ref, :recovery_operation_ref, :record_revision,
:record_digest, :created_at, :updated_at
)
""",
values,
)
class P5ProvenanceMigration:
"""Recoverable B0 legacy provenance backfill over an existing MemoryStore."""
def __init__(self, store: MemoryStore):
self.store = store
async def preview(self, *, limit: int = 5000) -> dict[str, Any]:
return await self.store._run_recoverable_database_operation(self._preview_sync, limit)
def _preview_sync(self, limit: int) -> dict[str, Any]:
bounded_limit = _limit(limit)
if not bounded_limit:
return {"mode": "preview", "ok": False, "readonly": True, "error_code": "limit_invalid", "count": 0}
with self.store._lock:
rows = self.store._conn.execute(
"""
SELECT m.id
FROM memories AS m
LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id
WHERE p.memory_id IS NULL
ORDER BY m.id
LIMIT ?
""",
(bounded_limit,),
).fetchall()
return {
"mode": "preview",
"ok": True,
"readonly": True,
"contract_name": PROVENANCE_CONTRACT_NAME,
"contract_version": PROVENANCE_CONTRACT_VERSION,
"contract_fingerprint": contract_fingerprint(),
"count": len(rows),
"write_count": 0,
}
async def apply(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
return await self.store._run_recoverable_database_operation(self._apply_sync, operation_id, limit)
def _apply_sync(self, operation_id: str, limit: int) -> dict[str, Any]:
clean_operation_id = _valid_operation_id(operation_id)
bounded_limit = _limit(limit)
if not clean_operation_id:
return {"mode": "apply", "ok": False, "operation_id": "", "error_code": "operation_id_required", "changed": 0}
if not bounded_limit:
return {"mode": "apply", "ok": False, "operation_id": clean_operation_id, "error_code": "limit_invalid", "changed": 0}
reference = _operation_ref(clean_operation_id)
with self.store._lock:
existing = self.store._conn.execute(
"SELECT state, applied_count, backup_path FROM p5_provenance_migration_runs WHERE operation_id=?",
(clean_operation_id,),
).fetchone()
if existing is not None:
if existing["state"] == "completed":
return {
"mode": "apply", "ok": True, "operation_id": clean_operation_id,
"changed": 0, "deduplicated": True, "backup_count": int(existing["applied_count"] or 0),
"backup_path": str(existing["backup_path"] or ""),
}
return {
"mode": "apply", "ok": False, "operation_id": clean_operation_id,
"error_code": "operation_not_reapplicable", "changed": 0,
}
try:
# This commits only prior state and creates the physical SQLite
# image before the first B0 mutation transaction is opened.
backup_path = str(self.store.backup(f".before_p5_provenance_b0.{clean_operation_id}"))
except Exception:
return {
"mode": "apply", "ok": False, "operation_id": clean_operation_id,
"error_code": "physical_backup_failed", "changed": 0, "backup_path": "",
}
with self.store._transaction_sync():
candidates = self.store._conn.execute(
"""
SELECT m.id
FROM memories AS m
LEFT JOIN p5_provenance_records AS p ON p.memory_id = m.id
WHERE p.memory_id IS NULL
ORDER BY m.id
LIMIT ?
""",
(bounded_limit,),
).fetchall()
now = utc_now()
self.store._conn.execute(
"""
INSERT INTO p5_provenance_migration_runs(
operation_id, operation_ref_hash, contract_name, contract_version,
contract_fingerprint, state, requested_limit, candidate_count,
applied_count, rollback_conflict_count, backup_path, created_at, updated_at
) VALUES(?,?,?,?,?,'running',?,?,0,0,?,?,?)
""",
(
clean_operation_id, reference, PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION, contract_fingerprint(), bounded_limit,
len(candidates), backup_path, now, now,
),
)
changed = 0
for row in candidates:
record = legacy_unresolved(row["id"], record_revision=1)
record["migration_operation_ref"] = reference
checked = validate_provenance_record(record)
digest = provenance_record_digest(record)
if not checked["ok"] or not digest:
raise RuntimeError("p5_provenance_legacy_record_invalid")
record_json = _canonical_json(record)
# The backup/manifest row is deliberately inserted before
# the matching provenance record mutation in this transaction.
self.store._conn.execute(
"""
INSERT INTO p5_provenance_migration_backups(
operation_id, memory_id, before_exists, before_record_json,
before_digest, before_revision, after_record_json, after_digest,
after_revision, apply_state, rollback_state, created_at
) VALUES(?,?,0,'','',0,?,?,?,'prepared','pending',?)
""",
(clean_operation_id, record["memory_id"], record_json, digest, record["record_revision"], now),
)
_insert_record(self.store._conn, record, digest=digest, now=now)
self.store._conn.execute(
"""
UPDATE p5_provenance_migration_backups
SET apply_state='applied', applied_at=?
WHERE operation_id=? AND memory_id=?
""",
(now, clean_operation_id, record["memory_id"]),
)
changed += 1
self.store._conn.execute(
"""
UPDATE p5_provenance_migration_runs
SET state='completed', applied_count=?, updated_at=?, completed_at=?
WHERE operation_id=?
""",
(changed, now, now, clean_operation_id),
)
return {
"mode": "apply", "ok": True, "operation_id": clean_operation_id,
"changed": changed, "deduplicated": False, "backup_count": changed,
"backup_path": backup_path,
}
async def rollback(self, operation_id: str) -> dict[str, Any]:
return await self.store._run_recoverable_database_operation(self._rollback_sync, operation_id)
def _rollback_sync(self, operation_id: str) -> dict[str, Any]:
clean_operation_id = _valid_operation_id(operation_id)
if not clean_operation_id:
return {"mode": "rollback", "ok": False, "operation_id": "", "error_code": "operation_id_required", "restored": 0, "conflicts": 0}
with self.store._lock:
with self.store._transaction_sync():
run = self.store._conn.execute(
"SELECT state, operation_ref_hash FROM p5_provenance_migration_runs WHERE operation_id=?",
(clean_operation_id,),
).fetchone()
if run is None:
return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_found", "restored": 0, "conflicts": 0}
if run["state"] == "rolled_back":
return {"mode": "rollback", "ok": True, "operation_id": clean_operation_id, "restored": 0, "conflicts": 0, "deduplicated": True}
if run["state"] not in {"completed", "rollback_conflicted"}:
return {"mode": "rollback", "ok": False, "operation_id": clean_operation_id, "error_code": "operation_not_rollbackable", "restored": 0, "conflicts": 0}
backups = self.store._conn.execute(
"""
SELECT * FROM p5_provenance_migration_backups
WHERE operation_id=? AND rollback_state='pending'
ORDER BY memory_id
""",
(clean_operation_id,),
).fetchall()
restored = 0
conflicts = 0
now = utc_now()
for backup in backups:
current = self.store._conn.execute(
"SELECT * FROM p5_provenance_records WHERE memory_id=?",
(backup["memory_id"],),
).fetchone()
expected = (
current is not None
and current["record_digest"] == backup["after_digest"]
and int(current["record_revision"]) == int(backup["after_revision"])
and current["migration_operation_ref"] == run["operation_ref_hash"]
)
if not expected:
self.store._conn.execute(
"""
UPDATE p5_provenance_migration_backups
SET rollback_state='conflict'
WHERE operation_id=? AND memory_id=?
""",
(clean_operation_id, backup["memory_id"]),
)
conflicts += 1
continue
if int(backup["before_exists"]):
try:
before = json.loads(backup["before_record_json"])
except (TypeError, ValueError):
before = None
before_digest = provenance_record_digest(before if isinstance(before, Mapping) else None)
if not before_digest or before_digest != backup["before_digest"]:
self.store._conn.execute(
"UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?",
(clean_operation_id, backup["memory_id"]),
)
conflicts += 1
continue
self.store._conn.execute("DELETE FROM p5_provenance_records WHERE memory_id=?", (backup["memory_id"],))
_insert_record(self.store._conn, before, digest=before_digest, now=now)
else:
deleted = self.store._conn.execute(
"DELETE FROM p5_provenance_records WHERE memory_id=? AND record_digest=? AND record_revision=? AND migration_operation_ref=?",
(backup["memory_id"], backup["after_digest"], backup["after_revision"], run["operation_ref_hash"]),
)
if deleted.rowcount != 1:
# The earlier SELECT predicate should make this
# unreachable under the store lock, but retain a
# conservative conflict if an adapter changes it.
self.store._conn.execute(
"UPDATE p5_provenance_migration_backups SET rollback_state='conflict' WHERE operation_id=? AND memory_id=?",
(clean_operation_id, backup["memory_id"]),
)
conflicts += 1
continue
self.store._conn.execute(
"""
UPDATE p5_provenance_migration_backups
SET rollback_state='restored', rolled_back_at=?
WHERE operation_id=? AND memory_id=?
""",
(now, clean_operation_id, backup["memory_id"]),
)
restored += 1
state = "rolled_back" if conflicts == 0 else "rollback_conflicted"
self.store._conn.execute(
"""
UPDATE p5_provenance_migration_runs
SET state=?, rollback_conflict_count=rollback_conflict_count + ?,
updated_at=?, rolled_back_at=?
WHERE operation_id=?
""",
(state, conflicts, now, now if conflicts == 0 else "", clean_operation_id),
)
return {
"mode": "rollback", "ok": conflicts == 0, "operation_id": clean_operation_id,
"restored": restored, "conflicts": conflicts, "deduplicated": False,
}
__all__ = [
"P5ProvenanceMigration",
"P5_PERSISTENCE_SCHEMA_VERSION",
"ensure_p5_provenance_persistence_schema",
]

查看文件

@ -41,6 +41,7 @@ class RetrievalEngine:
keyword_fallback_min_fts_candidates: int = 80,
knowledge_graph_enabled: bool = True,
knowledge_graph_expansion_limit: int = 12,
p5_recall_provenance_required: bool = False,
usage_recorder: Any | None = None,
):
self.store = store
@ -69,6 +70,7 @@ class RetrievalEngine:
)
self.knowledge_graph_enabled = bool(knowledge_graph_enabled)
self.knowledge_graph_expansion_limit = max(0, int(knowledge_graph_expansion_limit or 0))
self.p5_recall_provenance_required = bool(p5_recall_provenance_required)
self.usage_recorder = usage_recorder if callable(usage_recorder) else None
self._rank_path_info: dict[str, Any] = {}
self.last_path_info: dict[str, Any] = {
@ -102,7 +104,11 @@ class RetrievalEngine:
clean_text(getattr(getattr(item, "memory", None), "id", ""), 120)
for item in results or []
]
current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id])
active_ids = [memory_id for memory_id in memory_ids if memory_id]
if self.p5_recall_provenance_required:
current = await self.store.p5_recall_eligible_memories_by_ids(active_ids)
else:
current = await self.store.get_memories_by_ids(active_ids)
acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state()
validated: list[SearchResult] = []
for item, memory_id in zip(results or [], memory_ids):
@ -126,7 +132,11 @@ class RetrievalEngine:
) -> list[dict[str, str]]:
"""Drop cached diagnostic snippets that are no longer readable."""
memory_ids = [clean_text(item.get("id"), 120) for item in blocked if isinstance(item, dict)]
current = await self.store.get_memories_by_ids([memory_id for memory_id in memory_ids if memory_id])
active_ids = [memory_id for memory_id in memory_ids if memory_id]
if self.p5_recall_provenance_required:
current = await self.store.p5_recall_eligible_memories_by_ids(active_ids)
else:
current = await self.store.get_memories_by_ids(active_ids)
acl_state = await self._acl_state() if self.policy.enable_acl_rules else self._empty_acl_state()
validated: list[dict[str, str]] = []
for item in blocked or []:
@ -866,7 +876,11 @@ class RetrievalEngine:
query = clean_text(query, 1000)
terms = self._terms(query)
graph_terms = []
if self.knowledge_graph_enabled and self.knowledge_graph_expansion_limit > 0:
if (
not self.p5_recall_provenance_required
and self.knowledge_graph_enabled
and self.knowledge_graph_expansion_limit > 0
):
graph_labels = await self.store.related_knowledge_terms(
terms,
scope=ctx.scope,
@ -881,7 +895,11 @@ class RetrievalEngine:
)
expanded_terms = self._merge_terms(terms, graph_terms)
include_pending = not self.policy.hide_pending_review
ranked_candidates = await self.store.list_candidate_memories(limit=2000, include_pending=include_pending)
ranked_candidates = await self.store.list_candidate_memories(
limit=2000,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
current_window_candidates = await self.store.list_current_window_candidate_memories(
scope=ctx.scope,
session_id=ctx.session_id,
@ -889,6 +907,7 @@ class RetrievalEngine:
group_id=ctx.group_id,
limit=self.current_window_candidate_limit,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
time_window_candidates: list[MemoryRecord] = []
if time_intent is not None and time_intent.active:
@ -897,12 +916,14 @@ class RetrievalEngine:
time_intent.end_at,
limit=1600,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
keyword_terms = self._keyword_candidate_terms(query, expanded_terms)
fts_candidates = await self.store.list_fts_candidate_memories(
keyword_terms,
limit=1200,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
use_keyword_fallback = len(fts_candidates) < self.keyword_fallback_min_fts_candidates
keyword_candidates = (
@ -910,6 +931,7 @@ class RetrievalEngine:
keyword_terms,
limit=1200,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
if use_keyword_fallback
else []
@ -917,6 +939,7 @@ class RetrievalEngine:
vector_candidates, vector_scores, embedding_info = await self._embedding_candidate_memories(
query,
include_pending=include_pending,
require_p5_recall_provenance=self.p5_recall_provenance_required,
)
self._rank_path_info = embedding_info
self._rank_path_info.update(
@ -1010,6 +1033,7 @@ class RetrievalEngine:
query: str,
*,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> tuple[list[MemoryRecord], dict[str, float], dict[str, Any]]:
info: dict[str, Any] = {
"embedding_enabled": bool(self.embedding_enabled),
@ -1046,6 +1070,7 @@ class RetrievalEngine:
provider_id=self.embedding_provider_id,
limit=self.embedding_candidate_limit,
include_pending=include_pending,
require_p5_recall_provenance=require_p5_recall_provenance,
)
except Exception as error:
info["embedding_reason"] = f"embedding_store_error:{clean_text(error, 120)}"

查看文件

@ -9,6 +9,7 @@ import json
import hashlib
import inspect
import re
import threading
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
@ -92,9 +93,13 @@ from .user_memory_boundary import UserMemoryBoundaryError, validate_user_memory_
from .user_memory_migration import UserMemoryMigration
from .unified_person_persistence import UnifiedPersonPersistence, UnifiedPersonReferenceMigration
from .person_memory import PersonMemoryFacts, PersonMemoryMigration
from .p5_provenance_persistence import P5ProvenanceMigration
from .p5_b1_recall_gate import evaluate_bridge_serialization_gate, evaluate_normal_recall_gate
from .visibility import VisibilityPolicy
_P5_ATTESTATION_BIND_FALLBACK_LOCK = threading.RLock()
_RECENT_FACT_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
("meal", re.compile(r"吃过|吃完|吃了|没吃|还没吃|正在吃|吃饭|早饭|早餐|午饭|午餐|晚饭|晚餐|夜宵")),
("bath", re.compile(r"洗澡|洗完澡|冲澡|沐浴")),
@ -198,10 +203,14 @@ class MemoryCompanionService:
self._retrieval_result_cache_stats: dict[str, int] = {"hits": 0, "misses": 0, "evictions": 0}
self._RETRIEVAL_RESULT_CACHE_TTL: float = 45.0
self._RETRIEVAL_RESULT_CACHE_MAX: int = 128
self._p5_attestation_verifier: Any = None
self._p5_attestation_verifier_bound = False
self._p5_attestation_verifier_lock = threading.RLock()
self.user_memory_migration = UserMemoryMigration(self.store)
self.unified_person_persistence = UnifiedPersonPersistence(self.store)
self.unified_person_reference_migration = UnifiedPersonReferenceMigration(self.store)
self.person_memory_migration = PersonMemoryMigration(self.store)
self.p5_provenance_migration = P5ProvenanceMigration(self.store)
self.person_memory_facts = PersonMemoryFacts(self.store)
self.portable_archive = PortableMemoryArchive(self.store, self.data_dir)
self.chat_importer = HistoricalChatImporter(self)
@ -281,6 +290,28 @@ class MemoryCompanionService:
async def rollback_person_memory_migration(self, migration_id: str) -> dict[str, Any]:
return await self.person_memory_migration.rollback(migration_id)
async def preview_p5_provenance_migration(self, *, limit: int = 5000) -> dict[str, Any]:
return await self.p5_provenance_migration.preview(limit=limit)
async def apply_p5_provenance_migration(self, *, operation_id: str, limit: int = 5000) -> dict[str, Any]:
return await self.p5_provenance_migration.apply(operation_id=operation_id, limit=limit)
async def rollback_p5_provenance_migration(self, operation_id: str) -> dict[str, Any]:
return await self.p5_provenance_migration.rollback(operation_id)
def bind_p5_attestation_verifier(self, consume_callable: Any) -> bool:
"""Pin the first Bridge-provided in-process B1 verifier."""
if not callable(consume_callable):
return False
bind_lock = getattr(self, "_p5_attestation_verifier_lock", _P5_ATTESTATION_BIND_FALLBACK_LOCK)
with bind_lock:
if getattr(self, "_p5_attestation_verifier_bound", False):
return False
self._p5_attestation_verifier = consume_callable
self._p5_attestation_verifier_bound = True
return True
async def bridge_correct_person_memory(
self, *, memory_id: str, correction_event_id: str, replacement: MemoryRecord,
) -> dict[str, Any]:
@ -500,6 +531,84 @@ class MemoryCompanionService:
ctx.context_orchestration = project_context_state(state) or {}
ctx.context_orchestration_status = "accepted"
def _p5_b1_record_normal_recall_audit(self, *, event: Any, req: Any, reason: str) -> None:
"""Expose only a static gate reason; never retain a handle/snapshot."""
for carrier in (req, event):
if carrier is None:
continue
try:
setattr(carrier, "memory_companion_p5_b1_recall_gate_reason", reason)
except Exception:
pass
async def _p5_b1_normal_recall_injection(
self,
ctx: SessionContext,
req: Any,
*,
event: Any,
p3_state: Any,
) -> tuple[str | None, str]:
"""Gate only the normal request-time memory injection seam.
With the default switch off this calls the historical seam unchanged.
With it on, the pinned verifier must consume the opaque handle using
these exact request/event/P3 object identities before recall proceeds.
"""
if not self.config.bool("enable_p5_b1_recall_gate", False):
return await self.inject_memories(ctx, req, event=event), ""
reason = evaluate_normal_recall_gate(
getattr(self, "_p5_attestation_verifier", None),
req=req,
event=event,
p3_state=p3_state,
)
if reason:
self._p5_b1_record_normal_recall_audit(event=event, req=req, reason=reason)
return None, reason
return await self.inject_memories(ctx, req, event=event), ""
def _p5_b1_bridge_gate_reason(self, invocation: Any) -> str:
"""Consume an explicitly carried B1b invocation before Bridge recall.
``None`` means a legacy/background Bridge caller which is outside the
current-request B1b scope. A protected Companion path always carries
an opaque invocation, including an intentionally unusable one when
minting or verifier binding failed, so it fails closed here.
"""
if invocation is None:
return ""
if not self.config.bool("enable_p5_b1_bridge_gate", False):
return ""
return evaluate_bridge_serialization_gate(
getattr(self, "_p5_attestation_verifier", None),
invocation=invocation,
)
def _p5_b1_tool_recall_gate_reason(self) -> str:
"""Fail closed because tool callbacks do not carry a request invocation."""
config = getattr(self, "config", None)
getter = getattr(config, "bool", None)
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b1_tool_recall_gate", False) else ""
def _p5_b2_archive_read_gate_reason(self) -> str:
"""Reject scope-dict archive reads until a trusted identity contract exists."""
config = getattr(self, "config", None)
getter = getattr(config, "bool", None)
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_archive_read_gate", False) else ""
def _p5_b2_cross_user_read_gate_reason(self) -> str:
"""Reject scope-dict cross-user reads until a trusted identity contract exists."""
config = getattr(self, "config", None)
getter = getattr(config, "bool", None)
return "p5_attestation_missing" if callable(getter) and getter("enable_p5_b2_cross_user_read_gate", False) else ""
async def handle_llm_request(self, event: Any, req: Any) -> None:
ctx = await self.identity.resolve_event_context(event)
self._sanitize_session_context_message_text(ctx)
@ -533,8 +642,13 @@ class MemoryCompanionService:
# this memory plugin no longer advances relationship authority.
try:
injection = await self.inject_memories(ctx, req, event=event)
if injection is not None:
injection, p5_b1_reason = await self._p5_b1_normal_recall_injection(
ctx, req, event=event, p3_state=p3_state
)
if p5_b1_reason:
self._mark_memory_companion_injection_state(event, req, injected=False, conversation_memory=False, slot_map={})
self._p3_complete_memory(p3_state, ctx=ctx, injection="", event=event, req=req)
elif injection is not None:
self._p3_complete_memory(p3_state, ctx=ctx, injection=injection, event=event, req=req)
except Exception as exc:
logger.warning(
@ -1354,6 +1468,16 @@ class MemoryCompanionService:
time_range: dict[str, str] | None = None,
) -> dict[str, Any]:
payload = dict(scope or {})
p5_b2_reason = self._p5_b2_cross_user_read_gate_reason()
if p5_b2_reason:
return {
**user_fact_query_capabilities(available=True, warnings=[p5_b2_reason]),
"query": clean_text(query, 1400),
"scope": payload,
"memories": [],
"excluded": [],
"warnings": [p5_b2_reason],
}
requested_scope = clean_text(payload.get("scope"), 40).lower()
user_id = clean_text(payload.get("user_id"), 120)
group_id = clean_text(payload.get("group_id"), 120)
@ -1445,6 +1569,15 @@ class MemoryCompanionService:
time_range: dict[str, str] | None = None,
include_timeline: bool = True,
) -> dict[str, Any]:
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
if p5_b2_reason:
return {
"archive_api_version": ARCHIVE_API_VERSION,
"scope": await self.bridge_get_archive_scope(scope=scope),
"memories": [],
"excluded": [],
"warnings": [p5_b2_reason],
}
ctx = self._archive_context(scope)
query_text = clean_text(query, 1400)
gate_reason = self._p3_gate_reason(
@ -1515,6 +1648,13 @@ class MemoryCompanionService:
scope: dict[str, Any],
include_timeline: bool = True,
) -> dict[str, Any]:
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
if p5_b2_reason:
return {
"archive_api_version": "1.0",
"scope": await self.bridge_get_archive_scope(scope=scope),
"memories": [], "excluded": [], "warnings": [p5_b2_reason],
}
ctx = self._archive_context(scope)
gate_reason = self._p3_gate_reason(
ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None,
@ -1628,6 +1768,14 @@ class MemoryCompanionService:
limit: int = 30,
include_timeline: bool = True,
) -> dict[str, Any]:
p5_b2_reason = self._p5_b2_archive_read_gate_reason()
if p5_b2_reason:
return {
"archive_api_version": "1.0",
"scope": await self.bridge_get_archive_scope(scope=scope),
"start_at": clean_text(start_at, 80), "end_at": clean_text(end_at, 80),
"items": [], "warnings": [p5_b2_reason],
}
ctx = self._archive_context(scope)
gate_reason = self._p3_gate_reason(
ctx, state=consume_context_state(ctx.context_orchestration) if ctx.context_orchestration else None,
@ -1694,7 +1842,10 @@ class MemoryCompanionService:
max_chars: int | None = None,
companion_bot_mood: str = "",
companion_bot_energy: float = 0.0,
p5_b1_bridge_invocation: Any = None,
) -> str:
if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation):
return ""
ctx = self.session_context_from_bridge(session_context)
if self._p3_bridge_gate_reason(ctx):
return ""
@ -1722,6 +1873,7 @@ class MemoryCompanionService:
companion_bot_mood: str = "",
companion_bot_energy: float = 0.0,
retrieval_profile: str = "",
p5_b1_bridge_invocation: Any = None,
) -> str:
profile = clean_text(retrieval_profile, 40).lower()
fast_profile_enabled = {
@ -1743,6 +1895,7 @@ class MemoryCompanionService:
companion_bot_mood=companion_bot_mood,
companion_bot_energy=companion_bot_energy,
context_kind="outfit" if profile == "outfit_fast" else "schedule",
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
)
return await self.bridge_compose_injection(
query,
@ -1751,6 +1904,7 @@ class MemoryCompanionService:
max_chars=max_chars,
companion_bot_mood=companion_bot_mood,
companion_bot_energy=companion_bot_energy,
p5_b1_bridge_invocation=p5_b1_bridge_invocation,
)
async def _bridge_compose_schedule_fast_context(
@ -1763,7 +1917,10 @@ class MemoryCompanionService:
companion_bot_mood: str,
companion_bot_energy: float,
context_kind: str = "schedule",
p5_b1_bridge_invocation: Any = None,
) -> str:
if self._p5_b1_bridge_gate_reason(p5_b1_bridge_invocation):
return ""
started = time.monotonic()
ctx = self._normalized_session_context(self.session_context_from_bridge(session_context))
if self._p3_bridge_gate_reason(ctx):
@ -1781,6 +1938,7 @@ class MemoryCompanionService:
bot_id=ctx.bot_id,
limit=max(72, limit * 12) if outfit_focus else max(24, limit * 6),
strict_session_only=ctx.strict_session_only,
require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False),
)
policy = self.visibility_policy()
visible = [record for record in records if policy.is_visible(record, ctx)[0]]
@ -2453,6 +2611,7 @@ class MemoryCompanionService:
retrieval_mode="basic",
embedding_enabled=False,
knowledge_graph_enabled=False,
p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False),
)
async def _retrieval_engine(self, ctx: SessionContext, *, admin_read_all: bool = False) -> RetrievalEngine:
@ -2494,6 +2653,7 @@ class MemoryCompanionService:
),
knowledge_graph_enabled=self.config.bool("knowledge_graph.retrieval_expansion_enabled", True),
knowledge_graph_expansion_limit=self.config.int("knowledge_graph.expansion_limit", 12),
p5_recall_provenance_required=self.config.bool("enable_p5_b1_recall_gate", False),
usage_recorder=self._record_token_usage,
)
@ -2986,6 +3146,7 @@ class MemoryCompanionService:
provider_id=provider_id,
limit=self.config.int("retrieval.embedding_backfill_batch_size", 50),
include_pending=self.config.bool("retrieval.embedding_index_pending", False),
require_p5_recall_provenance=self.config.bool("enable_p5_b1_recall_gate", False),
)
for record in records:
if await self._embed_memory_record(provider, provider_id, record):
@ -3004,12 +3165,23 @@ class MemoryCompanionService:
return False
if record.review_status == "pending" and not self.config.bool("retrieval.embedding_index_pending", False):
return False
p5_recall_provenance_required = self.config.bool("enable_p5_b1_recall_gate", False)
if p5_recall_provenance_required:
eligible = await self.store.p5_recall_eligible_memory_ids([record.id])
if record.id not in eligible:
return False
text = self._memory_embedding_text(record)
text_hash = self._memory_embedding_text_hash(record)
if not text or not text_hash:
return False
async with self._embedding_background_semaphore:
try:
# Revalidate immediately before external egress. This closes
# the interval while waiting for the shared provider slot.
if p5_recall_provenance_required:
eligible = await self.store.p5_recall_eligible_memory_ids([record.id])
if record.id not in eligible:
return False
vector = await self._embed_text_with_provider(provider, text, provider_id=provider_id)
vector = self._normalize_embedding_vector(vector)
if not vector:
@ -4060,6 +4232,9 @@ class MemoryCompanionService:
reason = self._p3_gate_reason(ctx, event=event, state=state)
if reason:
return {"ok": False, "error": reason, "memories": []}
p5_b1_reason = self._p5_b1_tool_recall_gate_reason()
if p5_b1_reason:
return {"ok": False, "error": p5_b1_reason, "memories": []}
query = clean_text(query, 1000)
if not query:
return {"ok": False, "error": "empty query", "memories": []}

查看文件

@ -14,6 +14,66 @@ from .models import EntityRef, MemoryRecord, clean_text, json_dumps, json_loads,
from .unified_person_persistence import ensure_unified_person_persistence_schema
from .person_memory import apply_envelope, ensure_person_memory_schema, indexable, normalize_record
from .person_memory_contract import consume_person_memory_envelope
from .p5_provenance_persistence import ensure_p5_provenance_persistence_schema
from .p5_provenance import (
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
SOURCE_KINDS as P5_SOURCE_KINDS,
contract_fingerprint as p5_provenance_contract_fingerprint,
)
def _p5_recall_eligibility_predicate(memory_alias: str = "") -> tuple[str, list[str]]:
"""Return the B1e SQL predicate for complete, valid recall records only."""
prefix = f"{memory_alias}." if memory_alias else ""
observed_source_kinds = tuple(kind for kind in P5_SOURCE_KINDS if kind != "unknown")
source_kind_placeholders = ", ".join("?" for _ in observed_source_kinds)
return (
f"""
EXISTS(
SELECT 1
FROM p5_provenance_records AS p5
WHERE p5.memory_id={prefix}id
AND p5.contract_name=?
AND p5.contract_version=?
AND p5.contract_fingerprint=?
AND p5.provenance_state='observed'
AND p5.source_kind IN ({source_kind_placeholders})
AND p5.source_trust IN ('T0', 'T1', 'T2')
AND p5.firewall_status IN ('allowed', 'sanitized')
AND length(p5.source_event_ref_hash)=64
AND p5.source_event_ref_hash NOT GLOB '*[^0-9a-f]*'
AND length(p5.authority_attestation_ref_hash)=64
AND p5.authority_attestation_ref_hash NOT GLOB '*[^0-9a-f]*'
AND p5.source_event_ref_hash != p5.authority_attestation_ref_hash
AND typeof(p5.record_revision)='integer'
AND p5.record_revision >= 0
AND length(p5.record_digest)=64
AND p5.record_digest NOT GLOB '*[^0-9a-f]*'
AND (
p5.migration_operation_ref=''
OR (
length(p5.migration_operation_ref)=64
AND p5.migration_operation_ref NOT GLOB '*[^0-9a-f]*'
)
)
AND (
p5.recovery_operation_ref=''
OR (
length(p5.recovery_operation_ref)=64
AND p5.recovery_operation_ref NOT GLOB '*[^0-9a-f]*'
)
)
)
""",
[
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
p5_provenance_contract_fingerprint(),
*observed_source_kinds,
],
)
class MemoryStore:
@ -495,6 +555,7 @@ class MemoryStore:
self._ensure_memory_columns_sync()
ensure_unified_person_persistence_schema(self._conn)
ensure_person_memory_schema(self._conn)
ensure_p5_provenance_persistence_schema(self._conn)
self._conn.execute(
"CREATE INDEX IF NOT EXISTS idx_memories_domain ON memories(memory_domain, lifecycle, scope, group_id, subject_id)"
)
@ -667,6 +728,22 @@ class MemoryStore:
BEGIN
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
END;
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_insert
AFTER INSERT ON p5_provenance_records
BEGIN
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
END;
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_update
AFTER UPDATE ON p5_provenance_records
BEGIN
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
END;
CREATE TRIGGER IF NOT EXISTS trg_p5_provenance_retrieval_revision_delete
AFTER DELETE ON p5_provenance_records
BEGIN
UPDATE retrieval_revision SET revision = revision + 1 WHERE singleton = 1;
END;
"""
)
@ -3492,14 +3569,34 @@ class MemoryStore:
result.append(item)
return result
async def list_candidate_memories(self, limit: int = 500, include_pending: bool = False) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(self._list_candidate_memories_sync, limit, include_pending)
async def list_candidate_memories(
self,
limit: int = 500,
include_pending: bool = False,
*,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_candidate_memories_sync,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_candidate_memories_sync(self, limit: int, include_pending: bool) -> list[MemoryRecord]:
def _list_candidate_memories_sync(
self,
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
where = "lifecycle != 'archived'"
params: list[Any] = []
if not include_pending:
where += " AND review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate()
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
f"""
@ -3520,6 +3617,7 @@ class MemoryStore:
bot_id: str = "",
limit: int = 36,
strict_session_only: bool = False,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
"""Read a small schedule-focused snapshot without waiting on the shared writer lock."""
return await asyncio.to_thread(
@ -3529,6 +3627,7 @@ class MemoryStore:
bot_id,
limit,
strict_session_only,
require_p5_recall_provenance,
)
def _list_schedule_context_memories_sync(
@ -3538,6 +3637,7 @@ class MemoryStore:
bot_id: str,
limit: int,
strict_session_only: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
session_id = clean_text(session_id, 200)
user_id = clean_text(user_id, 120)
@ -3623,6 +3723,10 @@ class MemoryStore:
return []
where += " AND session_id=?"
params.append(session_id)
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate("memories")
where += f" AND {predicate}"
params.extend(predicate_params)
uri = self.db_path.resolve().as_uri() + "?mode=ro"
with closing(sqlite3.connect(uri, uri=True, timeout=0.35, check_same_thread=False)) as connection:
connection.row_factory = sqlite3.Row
@ -3664,6 +3768,7 @@ class MemoryStore:
group_id: str = "",
limit: int = 600,
include_pending: bool = False,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_current_window_candidate_memories_sync,
@ -3673,6 +3778,7 @@ class MemoryStore:
group_id,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_current_window_candidate_memories_sync(
@ -3683,6 +3789,7 @@ class MemoryStore:
group_id: str,
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
scope = clean_text(scope, 40).lower()
session_id = clean_text(session_id, 200)
@ -3704,6 +3811,10 @@ class MemoryStore:
where = "lifecycle != 'archived' AND (" + " OR ".join(clauses) + ")"
if not include_pending:
where += " AND review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate()
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
f"""
@ -3724,6 +3835,7 @@ class MemoryStore:
end_at: str,
limit: int = 1200,
include_pending: bool = False,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_time_window_candidate_memories_sync,
@ -3731,6 +3843,7 @@ class MemoryStore:
end_at,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_time_window_candidate_memories_sync(
@ -3739,15 +3852,21 @@ class MemoryStore:
end_at: str,
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
start = clean_text(start_at, 80)
end = clean_text(end_at, 80)
if not start or not end:
return []
where = "lifecycle != 'archived'"
params: list[Any] = [start, end, start, end, start, end]
time_params: list[Any] = [start, end, start, end, start, end]
params: list[Any] = []
if not include_pending:
where += " AND review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate()
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
f"""
@ -3764,7 +3883,7 @@ class MemoryStore:
importance DESC
LIMIT ?
""",
params + [max(1, int(limit or 1))],
params + time_params + [max(1, int(limit or 1))],
).fetchall()
return [MemoryRecord.from_row(row) for row in rows]
@ -3773,12 +3892,15 @@ class MemoryStore:
terms: list[str],
limit: int = 800,
include_pending: bool = False,
*,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_fts_candidate_memories_sync,
terms,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_fts_candidate_memories_sync(
@ -3786,6 +3908,7 @@ class MemoryStore:
terms: list[str],
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
if not self._fts_enabled:
return []
@ -3796,6 +3919,10 @@ class MemoryStore:
params: list[Any] = [query]
if not include_pending:
where += " AND m.review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
where += f" AND {predicate}"
params.extend(predicate_params)
try:
with self._lock:
rows = self._conn.execute(
@ -3845,12 +3972,15 @@ class MemoryStore:
terms: list[str],
limit: int = 800,
include_pending: bool = False,
*,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_keyword_candidate_memories_sync,
terms,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_keyword_candidate_memories_sync(
@ -3858,6 +3988,7 @@ class MemoryStore:
terms: list[str],
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
cleaned_terms = []
for term in terms or []:
@ -3894,6 +4025,10 @@ class MemoryStore:
)
params.extend([like] * len(columns))
where += " AND (" + " OR ".join(term_clauses) + ")"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate()
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
@ -4585,6 +4720,48 @@ class MemoryStore:
async def get_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
return await self._run_recoverable_database_operation(self._get_memories_by_ids_sync, memory_ids)
async def p5_recall_eligible_memory_ids(self, memory_ids: list[str]) -> set[str]:
"""Return only IDs that still satisfy the B1e SQL provenance rule."""
return await self._run_recoverable_database_operation(
self._p5_recall_eligible_memory_ids_sync,
memory_ids,
)
def _p5_recall_eligible_memory_ids_sync(self, memory_ids: list[str]) -> set[str]:
ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120)))
if not ids:
return set()
placeholders = ",".join("?" for _ in ids)
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
with self._lock:
rows = self._conn.execute(
f"SELECT m.id FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}",
[*ids, *predicate_params],
).fetchall()
return {clean_text(row["id"], 120) for row in rows if clean_text(row["id"], 120)}
async def p5_recall_eligible_memories_by_ids(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
"""Atomically re-read cached bodies under the B1e provenance predicate."""
return await self._run_recoverable_database_operation(
self._p5_recall_eligible_memories_by_ids_sync,
memory_ids,
)
def _p5_recall_eligible_memories_by_ids_sync(self, memory_ids: list[str]) -> dict[str, MemoryRecord]:
ids = list(dict.fromkeys(clean_text(item, 120) for item in memory_ids if clean_text(item, 120)))
if not ids:
return {}
placeholders = ",".join("?" for _ in ids)
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
with self._lock:
rows = self._conn.execute(
f"SELECT m.* FROM memories AS m WHERE m.id IN ({placeholders}) AND {predicate}",
[*ids, *predicate_params],
).fetchall()
return {row["id"]: MemoryRecord.from_row(row) for row in rows}
async def get_timeline_event(self, event_id: str) -> dict[str, Any] | None:
return await self._run_recoverable_database_operation(self._get_timeline_event_sync, event_id)
@ -5252,12 +5429,14 @@ class MemoryStore:
provider_id: str,
limit: int = 3000,
include_pending: bool = False,
require_p5_recall_provenance: bool = False,
) -> list[tuple[MemoryRecord, list[float], str]]:
return await self._run_recoverable_database_operation(
self._list_embedding_candidate_rows_sync,
provider_id,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_embedding_candidate_rows_sync(
@ -5265,6 +5444,7 @@ class MemoryStore:
provider_id: str,
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[tuple[MemoryRecord, list[float], str]]:
provider_id = clean_text(provider_id, 160)
if not provider_id:
@ -5275,7 +5455,7 @@ class MemoryStore:
if revision != self._embedding_candidate_cache_revision:
self._embedding_candidate_cache.clear()
self._embedding_candidate_cache_revision = revision
cache_key = (provider_id, bool(include_pending), safe_limit)
cache_key = (provider_id, bool(include_pending), safe_limit, bool(require_p5_recall_provenance))
cached = self._embedding_candidate_cache.get(cache_key)
if cached is not None:
return deepcopy(cached)
@ -5283,6 +5463,10 @@ class MemoryStore:
params: list[Any] = [provider_id]
if not include_pending:
where += " AND m.review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
f"""
@ -5323,12 +5507,14 @@ class MemoryStore:
provider_id: str,
limit: int = 80,
include_pending: bool = False,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
return await self._run_recoverable_database_operation(
self._list_memories_missing_embeddings_sync,
provider_id,
limit,
include_pending,
require_p5_recall_provenance,
)
def _list_memories_missing_embeddings_sync(
@ -5336,6 +5522,7 @@ class MemoryStore:
provider_id: str,
limit: int,
include_pending: bool,
require_p5_recall_provenance: bool = False,
) -> list[MemoryRecord]:
provider_id = clean_text(provider_id, 160)
if not provider_id:
@ -5344,6 +5531,10 @@ class MemoryStore:
params: list[Any] = [provider_id]
if not include_pending:
where += " AND m.review_status != 'pending'"
if require_p5_recall_provenance:
predicate, predicate_params = _p5_recall_eligibility_predicate("m")
where += f" AND {predicate}"
params.extend(predicate_params)
with self._lock:
rows = self._conn.execute(
f"""

查看文件

@ -0,0 +1,263 @@
from __future__ import annotations
import asyncio
import tempfile
import unittest
from pathlib import Path
from core.models import EntityRef, MemoryRecord, SearchResult, SessionContext
from core.p5_provenance import (
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
contract_fingerprint,
)
from core.retrieval import RetrievalEngine
from core.service import MemoryCompanionService
from core.store import MemoryStore
from core.visibility import VisibilityPolicy
_HASH = "a" * 64
_OTHER_HASH = "b" * 64
class _RerankProvider:
model = "test-reranker"
def __init__(self) -> None:
self.documents: list[str] = []
async def rerank(self, **kwargs):
self.documents = list(kwargs.get("documents") or [])
return [{"index": index, "relevance_score": 1.0} for index in range(len(self.documents))]
class _ServiceConfig:
def __init__(self, p5_required: bool) -> None:
self.p5_required = p5_required
def bool(self, key: str, default: bool = False) -> bool:
if key == "enable_p5_b1_recall_gate":
return self.p5_required
return default
def int(self, _key: str, default: int = 0) -> int:
return default
def float(self, _key: str, default: float = 0.0) -> float:
return default
def get(self, _key: str, default=None):
return default
class P5B1CandidateGateTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
self.temp = tempfile.TemporaryDirectory()
self.store = MemoryStore(Path(self.temp.name) / "memory.db")
self.store.initialize()
for identifier in ("eligible", "legacy", "t3", "t4", "missing", "invalid", "bad_kind", "same_hash"):
await self.store.insert_memory(
MemoryRecord(
id=identifier,
memory_domain="user_memory",
subject=EntityRef(kind="user", id="u1"),
scope="private",
session_id="s1",
content=f"needle {identifier} private body",
evidence="test",
lifecycle="stable_memory",
visibility="shareable",
importance=0.8,
)
)
self._put_provenance("eligible")
self._put_provenance("legacy", state="legacy_unresolved", trust="T4", firewall="unknown", reference="")
self._put_provenance("t3", trust="T3")
self._put_provenance("t4", trust="T4")
self._put_provenance("invalid", reference=_HASH.upper())
self._put_provenance("bad_kind", source_kind="message")
self._put_provenance("same_hash", source_event_hash=_HASH)
async def asyncTearDown(self) -> None:
self.store.close()
self.temp.cleanup()
def _put_provenance(
self,
memory_id: str,
*,
state: str = "observed",
trust: str = "T1",
firewall: str = "allowed",
reference: str = _HASH,
source_kind: str = "current_user_intent",
source_event_hash: str = _OTHER_HASH,
) -> None:
self.store._conn.execute(
"""
INSERT INTO p5_provenance_records(
memory_id, contract_name, contract_version, contract_fingerprint,
source_kind, source_trust, firewall_status, source_event_ref_hash,
authority_attestation_ref_hash, provenance_state,
migration_operation_ref, recovery_operation_ref, record_revision,
record_digest, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', '', 1, ?,
'2026-07-29T00:00:00+00:00', '2026-07-29T00:00:00+00:00')
""",
(
memory_id,
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
contract_fingerprint(),
source_kind,
trust,
firewall,
source_event_hash,
reference,
state,
_HASH,
),
)
self.store._conn.commit()
@staticmethod
def _ids(rows) -> set[str]:
return {row.id if hasattr(row, "id") else row[0].id for row in rows}
async def test_store_sql_fts_keyword_vector_and_windows_fail_closed(self) -> None:
self.assertGreaterEqual(len(await self.store.list_candidate_memories(limit=20)), 6)
self.assertEqual(
{"eligible"}, self._ids(await self.store.list_candidate_memories(limit=20, require_p5_recall_provenance=True))
)
self.assertEqual(
{"eligible"},
self._ids(
await self.store.list_current_window_candidate_memories(
scope="private", session_id="s1", user_id="u1", limit=20,
require_p5_recall_provenance=True,
)
),
)
self.assertEqual(
{"eligible"},
self._ids(
await self.store.list_time_window_candidate_memories(
"2026-01-01T00:00:00+00:00", "2027-01-01T00:00:00+00:00", limit=20,
require_p5_recall_provenance=True,
)
),
)
self.assertEqual(
{"eligible"},
self._ids(await self.store.list_fts_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)),
)
self.assertEqual(
{"eligible"},
self._ids(await self.store.list_keyword_candidate_memories(["needle"], limit=20, require_p5_recall_provenance=True)),
)
self.assertEqual(
{"eligible"},
self._ids(
await self.store.list_memories_missing_embeddings(
provider_id="embed", limit=20, require_p5_recall_provenance=True
)
),
)
await self.store.upsert_memory_embedding(memory_id="eligible", provider_id="embed", text_hash="e", vector=[1.0, 0.0])
await self.store.upsert_memory_embedding(memory_id="legacy", provider_id="embed", text_hash="l", vector=[1.0, 0.0])
self.assertEqual(
{"eligible"},
self._ids(
await self.store.list_embedding_candidate_rows(
provider_id="embed", limit=20, require_p5_recall_provenance=True
)
),
)
self.store._conn.execute("UPDATE p5_provenance_records SET source_trust='T3' WHERE memory_id='eligible'")
self.store._conn.commit()
self.assertEqual(
set(),
self._ids(
await self.store.list_embedding_candidate_rows(
provider_id="embed", limit=20, require_p5_recall_provenance=True
)
),
)
async def test_engine_excludes_unattested_body_before_rerank_and_revalidates_cache(self) -> None:
provider = _RerankProvider()
engine = RetrievalEngine(
self.store,
VisibilityPolicy(enable_acl_rules=False),
retrieval_mode="rerank",
rerank_provider=provider,
rerank_provider_id="test",
embedding_enabled=False,
knowledge_graph_enabled=True,
p5_recall_provenance_required=True,
)
ctx = SessionContext(session_id="s1", scope="private", user_id="u1")
results = await engine.search("needle", ctx, 10)
self.assertEqual({"eligible"}, {item.memory.id for item in results})
self.assertEqual(1, len(provider.documents))
self.assertIn("needle eligible private body", provider.documents[0])
self.assertNotIn("needle legacy private body", provider.documents[0])
cached = await engine.revalidate_cached_results(
[SearchResult(memory=(await self.store.get_memory("legacy")), score=1.0)], ctx
)
self.assertEqual([], cached)
default_off = RetrievalEngine(
self.store,
VisibilityPolicy(enable_acl_rules=False),
retrieval_mode="basic",
embedding_enabled=False,
knowledge_graph_enabled=False,
p5_recall_provenance_required=False,
)
legacy_visible = await default_off.search("needle legacy", ctx, 10)
self.assertIn("legacy", {item.memory.id for item in legacy_visible})
async def test_service_propagates_recall_switch_to_retrieval_and_cache_validation_engines(self) -> None:
async def no_rerank_provider(*_args, **_kwargs):
return None, ""
for enabled in (False, True):
service = object.__new__(MemoryCompanionService)
service.store = self.store
service.config = _ServiceConfig(enabled)
service.visibility_policy = lambda **_kwargs: VisibilityPolicy(enable_acl_rules=False)
service._resolve_rerank_provider = no_rerank_provider
service._record_token_usage = lambda *_args, **_kwargs: None
service._embedding_timeout_ms = lambda: 1200
engine = await service._retrieval_engine(SessionContext())
validation = service._retrieval_validation_engine()
self.assertEqual(enabled, engine.p5_recall_provenance_required)
self.assertEqual(enabled, validation.p5_recall_provenance_required)
async def test_embedding_provider_rechecks_provenance_before_body_egress(self) -> None:
calls: list[str] = []
async def embed(_provider, text: str, *, provider_id: str):
calls.append(text)
return [1.0, 0.0]
def build_service(enabled: bool):
service = object.__new__(MemoryCompanionService)
service.store = self.store
service.config = _ServiceConfig(enabled)
service._embedding_background_semaphore = asyncio.Semaphore(1)
service._embed_text_with_provider = embed
return service
legacy = await self.store.get_memory("legacy")
self.assertFalse(await build_service(True)._embed_memory_record(object(), "embed-guard", legacy))
self.assertEqual([], calls)
self.assertTrue(await build_service(False)._embed_memory_record(object(), "embed-default-off", legacy))
self.assertEqual(1, len(calls))
if __name__ == "__main__":
unittest.main()

查看文件

@ -0,0 +1,337 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import threading
import unittest
from core.bridge import MemoryCompanionBridge
from core.p5_b1_recall_gate import (
BRIDGE_SERIALIZATION_SINK,
MEMORY_RECALL_SINK,
P5_B1_ATTESTATION_HANDLES_ATTRIBUTE,
)
from core.service import MemoryCompanionService
class _Config:
def __init__(self, enabled: bool, *, bridge_enabled: bool | None = None):
self.enabled = enabled
self.bridge_enabled = enabled if bridge_enabled is None else bridge_enabled
def bool(self, key: str, default: bool = False) -> bool:
if key == "enable_p5_b1_recall_gate":
return self.enabled
if key == "enable_p5_b1_bridge_gate":
return self.bridge_enabled
return default
def int(self, key: str, default: int = 0) -> int:
return default
class _Verifier:
def __init__(self, *, disposition: str = "allow", sink: str = MEMORY_RECALL_SINK):
self.disposition = disposition
self.sink = sink
self.calls: list[tuple[object, object, object, object, str]] = []
self.consumed: set[int] = set()
def __call__(self, handle, request, event, p3_state, sink):
self.calls.append((handle, request, event, p3_state, sink))
marker = id(handle)
if marker in self.consumed:
return None
self.consumed.add(marker)
return SimpleNamespace(disposition=self.disposition, sink=self.sink, secret_snapshot_prose="never retained")
def _service(*, enabled: bool, verifier=None):
service = object.__new__(MemoryCompanionService)
service.config = _Config(enabled)
service._p5_attestation_verifier = verifier
service._p5_attestation_verifier_bound = verifier is not None
service.inject_calls: list[tuple[object, object, object]] = []
async def inject(ctx, req, *, event):
service.inject_calls.append((ctx, req, event))
return "normal-memory-injection"
service.inject_memories = inject
return service
class _BridgeInvocation:
def __init__(self, handle, request, event, p3_state):
self.values = (handle, request, event, p3_state)
def _p5_b1_bridge_attestation_arguments(self):
return self.values
def _carriers(handle=object()):
request = SimpleNamespace()
event = SimpleNamespace()
setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {MEMORY_RECALL_SINK: handle})
return request, event, handle
class P5B1NormalRecallGateTests(unittest.TestCase):
def test_first_bind_pins_service_and_rejects_rebind_or_replacement(self):
service = _service(enabled=False)
bridge = MemoryCompanionBridge(service)
first = _Verifier()
second = _Verifier()
self.assertTrue(bridge.bind_p5_attestation_verifier(first))
self.assertIs(first, service._p5_attestation_verifier)
self.assertFalse(bridge.bind_p5_attestation_verifier(first))
self.assertFalse(bridge.bind_p5_attestation_verifier(second))
self.assertIs(first, service._p5_attestation_verifier)
self.assertFalse(service.bind_p5_attestation_verifier(second))
def test_concurrent_first_bind_has_one_winner_and_never_replaces_it(self):
service = _service(enabled=False)
bridge = MemoryCompanionBridge(service)
first = _Verifier()
second = _Verifier()
start = threading.Barrier(3)
results: list[tuple[object, bool]] = []
result_lock = threading.Lock()
def bind(verifier):
start.wait()
value = bridge.bind_p5_attestation_verifier(verifier)
with result_lock:
results.append((verifier, value))
first_thread = threading.Thread(target=bind, args=(first,))
second_thread = threading.Thread(target=bind, args=(second,))
first_thread.start()
second_thread.start()
start.wait()
first_thread.join(timeout=5)
second_thread.join(timeout=5)
self.assertFalse(first_thread.is_alive())
self.assertFalse(second_thread.is_alive())
winners = [verifier for verifier, accepted in results if accepted]
self.assertEqual(1, len(winners))
self.assertIs(winners[0], bridge._p5_attestation_verifier)
self.assertIs(winners[0], service._p5_attestation_verifier)
direct_service = _service(enabled=False)
direct_start = threading.Barrier(3)
direct_results: list[tuple[object, bool]] = []
def direct_bind(verifier):
direct_start.wait()
with result_lock:
direct_results.append((verifier, direct_service.bind_p5_attestation_verifier(verifier)))
direct_first = threading.Thread(target=direct_bind, args=(first,))
direct_second = threading.Thread(target=direct_bind, args=(second,))
direct_first.start()
direct_second.start()
direct_start.wait()
direct_first.join(timeout=5)
direct_second.join(timeout=5)
direct_winners = [verifier for verifier, accepted in direct_results if accepted]
self.assertEqual(1, len(direct_winners))
self.assertIs(direct_winners[0], direct_service._p5_attestation_verifier)
def test_default_off_preserves_normal_injection_without_verifier_or_handle(self):
service = _service(enabled=False)
request, event, _ = _carriers()
setattr(request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {})
injection, reason = asyncio.run(
service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=None)
)
self.assertEqual(("normal-memory-injection", ""), (injection, reason))
self.assertEqual(1, len(service.inject_calls))
def test_valid_allow_uses_exact_runtime_identities_and_reaches_normal_seam(self):
verifier = _Verifier()
service = _service(enabled=True, verifier=verifier)
request, event, handle = _carriers()
p3_state = object()
ctx = object()
injection, reason = asyncio.run(
service._p5_b1_normal_recall_injection(ctx, request, event=event, p3_state=p3_state)
)
self.assertEqual(("normal-memory-injection", ""), (injection, reason))
self.assertEqual([(handle, request, event, p3_state, MEMORY_RECALL_SINK)], verifier.calls)
self.assertEqual([(ctx, request, event)], service.inject_calls)
self.assertFalse(hasattr(request, "memory_companion_p5_b1_recall_gate_reason"))
def test_missing_fake_replay_wrong_identity_sink_and_nonallow_fail_closed(self):
cases = []
missing_service = _service(enabled=True, verifier=_Verifier())
missing_request, missing_event, _ = _carriers()
setattr(missing_request, P5_B1_ATTESTATION_HANDLES_ATTRIBUTE, {})
cases.append((missing_service, missing_request, missing_event, object(), "p5_b1_handle_missing"))
fake_service = _service(enabled=True, verifier=lambda *args: None)
fake_request, fake_event, _ = _carriers()
cases.append((fake_service, fake_request, fake_event, object(), "p5_b1_verifier_rejected"))
sink_service = _service(enabled=True, verifier=_Verifier(sink="bridge_serialization"))
sink_request, sink_event, _ = _carriers()
cases.append((sink_service, sink_request, sink_event, object(), "p5_b1_sink_mismatch"))
denied_service = _service(enabled=True, verifier=_Verifier(disposition="shadow_quarantine"))
denied_request, denied_event, _ = _carriers()
cases.append((denied_service, denied_request, denied_event, object(), "p5_b1_disposition_denied"))
p3_service = _service(enabled=True, verifier=_Verifier())
p3_request, p3_event, _ = _carriers()
cases.append((p3_service, p3_request, p3_event, None, "p5_b1_p3_unavailable"))
for service, request, event, p3_state, expected in cases:
with self.subTest(expected=expected):
injection, reason = asyncio.run(
service._p5_b1_normal_recall_injection(object(), request, event=event, p3_state=p3_state)
)
self.assertIsNone(injection)
self.assertEqual(expected, reason)
self.assertEqual([], service.inject_calls)
self.assertEqual(expected, request.memory_companion_p5_b1_recall_gate_reason)
rendered = repr(vars(request)) + repr(vars(event))
self.assertNotIn("secret_snapshot_prose", rendered)
replay_verifier = _Verifier()
replay_service = _service(enabled=True, verifier=replay_verifier)
replay_request, replay_event, _ = _carriers()
first, first_reason = asyncio.run(
replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object())
)
second, second_reason = asyncio.run(
replay_service._p5_b1_normal_recall_injection(object(), replay_request, event=replay_event, p3_state=object())
)
self.assertEqual(("normal-memory-injection", ""), (first, first_reason))
self.assertEqual((None, "p5_b1_verifier_rejected"), (second, second_reason))
def test_rejected_identity_verifier_receives_no_fallback_or_snapshot(self):
expected_request, expected_event, _ = _carriers()
def exact_only(handle, request, event, p3_state, sink):
if request is not expected_request or event is not expected_event or sink != MEMORY_RECALL_SINK:
return None
return SimpleNamespace(disposition="allow", sink=MEMORY_RECALL_SINK)
service = _service(enabled=True, verifier=exact_only)
replacement_request, replacement_event, _ = _carriers()
injection, reason = asyncio.run(
service._p5_b1_normal_recall_injection(
object(), replacement_request, event=replacement_event, p3_state=object()
)
)
self.assertEqual((None, "p5_b1_verifier_rejected"), (injection, reason))
self.assertEqual([], service.inject_calls)
def test_bridge_gate_is_default_off_and_enabled_denials_stop_before_retrieval(self):
request = SimpleNamespace()
event = SimpleNamespace()
p3_state = object()
handle = object()
def build_bridge_service(*, bridge_enabled: bool, verifier):
service = object.__new__(MemoryCompanionService)
service.config = _Config(False, bridge_enabled=bridge_enabled)
service._p5_attestation_verifier = verifier
service._p5_attestation_verifier_bound = verifier is not None
service.retrieval_calls = 0
service.session_context_from_bridge = lambda _value: SimpleNamespace(message_text="query")
service._p3_bridge_gate_reason = lambda _ctx: ""
async def compose(*_args, **_kwargs):
service.retrieval_calls += 1
return "bridge-memory"
service._compose_memory_injection = compose
return service
allowed = _Verifier(sink=BRIDGE_SERIALIZATION_SINK)
enabled_service = build_bridge_service(
bridge_enabled=True,
verifier=allowed,
)
result = asyncio.run(
enabled_service.bridge_compose_injection(
"query",
session_context={},
p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state),
)
)
self.assertEqual("bridge-memory", result)
self.assertEqual(1, enabled_service.retrieval_calls)
self.assertEqual(
[(handle, request, event, p3_state, BRIDGE_SERIALIZATION_SINK)],
allowed.calls,
)
replay = build_bridge_service(bridge_enabled=True, verifier=allowed)
replay_invocation = _BridgeInvocation(handle, request, event, p3_state)
self.assertEqual(
"",
asyncio.run(
replay.bridge_compose_injection(
"query", session_context={}, p5_b1_bridge_invocation=replay_invocation
)
),
)
self.assertEqual(0, replay.retrieval_calls)
for verifier in (
_Verifier(disposition="shadow_quarantine", sink=BRIDGE_SERIALIZATION_SINK),
_Verifier(sink=MEMORY_RECALL_SINK),
):
with self.subTest(verifier=verifier.disposition + verifier.sink):
denied = build_bridge_service(bridge_enabled=True, verifier=verifier)
self.assertEqual(
"",
asyncio.run(
denied.bridge_compose_injection(
"query",
session_context={},
p5_b1_bridge_invocation=_BridgeInvocation(object(), request, event, p3_state),
)
),
)
self.assertEqual(0, denied.retrieval_calls)
for invocation in (
None,
{},
_BridgeInvocation(object(), SimpleNamespace(), event, p3_state),
_BridgeInvocation(object(), request, event, p3_state),
):
with self.subTest(invocation=type(invocation).__name__):
denied = build_bridge_service(bridge_enabled=True, verifier=lambda *args: None)
blocked = asyncio.run(
denied.bridge_compose_injection(
"query",
session_context={},
p5_b1_bridge_invocation=invocation,
)
)
if invocation is None:
self.assertEqual("bridge-memory", blocked)
self.assertEqual(1, denied.retrieval_calls)
else:
self.assertEqual("", blocked)
self.assertEqual(0, denied.retrieval_calls)
default_off = build_bridge_service(bridge_enabled=False, verifier=None)
legacy = asyncio.run(
default_off.bridge_compose_injection(
"query",
session_context={},
p5_b1_bridge_invocation=_BridgeInvocation(handle, request, event, p3_state),
)
)
self.assertEqual("bridge-memory", legacy)
self.assertEqual(1, default_off.retrieval_calls)
if __name__ == "__main__":
unittest.main()

查看文件

@ -0,0 +1,117 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import unittest
from core.service import MemoryCompanionService
class _Config:
def __init__(self, **flags: bool) -> None:
self.flags = flags
def bool(self, key: str, default: bool = False) -> bool:
return bool(self.flags.get(key, default))
def int(self, _key: str, default: int = 0) -> int:
return default
class _Identity:
async def resolve_event_context(self, _event):
return SimpleNamespace()
def _service(**flags: bool):
service = object.__new__(MemoryCompanionService)
service.config = _Config(**flags)
service.identity = _Identity()
service._p3_prepare_context = lambda _ctx, **_kwargs: object()
service._p3_gate_reason = lambda *_args, **_kwargs: ""
service._archive_context = lambda _scope: SimpleNamespace(
session_id="s1", scope="private", platform="test", user_id="u1", group_id="", bot_id="b1",
message_id="m1", strict_session_only=True, context_orchestration=None,
)
async def archive_scope(*, scope=None):
return {"scope": dict(scope or {})}
service.bridge_get_archive_scope = archive_scope
return service
class P5B1UnverifiableReadGateTests(unittest.TestCase):
def test_tool_recall_default_off_preserves_search_and_enabled_stops_before_search(self):
default_off = _service()
default_off.search_calls = 0
async def search(*_args, **_kwargs):
default_off.search_calls += 1
return []
default_off.search = search
self.assertEqual(
{"ok": True, "memories": []}, asyncio.run(default_off.tool_recall(object(), "needle"))
)
self.assertEqual(1, default_off.search_calls)
enabled = _service(enable_p5_b1_tool_recall_gate=True)
async def forbidden_search(*_args, **_kwargs):
raise AssertionError("tool gate must stop before search")
enabled.search = forbidden_search
self.assertEqual(
{"ok": False, "error": "p5_attestation_missing", "memories": []},
asyncio.run(enabled.tool_recall(object(), "needle")),
)
def test_archive_gate_stops_search_evidence_and_timeline_before_data_reads(self):
service = _service(enable_p5_b2_archive_read_gate=True)
async def forbidden(*_args, **_kwargs):
raise AssertionError("archive gate must stop before a data read")
service.search = forbidden
service.store = SimpleNamespace(get_memories_by_ids=forbidden, get_timeline_event=forbidden)
searched = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"}))
evidence = asyncio.run(service.bridge_get_archive_evidence(["m1"], scope={"scope": "private"}))
context = asyncio.run(
service.bridge_get_archive_session_context(
scope={"scope": "private"}, start_at="2026-01-01T00:00:00+00:00", end_at="2026-02-01T00:00:00+00:00"
)
)
self.assertEqual(["p5_attestation_missing"], searched["warnings"])
self.assertEqual(([], [], ["p5_attestation_missing"]), (evidence["memories"], evidence["excluded"], evidence["warnings"]))
self.assertEqual(([], ["p5_attestation_missing"]), (context["items"], context["warnings"]))
def test_archive_default_off_reaches_legacy_search(self):
service = _service()
service.search_calls = 0
async def search(*_args, **_kwargs):
service.search_calls += 1
return []
service.search = search
service.archive_visibility_policy = lambda: object()
result = asyncio.run(service.bridge_search_archive_memories("needle", scope={"scope": "private"}))
self.assertEqual([], result["memories"])
self.assertEqual(1, service.search_calls)
def test_cross_user_gate_returns_empty_without_delegating_to_archive(self):
service = _service(enable_p5_b2_cross_user_read_gate=True)
async def forbidden(*_args, **_kwargs):
raise AssertionError("cross-user gate must stop before archive delegation")
service.bridge_search_archive_memories = forbidden
result = asyncio.run(
service.bridge_query_user_facts("needle", scope={"scope": "private", "user_id": "u1"})
)
self.assertEqual(([], [], ["p5_attestation_missing"]), (result["memories"], result["excluded"], result["warnings"]))
if __name__ == "__main__":
unittest.main()

229
tests/test_p5_provenance.py 普通文件
查看文件

@ -0,0 +1,229 @@
from __future__ import annotations
import copy
from dataclasses import dataclass
import hashlib
import json
import unittest
from core.p5_provenance import (
COMPANION_ATTESTATION_ISSUER,
P3_CONTRACT_FINGERPRINT,
P3_CONTRACT_NAME,
P3_CONTRACT_VERSION,
PROVENANCE_CONTRACT_NAME,
PROVENANCE_CONTRACT_VERSION,
apply_planned_operation,
contract_descriptor,
contract_fingerprint,
legacy_unresolved,
observed_from_companion_snapshot,
plan_legacy_migration,
plan_owner_recovery,
rollback_planned_operation,
validate_provenance_record,
)
def _hash(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
@dataclass(frozen=True)
class _CompanionStyleSnapshot:
schema_version: str
contract_name: str
contract_version: str
contract_fingerprint: str
issuer: str
issuer_epoch: str
authority_attestation_ref_hash: str
request_hash: str
session_hash: str
source_kind: str
source_trust: str
firewall_status: str
disposition: str
reason_codes: tuple[str, ...]
source_event_ref_hash: str
derived_from_ref_hash: str
provenance_state: str
p3_contract_name: str
p3_contract_version: str
p3_contract_fingerprint: str
sink: str
def _snapshot(**overrides):
value = {
"schema_version": "ops.p5.attestation.v1",
"contract_name": PROVENANCE_CONTRACT_NAME,
"contract_version": PROVENANCE_CONTRACT_VERSION,
"contract_fingerprint": contract_fingerprint(),
"issuer": COMPANION_ATTESTATION_ISSUER,
"issuer_epoch": _hash("issuer-epoch-1"),
"p3_contract_name": P3_CONTRACT_NAME,
"p3_contract_version": P3_CONTRACT_VERSION,
"p3_contract_fingerprint": P3_CONTRACT_FINGERPRINT,
"source_kind": "forwarded_text",
"source_trust": "T3",
"firewall_status": "sanitized",
"disposition": "shadow_quarantine",
"reason_codes": ("p5b_attested",),
"source_event_ref_hash": _hash("source-event-1"),
"authority_attestation_ref_hash": _hash("companion-attestation-1"),
"request_hash": _hash("request-1"),
"session_hash": _hash("session-1"),
"derived_from_ref_hash": "",
"provenance_state": "observed",
"sink": "memory_recall",
}
value.update(overrides)
return value
class P5ProvenanceTests(unittest.TestCase):
def test_descriptor_and_legacy_record_are_json_serializable(self):
descriptor = contract_descriptor()
expected = {
"contract_name": "ops.p5.provenance.v1",
"contract_version": "1.0",
"record_fields": [
"contract_name", "contract_version", "contract_fingerprint", "memory_id",
"source_kind", "source_trust", "firewall_status", "source_event_ref_hash",
"authority_attestation_ref_hash", "provenance_state", "migration_operation_ref",
"recovery_operation_ref", "record_revision",
],
"provenance_states": ["observed", "legacy_unresolved", "owner_recovered", "invalid"],
"trust_levels": ["T0", "T1", "T2", "T3", "T4"],
"source_kinds": [
"policy_config", "verified_authorization", "current_user_intent", "forwarded_text",
"quoted_text", "vision_summary", "tool_output", "web_extract", "memory_recall",
"derived_summary", "legacy_memory", "unknown",
],
"firewall_statuses": [
"allowed", "sanitized", "blocked", "rejected", "quarantined", "unavailable", "unknown",
],
"dispositions": ["allow", "shadow_quarantine", "deny_high_risk"],
"hash_format": "sha256_hex_lower",
"attestation_issuer": "private_companion",
"p3_contract_name": P3_CONTRACT_NAME,
"p3_contract_version": P3_CONTRACT_VERSION,
}
self.assertEqual(expected, descriptor)
canonical = json.dumps(expected, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
self.assertEqual(hashlib.sha256(canonical.encode("utf-8")).hexdigest(), contract_fingerprint())
record = legacy_unresolved("memory-42", record_revision=4)
self.assertEqual(PROVENANCE_CONTRACT_NAME, descriptor["contract_name"])
self.assertEqual(PROVENANCE_CONTRACT_VERSION, descriptor["contract_version"])
self.assertEqual(64, len(contract_fingerprint()))
self.assertEqual("legacy_unresolved", record["provenance_state"])
self.assertEqual(("unknown", "T4", "unknown"), (record["source_kind"], record["source_trust"], record["firewall_status"]))
self.assertTrue(validate_provenance_record(record)["ok"])
json.dumps({"descriptor": descriptor, "record": record}, ensure_ascii=False, sort_keys=True)
def test_only_attested_exact_p3_snapshot_can_be_observed(self):
observed = observed_from_companion_snapshot(
"memory-42",
_snapshot(
source_event_ref_hash=f"sha256:{_hash('source-event-1').upper()}",
authority_attestation_ref_hash=f"sha256:{_hash('companion-attestation-1').upper()}",
),
record_revision=7,
)
self.assertEqual("observed", observed["provenance_state"])
self.assertTrue(validate_provenance_record(observed)["ok"])
self.assertEqual(_hash("source-event-1"), observed["source_event_ref_hash"])
self.assertEqual(_hash("companion-attestation-1"), observed["authority_attestation_ref_hash"])
self.assertNotIn("disposition", observed)
noncanonical_record = copy.deepcopy(observed)
noncanonical_record["source_event_ref_hash"] = f"sha256:{_hash('source-event-1')}"
self.assertFalse(validate_provenance_record(noncanonical_record)["ok"])
companion_style = _CompanionStyleSnapshot(**_snapshot())
from_dataclass = observed_from_companion_snapshot("memory-43", companion_style)
self.assertEqual("observed", from_dataclass["provenance_state"])
for name, override in (
("pseudo source plugin", {"source_kind": "source_plugin"}),
("p5 descriptor mismatch", {"contract_fingerprint": _hash("wrong-p5")}),
("p3 mismatch", {"p3_contract_fingerprint": _hash("wrong-p3")}),
("missing attestation", {"authority_attestation_ref_hash": ""}),
("invalid trust", {"source_trust": "T9"}),
("conflicting source hashes", {"source_hash": _hash("other-source")}),
("same roles hash", {"authority_attestation_ref_hash": _hash("source-event-1")}),
("status used as disposition", {"disposition": "sanitized"}),
("disposition used as status", {"firewall_status": "shadow_quarantine"}),
):
with self.subTest(name=name):
rejected = observed_from_companion_snapshot("memory-42", _snapshot(**override))
self.assertEqual("invalid", rejected["provenance_state"])
def test_preview_is_zero_write_and_does_not_mutate_or_capture_content(self):
source = [{"memory_id": "memory-42", "record_revision": 4}]
before = copy.deepcopy(source)
preview = plan_legacy_migration(source, operation_ref_hash=_hash("migration-42"))
self.assertEqual(before, source)
self.assertEqual(("preview", True, 0), (preview["mode"], preview["readonly"], preview["write_count"]))
self.assertEqual(1, len(preview["operations"]))
self.assertEqual([], preview["error_codes"])
self.assertEqual("legacy_unresolved", preview["operations"][0]["after_record"]["provenance_state"])
hostile = [{"memory_id": "memory-43", "record_revision": 0, "content": "do not persist this"}]
rejected = plan_legacy_migration(hostile, operation_ref_hash=_hash("migration-43"))
rendered = json.dumps(rejected, ensure_ascii=False, sort_keys=True)
self.assertNotIn("do not persist this", rendered)
self.assertNotIn("memory-43", rendered)
self.assertEqual([], rejected["operations"])
def test_apply_retry_and_rollback_are_revision_guarded(self):
preview = plan_legacy_migration(
[{"memory_id": "memory-42", "record_revision": 4}], operation_ref_hash=_hash("migration-42")
)
operation = preview["operations"][0]
current = {"memory_id": "memory-42", "record_revision": 4}
applied = apply_planned_operation(current, operation)
self.assertEqual((True, "applied", True), (applied["ok"], applied["status"], applied["changed"]))
retry = apply_planned_operation(applied["record"], operation)
self.assertEqual((True, "idempotent", False), (retry["ok"], retry["status"], retry["changed"]))
concurrent = copy.deepcopy(applied["record"])
concurrent["record_revision"] += 1
rejected = rollback_planned_operation(concurrent, operation)
self.assertEqual((False, "revision_conflict"), (rejected["ok"], rejected["status"]))
rollback = rollback_planned_operation(applied["record"], operation)
self.assertEqual((True, "rolled_back", True), (rollback["ok"], rollback["status"], rollback["removed"]))
self.assertIsNone(rollback["record"])
def test_owner_recovery_requires_companion_evidence_and_keeps_operation_ref(self):
legacy = legacy_unresolved("memory-42", record_revision=2)
planned = plan_owner_recovery(
legacy, _snapshot(), recovery_operation_ref_hash=_hash("recovery-42")
)
self.assertTrue(planned["ok"])
after = planned["operation"]["after_record"]
self.assertEqual("owner_recovered", after["provenance_state"])
self.assertEqual(_hash("recovery-42"), after["recovery_operation_ref"])
rejected = plan_owner_recovery(
legacy,
_snapshot(p3_contract_fingerprint=_hash("wrong-p3")),
recovery_operation_ref_hash=_hash("recovery-43"),
)
self.assertFalse(rejected["ok"])
self.assertEqual(["companion_snapshot_invalid"], rejected["error_codes"])
def test_record_validation_rejects_raw_source_ids_and_bodies_without_echo(self):
record = legacy_unresolved("memory-42")
record["source_event_id"] = "raw-event-99"
record["content"] = "hostile body"
result = validate_provenance_record(record)
self.assertFalse(result["ok"])
rendered = json.dumps(result, ensure_ascii=False, sort_keys=True)
self.assertNotIn("raw-event-99", rendered)
self.assertNotIn("hostile body", rendered)
self.assertIn("forbidden_raw_field", result["error_codes"])
if __name__ == "__main__":
unittest.main()

查看文件

@ -0,0 +1,184 @@
from __future__ import annotations
import asyncio
import sqlite3
import tempfile
import unittest
from pathlib import Path
from core.bridge import MemoryCompanionBridge
from core.models import MemoryRecord
from core.p5_provenance_persistence import P5ProvenanceMigration
from core.service import MemoryCompanionService
from core.store import MemoryStore
class _MigrationDouble:
async def preview(self, *, limit: int):
return {"mode": "preview", "ok": True, "limit": limit}
async def apply(self, *, operation_id: str, limit: int):
return {"mode": "apply", "ok": True, "operation_id": operation_id, "limit": limit}
async def rollback(self, operation_id: str):
return {"mode": "rollback", "ok": True, "operation_id": operation_id}
class P5ProvenancePersistenceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.store = MemoryStore(Path(self.temp.name) / "memory.db")
self.store.initialize()
self.migration = P5ProvenanceMigration(self.store)
def tearDown(self):
self.store.close()
self.temp.cleanup()
def _seed(self, identifier: str, *, source_plugin: str = "memory_companion") -> str:
return asyncio.run(
self.store.insert_memory(
MemoryRecord(
id=identifier,
memory_type="observation",
scope="private",
session_id="private:test",
content=f"memory {identifier}",
evidence="test evidence",
source_plugin=source_plugin,
)
)
)
def test_schema_is_additive_and_preview_is_zero_write(self):
self._seed("memory-001")
tables = {
row["name"]
for row in self.store._conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'p5_provenance_%'"
).fetchall()
}
self.assertEqual(
{
"p5_provenance_records",
"p5_provenance_migration_runs",
"p5_provenance_migration_backups",
},
tables,
)
before_changes = self.store._conn.total_changes
preview = asyncio.run(self.migration.preview(limit=50))
self.assertEqual(("preview", True, True, 1, 0), (
preview["mode"], preview["ok"], preview["readonly"], preview["count"], preview["write_count"],
))
self.assertEqual(before_changes, self.store._conn.total_changes)
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
def test_apply_creates_legacy_records_and_backup_manifest_idempotently(self):
self._seed("memory-001", source_plugin="source_plugin")
self._seed("memory-002")
result = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50))
self.assertEqual(("apply", True, 2, False, 2), (
result["mode"], result["ok"], result["changed"], result["deduplicated"], result["backup_count"],
))
backup_path = Path(result["backup_path"])
self.assertTrue(backup_path.is_file())
backup_connection = sqlite3.connect(backup_path)
try:
self.assertEqual(0, backup_connection.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
finally:
backup_connection.close()
rows = self.store._conn.execute(
"SELECT source_kind, source_trust, firewall_status, provenance_state, source_event_ref_hash, authority_attestation_ref_hash FROM p5_provenance_records ORDER BY memory_id"
).fetchall()
self.assertEqual(2, len(rows))
for row in rows:
self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved", "", ""), tuple(row))
manifest = self.store._conn.execute(
"SELECT before_exists, before_digest, before_revision, after_digest, after_revision, apply_state FROM p5_provenance_migration_backups ORDER BY memory_id"
).fetchall()
self.assertEqual(2, len(manifest))
self.assertTrue(all(tuple(row)[0:3] == (0, "", 0) and row[4] == 1 and row[5] == "applied" for row in manifest))
retry = asyncio.run(self.migration.apply(operation_id="p5-b0-apply-1", limit=50))
self.assertEqual((True, 0, True, 2), (retry["ok"], retry["changed"], retry["deduplicated"], retry["backup_count"]))
self.assertEqual(str(backup_path), retry["backup_path"])
def test_invalid_or_failed_backup_leaves_no_run_or_provenance_mutation(self):
self._seed("memory-001")
backup_glob = "memory.backup.*.before_p5_provenance_b0.*.db"
before_paths = set(Path(self.temp.name).glob(backup_glob))
invalid = asyncio.run(self.migration.apply(operation_id="", limit=50))
self.assertEqual("operation_id_required", invalid["error_code"])
self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob)))
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0])
original_backup = self.store.backup
self.store.backup = lambda suffix="": (_ for _ in ()).throw(RuntimeError("backup unavailable"))
try:
failed = asyncio.run(self.migration.apply(operation_id="p5-b0-backup-fail", limit=50))
finally:
self.store.backup = original_backup
self.assertEqual((False, "physical_backup_failed", 0), (failed["ok"], failed["error_code"], failed["changed"]))
self.assertEqual(before_paths, set(Path(self.temp.name).glob(backup_glob)))
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_migration_runs").fetchone()[0])
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
def test_rollback_deletes_only_untouched_operation_owned_records(self):
self._seed("memory-001")
asyncio.run(self.migration.apply(operation_id="p5-b0-rollback-1", limit=50))
rollback = asyncio.run(self.migration.rollback("p5-b0-rollback-1"))
self.assertEqual(("rollback", True, 1, 0), (rollback["mode"], rollback["ok"], rollback["restored"], rollback["conflicts"]))
self.assertEqual(0, self.store._conn.execute("SELECT COUNT(*) FROM p5_provenance_records").fetchone()[0])
duplicate = asyncio.run(self.migration.rollback("p5-b0-rollback-1"))
self.assertTrue(duplicate["deduplicated"])
def test_rollback_cas_conflict_preserves_independent_newer_record(self):
self._seed("memory-001")
asyncio.run(self.migration.apply(operation_id="p5-b0-conflict-1", limit=50))
self.store._conn.execute(
"UPDATE p5_provenance_records SET record_revision=record_revision+1 WHERE memory_id='memory-001'"
)
self.store._conn.commit()
rollback = asyncio.run(self.migration.rollback("p5-b0-conflict-1"))
self.assertEqual((False, 0, 1), (rollback["ok"], rollback["restored"], rollback["conflicts"]))
record = self.store._conn.execute(
"SELECT record_revision, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'"
).fetchone()
self.assertEqual((2, "legacy_unresolved"), tuple(record))
backup = self.store._conn.execute(
"SELECT rollback_state FROM p5_provenance_migration_backups WHERE operation_id='p5-b0-conflict-1' AND memory_id='memory-001'"
).fetchone()
self.assertEqual("conflict", backup[0])
def test_forged_source_and_p3_metadata_never_upgrades_legacy_backfill(self):
self._seed("memory-001", source_plugin="source_plugin")
self.store._conn.execute(
"UPDATE memories SET metadata=? WHERE id='memory-001'",
('{"source_kind":"source_plugin","p3_contract_fingerprint":"forged"}',),
)
self.store._conn.commit()
asyncio.run(self.migration.apply(operation_id="p5-b0-forged-1", limit=50))
record = self.store._conn.execute(
"SELECT source_kind, source_trust, firewall_status, provenance_state FROM p5_provenance_records WHERE memory_id='memory-001'"
).fetchone()
self.assertEqual(("unknown", "T4", "unknown", "legacy_unresolved"), tuple(record))
def test_service_and_bridge_migration_wrappers_and_unavailable_bridge(self):
service = object.__new__(MemoryCompanionService)
service.p5_provenance_migration = _MigrationDouble()
self.assertEqual("preview", asyncio.run(service.preview_p5_provenance_migration(limit=7))["mode"])
self.assertEqual("op-1", asyncio.run(service.apply_p5_provenance_migration(operation_id="op-1", limit=7))["operation_id"])
self.assertEqual("rollback", asyncio.run(service.rollback_p5_provenance_migration("op-1"))["mode"])
unavailable = MemoryCompanionBridge(object())
result = asyncio.run(unavailable.preview_p5_provenance_migration(limit=7))
self.assertEqual((False, "bridge_method_unavailable"), (result["ok"], result["error_code"]))
applied = asyncio.run(unavailable.apply_p5_provenance_migration(operation_id="op-1", limit=7))
self.assertEqual((False, "bridge_method_unavailable", 0), (applied["ok"], applied["error_code"], applied["changed"]))
rolled_back = asyncio.run(unavailable.rollback_p5_provenance_migration("op-1"))
self.assertEqual((False, "bridge_method_unavailable", 0), (rolled_back["ok"], rolled_back["error_code"], rolled_back["restored"]))
self.assertFalse(unavailable.probe_p5_provenance_migration_capabilities()["available"])
if __name__ == "__main__":
unittest.main()