astrbot_plugin_private_comp.../bot_personal_outbox.py

350 行
15 KiB
Python

# -*- coding: utf-8 -*-
"""Durable, failure-contained delivery queue for Bot Personal archives.
The companion plugin owns the local write. A remote MemoryCompanion Bridge is
only a delivery target, so a missing or failing bridge cannot interrupt chat,
agenda maintenance, or proactive messaging.
"""
from __future__ import annotations
import asyncio
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
from typing import Any, Awaitable, Callable, Mapping
from .bot_personal_dto import BotPersonalArchiveDTO, build_bot_personal_dto
OUTBOX_STATES = frozenset({"pending", "retry", "sent", "deduplicated", "dead_letter", "invalid"})
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
def _timestamp(value: Any, fallback: float | None = None) -> float:
if isinstance(value, (int, float)):
return float(value)
try:
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
except (TypeError, ValueError, OverflowError):
return float(fallback if fallback is not None else 0.0)
def _fingerprint(envelope: Mapping[str, Any]) -> str:
canonical = json.dumps(dict(envelope), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
@dataclass(frozen=True)
class OutboxResult:
ok: bool
state: str
idempotency_key: str
record_id: str = ""
deduplicated: bool = False
version: int = 0
error_code: str | None = None
attempts: int = 0
def as_dict(self) -> dict[str, Any]:
return {
"ok": self.ok,
"state": self.state,
"idempotency_key": self.idempotency_key,
"record_id": self.record_id,
"deduplicated": self.deduplicated,
"version": self.version,
"error_code": self.error_code,
"attempts": self.attempts,
}
class BotPersonalOutbox:
"""Persisted queue backed by ``plugin.data['bot_personal_outbox']``.
``clock`` returns wall-clock seconds and is injectable for deterministic
tests. ``sender`` is called only by :meth:`drain`; it should return the
structured Bridge result and must not raise for ordinary remote failures.
"""
def __init__(
self,
data: dict[str, Any],
*,
save: Callable[[], Any] | None = None,
clock: Callable[[], float] | None = None,
max_attempts: int = 5,
base_backoff_seconds: float = 30.0,
max_backoff_seconds: float = 3600.0,
) -> None:
self.data = data
self.save = save
self.clock = clock or __import__("time").time
self.max_attempts = max(1, int(max_attempts or 5))
self.base_backoff_seconds = max(0.0, float(base_backoff_seconds or 30.0))
self.max_backoff_seconds = max(self.base_backoff_seconds, float(max_backoff_seconds or 3600.0))
self._lock = asyncio.Lock()
entries = self.data.setdefault("bot_personal_outbox", [])
if not isinstance(entries, list):
self.data["bot_personal_outbox"] = []
@property
def entries(self) -> list[dict[str, Any]]:
entries = self.data.get("bot_personal_outbox")
if not isinstance(entries, list):
entries = []
self.data["bot_personal_outbox"] = entries
return [item for item in entries if isinstance(item, dict)]
def _replace_entries(self, entries: list[dict[str, Any]]) -> None:
self.data["bot_personal_outbox"] = entries
def _persist(self) -> None:
if not callable(self.save):
return
try:
result = self.save()
if asyncio.iscoroutine(result) or hasattr(result, "__await__"):
asyncio.create_task(result)
except Exception:
# The in-memory queue remains authoritative for this process; the
# next normal store save can still persist the already-mutated data.
return
@staticmethod
def _key(memory_type: str, idempotency_key: str) -> str:
return f"{str(memory_type or '').strip()}::{str(idempotency_key or '').strip()}"
def _find(self, entries: list[dict[str, Any]], memory_type: str, key: str) -> dict[str, Any] | None:
composite = self._key(memory_type, key)
for entry in entries:
if self._key(entry.get("memory_type"), entry.get("idempotency_key")) == composite:
return entry
return None
def _normalise_sender_result(self, result: Any) -> dict[str, Any]:
if not isinstance(result, dict):
return {"ok": False, "state": "retry", "error_code": "invalid_bridge_response"}
normalized = dict(result)
normalized["ok"] = bool(result.get("ok"))
normalized["deduplicated"] = bool(result.get("deduplicated"))
normalized["version"] = int(result.get("version") or 0)
normalized["state"] = str(result.get("state") or ("sent" if normalized["ok"] else "retry"))
return normalized
def _backoff(self, attempts: int) -> float:
exponent = max(0, int(attempts) - 1)
return min(self.max_backoff_seconds, self.base_backoff_seconds * (2 ** exponent))
async def enqueue(
self,
*,
memory_type: str,
payload: dict[str, Any],
idempotency_key: str,
occurred_at: str,
now: float | None = None,
version: int = 1,
source_refs: list[str] | None = None,
sender: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]] | None = None,
**kwargs: Any,
) -> dict[str, Any]:
"""Validate, locally record, then optionally attempt immediate delivery."""
try:
dto = build_bot_personal_dto(
memory_type=memory_type,
payload=payload,
idempotency_key=idempotency_key,
occurred_at=occurred_at,
version=version,
**kwargs,
)
except Exception as exc:
return OutboxResult(
ok=False,
state="invalid",
idempotency_key=str(idempotency_key or ""),
error_code=getattr(exc, "error_code", "invalid"),
).as_dict()
envelope = dto.envelope()
if source_refs:
envelope["source_refs"] = list(dict.fromkeys(str(item).strip()[:240] for item in source_refs if str(item).strip()))
envelope_fingerprint = _fingerprint(envelope)
current = float(self.clock() if now is None else now)
async with self._lock:
entries = self.entries
existing = self._find(entries, dto.memory_type, dto.idempotency_key)
if existing is not None:
old_version = int(existing.get("version") or 0)
old_fingerprint = str(existing.get("payload_fingerprint") or "")
if old_version > dto.version:
return OutboxResult(
ok=False,
state="stale_version",
idempotency_key=dto.idempotency_key,
record_id=str(existing.get("record_id") or dto.record_id),
version=old_version,
error_code="stale_version",
attempts=int(existing.get("attempts") or 0),
).as_dict()
if old_version == dto.version and old_fingerprint == envelope_fingerprint:
return OutboxResult(
ok=existing.get("state") in {"sent", "deduplicated"},
state=str(existing.get("state") or "pending"),
idempotency_key=dto.idempotency_key,
record_id=str(existing.get("record_id") or dto.record_id),
deduplicated=True,
version=old_version,
error_code=existing.get("last_error"),
attempts=int(existing.get("attempts") or 0),
).as_dict()
if old_version == dto.version and old_fingerprint != envelope_fingerprint:
return OutboxResult(
ok=False,
state="version_conflict",
idempotency_key=dto.idempotency_key,
record_id=str(existing.get("record_id") or dto.record_id),
version=old_version,
error_code="version_conflict",
attempts=int(existing.get("attempts") or 0),
).as_dict()
existing.update({
"envelope": deepcopy(envelope),
"payload_fingerprint": envelope_fingerprint,
"version": dto.version,
"state": "pending",
"attempts": 0,
"next_attempt_at": current,
"last_error": "",
"updated_at": _now_iso(),
"dead_letter_at": "",
})
entry = existing
else:
entry = {
"outbox_id": f"outbox_{hashlib.sha1(self._key(dto.memory_type, dto.idempotency_key).encode()).hexdigest()[:20]}",
"memory_type": dto.memory_type,
"idempotency_key": dto.idempotency_key,
"record_id": dto.record_id,
"version": dto.version,
"envelope": deepcopy(envelope),
"payload_fingerprint": envelope_fingerprint,
"state": "pending",
"attempts": 0,
"next_attempt_at": current,
"last_error": "",
"created_at": _now_iso(),
"updated_at": _now_iso(),
"sent_at": "",
"dead_letter_at": "",
}
entries.append(entry)
self._replace_entries(entries)
self._persist()
result = OutboxResult(
ok=False,
state="pending",
idempotency_key=dto.idempotency_key,
record_id=dto.record_id,
version=dto.version,
).as_dict()
if callable(sender):
drained = await self.drain(sender, now=current, limit=1)
if drained:
return drained[0]
return result
async def drain(
self,
sender: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]],
*,
now: float | None = None,
limit: int = 16,
) -> list[dict[str, Any]]:
"""Try due entries once; retries are scheduled without sleeping."""
current = float(self.clock() if now is None else now)
results: list[dict[str, Any]] = []
for _ in range(max(0, int(limit or 0))):
async with self._lock:
due = [
item for item in self.entries
if item.get("state") in {"pending", "retry"}
and _timestamp(item.get("next_attempt_at"), current) <= current
]
if not due:
break
entry = due[0]
entry["attempts"] = int(entry.get("attempts") or 0) + 1
entry["updated_at"] = _now_iso()
envelope = deepcopy(entry.get("envelope") or {})
attempt = int(entry["attempts"])
self._persist()
try:
response = self._normalise_sender_result(await sender(envelope))
except Exception as exc:
response = {"ok": False, "state": "retry", "error_code": type(exc).__name__}
async with self._lock:
current_entry = self._find(self.entries, entry.get("memory_type", ""), entry.get("idempotency_key", ""))
if current_entry is None:
continue
current_entry["updated_at"] = _now_iso()
current_entry["last_error"] = str(response.get("error_code") or "")[:120]
current_entry["remote_record_id"] = str(response.get("record_id") or "")[:160]
current_entry["remote_version"] = int(response.get("version") or 0)
if response.get("ok"):
current_entry["state"] = "deduplicated" if response.get("deduplicated") else "sent"
current_entry["sent_at"] = _now_iso()
current_entry["next_attempt_at"] = 0
elif response.get("state") in {"invalid", "version_conflict", "stale_version"}:
current_entry["state"] = "dead_letter" if response.get("state") != "invalid" else "invalid"
current_entry["dead_letter_at"] = _now_iso()
current_entry["next_attempt_at"] = 0
elif attempt >= self.max_attempts:
current_entry["state"] = "dead_letter"
current_entry["dead_letter_at"] = _now_iso()
current_entry["next_attempt_at"] = 0
else:
current_entry["state"] = "retry"
current_entry["next_attempt_at"] = current + self._backoff(attempt)
self._persist()
results.append({
"ok": bool(response.get("ok")),
"state": current_entry["state"],
"idempotency_key": current_entry.get("idempotency_key", ""),
"record_id": current_entry.get("remote_record_id") or current_entry.get("record_id", ""),
"deduplicated": bool(response.get("deduplicated")),
"version": int(response.get("version") or current_entry.get("version") or 0),
"error_code": current_entry.get("last_error") or None,
"attempts": attempt,
})
return results
async def retry_dead_letter(self, idempotency_key: str, *, now: float | None = None) -> dict[str, Any]:
current = float(self.clock() if now is None else now)
async with self._lock:
for entry in self.entries:
if entry.get("idempotency_key") != idempotency_key:
continue
if entry.get("state") != "dead_letter":
return {"ok": False, "state": str(entry.get("state") or "pending"), "idempotency_key": idempotency_key}
entry.update({"state": "retry", "next_attempt_at": current, "last_error": "", "dead_letter_at": "", "updated_at": _now_iso()})
self._persist()
return {"ok": True, "state": "retry", "idempotency_key": idempotency_key, "attempts": int(entry.get("attempts") or 0)}
return {"ok": False, "state": "missing", "idempotency_key": idempotency_key, "error_code": "not_found"}
def status(self) -> dict[str, Any]:
entries = self.entries
counts = {state: 0 for state in OUTBOX_STATES}
for entry in entries:
state = str(entry.get("state") or "pending")
counts[state] = counts.get(state, 0) + 1
return {"total": len(entries), "counts": counts, "pending": counts.get("pending", 0), "retry": counts.get("retry", 0), "sent": counts.get("sent", 0) + counts.get("deduplicated", 0), "dead_letter": counts.get("dead_letter", 0)}
__all__ = ["BotPersonalOutbox", "OutboxResult", "OUTBOX_STATES"]