525 行
20 KiB
Python
525 行
20 KiB
Python
"""Deterministic interpretation of conversational workflow candidate feedback.
|
||
|
||
The interpreter deliberately does not query a model and never treats a
|
||
candidate identifier as trustworthy by itself. Every user- or model-supplied
|
||
display code is resolved through the candidate whitelist owned by the caller.
|
||
This makes the module suitable for chat commands without allowing an LLM to
|
||
invent candidates from another round or workflow.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Mapping, Sequence
|
||
import re
|
||
import unicodedata
|
||
from typing import Any, Literal
|
||
|
||
from backend.dto.workflow_feedback_dto import (
|
||
WorkflowFeedbackCandidateRef,
|
||
WorkflowFeedbackInterpretation,
|
||
)
|
||
|
||
|
||
FeedbackAction = Literal["liked", "rejected", "strict_rejected", "final"]
|
||
|
||
|
||
class WorkflowFeedbackInterpretationError(ValueError):
|
||
"""Raised when feedback cannot be resolved safely and deterministically."""
|
||
|
||
def __init__(self, reason: str, message: str, details: Mapping[str, Any] | None = None) -> None:
|
||
super().__init__(message)
|
||
self.reason = reason
|
||
self.details = dict(details or {})
|
||
|
||
|
||
_ACTION_RE = re.compile(
|
||
r"(?P<strict_rejected>严格拒绝|强烈拒绝|彻底拒绝|\b(?:strict|hard)\s+reject(?:ed)?\b)"
|
||
r"|(?P<rejected>不喜欢|不要|拒绝|淘汰|排除|删除|"
|
||
r"\b(?:reject(?:ed)?|discard(?:ed)?|drop(?:ped)?|remove(?:d)?|exclude(?:d)?)\b)"
|
||
r"|(?P<final>最终采用|确认采用|采用|定稿|选定|就用|"
|
||
r"\b(?:adopt(?:ed)?|final(?:ize|ise|ized|ised)?|select(?:ed)?|use)\b)"
|
||
r"|(?P<liked>保留|喜欢|偏好|可用|"
|
||
r"\b(?:keep|like(?:d)?|prefer(?:red)?|approve(?:d)?)\b)",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
_DISPLAY_CODE_OR_NUMBER_RE = re.compile(
|
||
r"(?<![A-Za-z0-9])"
|
||
r"(?:"
|
||
r"(?=[A-Za-z0-9-]*\d)[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z0-9]+)+"
|
||
r"|\d{1,3}"
|
||
r")"
|
||
r"(?![A-Za-z0-9])",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
_CHINESE_ORDINAL_RE = re.compile(
|
||
r"(?:候选|第)(?P<prefixed>[一二三四五六七八九十])(?:张|号)?"
|
||
r"|(?P<suffixed>[一二三四五六七八九十])(?:张|号)",
|
||
)
|
||
|
||
_NEXT_ROUND_MARKER_RE = re.compile(
|
||
r"(?:下一轮|下轮|下一版|next\s+(?:round|batch))"
|
||
r"(?:\s*(?:提示词|prompt(?:\s+patch)?))?",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
_MATERIAL_MARKER_RE = re.compile(
|
||
r"(?:素材(?:意图|要求)?|参考素材|material(?:\s+intent)?|reference\s+material)",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
_CHINESE_NUMBERS = {
|
||
"一": 1,
|
||
"二": 2,
|
||
"三": 3,
|
||
"四": 4,
|
||
"五": 5,
|
||
"六": 6,
|
||
"七": 7,
|
||
"八": 8,
|
||
"九": 9,
|
||
"十": 10,
|
||
}
|
||
|
||
|
||
class _CandidateWhitelist:
|
||
def __init__(self, candidates: Mapping[str, str]) -> None:
|
||
self._exact: dict[str, WorkflowFeedbackCandidateRef] = {}
|
||
self._numeric_aliases: dict[str, list[WorkflowFeedbackCandidateRef]] = {}
|
||
for display_code, candidate_id in candidates.items():
|
||
code = str(display_code or "").strip()
|
||
identifier = str(candidate_id or "").strip()
|
||
if not code or not identifier:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_candidate_whitelist",
|
||
"candidate whitelist entries require display_code and candidate_id",
|
||
{"display_code": code, "candidate_id": identifier},
|
||
)
|
||
normalized_code = _normalize_display_code(code)
|
||
current = self._exact.get(normalized_code)
|
||
if current and current.candidate_id != identifier:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_candidate_whitelist",
|
||
"candidate whitelist contains duplicate display codes",
|
||
{"display_code": code},
|
||
)
|
||
reference = current or WorkflowFeedbackCandidateRef(
|
||
candidate_id=identifier,
|
||
display_code=code,
|
||
)
|
||
self._exact[normalized_code] = reference
|
||
terminal_index = _terminal_candidate_index(normalized_code)
|
||
if terminal_index is not None:
|
||
alias = str(terminal_index)
|
||
aliases = self._numeric_aliases.setdefault(alias, [])
|
||
if all(item.candidate_id != reference.candidate_id for item in aliases):
|
||
aliases.append(reference)
|
||
|
||
@property
|
||
def allowed_display_codes(self) -> tuple[str, ...]:
|
||
return tuple(reference.display_code for reference in self._exact.values())
|
||
|
||
def resolve(self, raw_token: str) -> WorkflowFeedbackCandidateRef:
|
||
token = _normalize_reference_token(raw_token)
|
||
exact = self._exact.get(token)
|
||
if exact:
|
||
return exact
|
||
if token.isdigit():
|
||
aliases = self._numeric_aliases.get(str(int(token)), [])
|
||
if len(aliases) == 1:
|
||
return aliases[0]
|
||
if len(aliases) > 1:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"ambiguous_candidate_reference",
|
||
f"candidate reference {raw_token!r} is ambiguous",
|
||
{
|
||
"reference": raw_token,
|
||
"matching_display_codes": [item.display_code for item in aliases],
|
||
},
|
||
)
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"unknown_candidate_reference",
|
||
f"candidate reference {raw_token!r} is not in the current whitelist",
|
||
{
|
||
"reference": raw_token,
|
||
"allowed_display_codes": list(self.allowed_display_codes),
|
||
},
|
||
)
|
||
|
||
|
||
class WorkflowFeedbackInterpreter:
|
||
"""Interpret chat feedback and validate optional LLM-structured output."""
|
||
|
||
def interpret(
|
||
self,
|
||
text: str,
|
||
*,
|
||
candidate_whitelist: Mapping[str, str],
|
||
) -> WorkflowFeedbackInterpretation:
|
||
source_text = str(text or "").strip()
|
||
normalized_text = _normalize_text(source_text)
|
||
candidates = _CandidateWhitelist(candidate_whitelist)
|
||
actions: dict[FeedbackAction, list[WorkflowFeedbackCandidateRef]] = {
|
||
"liked": [],
|
||
"rejected": [],
|
||
"strict_rejected": [],
|
||
"final": [],
|
||
}
|
||
|
||
pending_tokens: list[str] = []
|
||
active_prefix_action: FeedbackAction | None = None
|
||
for fragment in _feedback_fragments(normalized_text):
|
||
if _contains_directive_marker(fragment):
|
||
pending_tokens = []
|
||
active_prefix_action = None
|
||
continue
|
||
action_matches = list(_ACTION_RE.finditer(fragment))
|
||
candidate_tokens = _candidate_tokens(fragment)
|
||
if not action_matches:
|
||
if candidate_tokens:
|
||
if active_prefix_action:
|
||
self._append_resolved(
|
||
actions,
|
||
active_prefix_action,
|
||
candidate_tokens,
|
||
candidates,
|
||
)
|
||
else:
|
||
pending_tokens.extend(candidate_tokens)
|
||
continue
|
||
if len(action_matches) > 1:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"ambiguous_feedback_clause",
|
||
"one feedback clause contains multiple actions; split it with punctuation",
|
||
{"clause": fragment},
|
||
)
|
||
|
||
action_match = action_matches[0]
|
||
action = _action_from_match(action_match)
|
||
first_candidate_start = _first_candidate_start(fragment)
|
||
action_is_prefix = first_candidate_start is None or action_match.start() <= first_candidate_start
|
||
tokens = [*pending_tokens, *candidate_tokens]
|
||
pending_tokens = []
|
||
if not tokens:
|
||
active_prefix_action = None
|
||
continue
|
||
self._append_resolved(actions, action, tokens, candidates)
|
||
active_prefix_action = action if action_is_prefix else None
|
||
|
||
material_intent, next_round_prompt_patch = _extract_directives(normalized_text)
|
||
_validate_action_conflicts(actions)
|
||
final_candidates = actions["final"]
|
||
if len(final_candidates) > 1:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"multiple_final_candidates",
|
||
"only one candidate can be marked final",
|
||
{"display_codes": [candidate.display_code for candidate in final_candidates]},
|
||
)
|
||
return WorkflowFeedbackInterpretation(
|
||
liked=tuple(actions["liked"]),
|
||
rejected=tuple(actions["rejected"]),
|
||
strict_rejected=tuple(actions["strict_rejected"]),
|
||
final=final_candidates[0] if final_candidates else None,
|
||
material_intent=material_intent,
|
||
next_round_prompt_patch=next_round_prompt_patch,
|
||
source="deterministic",
|
||
source_text=source_text,
|
||
)
|
||
|
||
def validate_llm_result(
|
||
self,
|
||
payload: Mapping[str, Any],
|
||
*,
|
||
candidate_whitelist: Mapping[str, str],
|
||
source_text: str = "",
|
||
) -> WorkflowFeedbackInterpretation:
|
||
"""Validate offline LLM output without trusting its candidate IDs.
|
||
|
||
Candidate fields must contain display codes (or objects with a
|
||
``display_code`` key). A raw ``candidate_id`` is intentionally not
|
||
accepted because it bypasses the caller-owned round whitelist.
|
||
"""
|
||
|
||
if not isinstance(payload, Mapping):
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_llm_result",
|
||
"LLM feedback result must be an object",
|
||
)
|
||
_reject_llm_candidate_ids(payload)
|
||
candidates = _CandidateWhitelist(candidate_whitelist)
|
||
actions: dict[FeedbackAction, list[WorkflowFeedbackCandidateRef]] = {
|
||
"liked": [],
|
||
"rejected": [],
|
||
"strict_rejected": [],
|
||
"final": [],
|
||
}
|
||
for action in ("liked", "rejected", "strict_rejected", "final"):
|
||
tokens = _llm_display_codes(payload.get(action), field=action)
|
||
self._append_resolved(actions, action, tokens, candidates)
|
||
|
||
_validate_action_conflicts(actions)
|
||
final_candidates = actions["final"]
|
||
if len(final_candidates) > 1:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"multiple_final_candidates",
|
||
"only one candidate can be marked final",
|
||
{"display_codes": [candidate.display_code for candidate in final_candidates]},
|
||
)
|
||
positive_patch = _optional_text(
|
||
payload.get("positive_patch") or payload.get("next_round_prompt_patch"),
|
||
field="positive_patch",
|
||
)
|
||
return WorkflowFeedbackInterpretation(
|
||
liked=tuple(actions["liked"]),
|
||
rejected=tuple(actions["rejected"]),
|
||
strict_rejected=tuple(actions["strict_rejected"]),
|
||
final=final_candidates[0] if final_candidates else None,
|
||
material_intent=_optional_text(payload.get("material_intent"), field="material_intent"),
|
||
next_round_prompt_patch=_optional_text(
|
||
payload.get("next_round_prompt_patch") or payload.get("positive_patch"),
|
||
field="next_round_prompt_patch",
|
||
),
|
||
positive_patch=positive_patch,
|
||
negative_patch=_optional_text(payload.get("negative_patch"), field="negative_patch"),
|
||
source="llm_validated",
|
||
source_text=str(source_text or "").strip(),
|
||
)
|
||
|
||
@staticmethod
|
||
def _append_resolved(
|
||
actions: dict[FeedbackAction, list[WorkflowFeedbackCandidateRef]],
|
||
action: FeedbackAction,
|
||
tokens: Sequence[str],
|
||
candidates: _CandidateWhitelist,
|
||
) -> None:
|
||
target = actions[action]
|
||
known_ids = {candidate.candidate_id for candidate in target}
|
||
for token in tokens:
|
||
candidate = candidates.resolve(token)
|
||
if candidate.candidate_id not in known_ids:
|
||
target.append(candidate)
|
||
known_ids.add(candidate.candidate_id)
|
||
|
||
|
||
def interpret_workflow_feedback(
|
||
text: str,
|
||
*,
|
||
candidate_whitelist: Mapping[str, str],
|
||
) -> WorkflowFeedbackInterpretation:
|
||
"""Convenience entry point for deterministic interpretation."""
|
||
|
||
return WorkflowFeedbackInterpreter().interpret(
|
||
text,
|
||
candidate_whitelist=candidate_whitelist,
|
||
)
|
||
|
||
|
||
def _normalize_text(value: str) -> str:
|
||
return unicodedata.normalize("NFKC", value).strip()
|
||
|
||
|
||
def _normalize_display_code(value: str) -> str:
|
||
return _normalize_text(value).upper()
|
||
|
||
|
||
def _normalize_reference_token(value: str) -> str:
|
||
token = _normalize_text(value).strip("#[](){}<>《》【】 ")
|
||
token = re.sub(r"^(?:候选|第|candidate\s*#?|no\.?\s*)", "", token, flags=re.IGNORECASE)
|
||
token = re.sub(r"(?:张|号)$", "", token)
|
||
if token in _CHINESE_NUMBERS:
|
||
token = str(_CHINESE_NUMBERS[token])
|
||
return token.upper()
|
||
|
||
|
||
def _terminal_candidate_index(display_code: str) -> int | None:
|
||
match = re.search(r"(?:^|[-_])(?:[A-Z]*)(\d+)$", display_code, flags=re.IGNORECASE)
|
||
if not match:
|
||
return int(display_code) if display_code.isdigit() else None
|
||
return int(match.group(1))
|
||
|
||
|
||
def _feedback_fragments(text: str) -> tuple[str, ...]:
|
||
return tuple(fragment.strip() for fragment in re.split(r"[,,;;。\n]+", text) if fragment.strip())
|
||
|
||
|
||
def _candidate_token_matches(fragment: str) -> list[tuple[int, int, str]]:
|
||
matches = [
|
||
(match.start(), match.end(), match.group(0))
|
||
for match in _DISPLAY_CODE_OR_NUMBER_RE.finditer(fragment)
|
||
]
|
||
for match in _CHINESE_ORDINAL_RE.finditer(fragment):
|
||
value = match.group("prefixed") or match.group("suffixed")
|
||
matches.append((match.start(), match.end(), str(_CHINESE_NUMBERS[value])))
|
||
matches.sort(key=lambda item: (item[0], -(item[1] - item[0])))
|
||
deduplicated: list[tuple[int, int, str]] = []
|
||
for item in matches:
|
||
if any(item[0] < current[1] and item[1] > current[0] for current in deduplicated):
|
||
continue
|
||
deduplicated.append(item)
|
||
return deduplicated
|
||
|
||
|
||
def _candidate_tokens(fragment: str) -> list[str]:
|
||
return [item[2] for item in _candidate_token_matches(fragment)]
|
||
|
||
|
||
def _first_candidate_start(fragment: str) -> int | None:
|
||
matches = _candidate_token_matches(fragment)
|
||
return matches[0][0] if matches else None
|
||
|
||
|
||
def _action_from_match(match: re.Match[str]) -> FeedbackAction:
|
||
for action in ("strict_rejected", "rejected", "final", "liked"):
|
||
if match.group(action) is not None:
|
||
return action # type: ignore[return-value]
|
||
raise AssertionError("feedback action regex produced no named group")
|
||
|
||
|
||
def _contains_directive_marker(fragment: str) -> bool:
|
||
return bool(_NEXT_ROUND_MARKER_RE.search(fragment) or _MATERIAL_MARKER_RE.search(fragment))
|
||
|
||
|
||
def _extract_directives(text: str) -> tuple[str | None, str | None]:
|
||
material_values: list[str] = []
|
||
next_round_values: list[str] = []
|
||
for fragment in _feedback_fragments(text):
|
||
markers: list[tuple[int, int, Literal["material", "next_round"]]] = []
|
||
markers.extend(
|
||
(match.start(), match.end(), "material")
|
||
for match in _MATERIAL_MARKER_RE.finditer(fragment)
|
||
)
|
||
markers.extend(
|
||
(match.start(), match.end(), "next_round")
|
||
for match in _NEXT_ROUND_MARKER_RE.finditer(fragment)
|
||
)
|
||
markers.sort(key=lambda item: item[0])
|
||
for index, (_, marker_end, kind) in enumerate(markers):
|
||
value_end = markers[index + 1][0] if index + 1 < len(markers) else len(fragment)
|
||
value = fragment[marker_end:value_end]
|
||
value = _clean_directive_value(value, kind=kind)
|
||
if not value:
|
||
continue
|
||
target = material_values if kind == "material" else next_round_values
|
||
if value not in target:
|
||
target.append(value)
|
||
return _join_directive_values(material_values), _join_directive_values(next_round_values)
|
||
|
||
|
||
def _clean_directive_value(value: str, *, kind: Literal["material", "next_round"]) -> str:
|
||
cleaned = value.strip(" \t::=-")
|
||
if kind == "material":
|
||
cleaned = re.sub(r"^(?:请|要|用|使用|选择|改成|设为)\s*", "", cleaned, flags=re.IGNORECASE)
|
||
else:
|
||
cleaned = re.sub(r"^(?:请|要|改成|调整为|设为)\s*", "", cleaned, flags=re.IGNORECASE)
|
||
cleaned = re.sub(r"(?:并且|以及|和|and)\s*$", "", cleaned, flags=re.IGNORECASE)
|
||
return cleaned.strip(" \t::=-") or ""
|
||
|
||
|
||
def _join_directive_values(values: Sequence[str]) -> str | None:
|
||
return ";".join(values) if values else None
|
||
|
||
|
||
def _llm_display_codes(value: Any, *, field: str) -> list[str]:
|
||
if value is None or value == "":
|
||
return []
|
||
items: Sequence[Any]
|
||
if isinstance(value, (str, Mapping)):
|
||
items = [value]
|
||
elif isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)):
|
||
items = value
|
||
else:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_llm_result",
|
||
f"LLM field {field!r} must contain display codes",
|
||
{"field": field},
|
||
)
|
||
display_codes: list[str] = []
|
||
for item in items:
|
||
if isinstance(item, str):
|
||
display_code = item.strip()
|
||
elif isinstance(item, Mapping):
|
||
raw_display_code = item.get("display_code", item.get("code"))
|
||
display_code = raw_display_code.strip() if isinstance(raw_display_code, str) else ""
|
||
else:
|
||
display_code = ""
|
||
if not display_code:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_llm_result",
|
||
f"LLM field {field!r} contains an item without display_code",
|
||
{"field": field},
|
||
)
|
||
display_codes.append(display_code)
|
||
return display_codes
|
||
|
||
|
||
def _reject_llm_candidate_ids(value: Any, *, path: str = "$") -> None:
|
||
"""Reject model output that tries to bypass display-code resolution.
|
||
|
||
Candidate IDs are server-owned opaque identifiers. Even when an object
|
||
also contains a valid display code, accepting the ID would make the
|
||
contract ambiguous and encourages callers to trust model-invented IDs.
|
||
"""
|
||
|
||
if isinstance(value, Mapping):
|
||
for key, item in value.items():
|
||
normalized = str(key).strip().lower()
|
||
if normalized in {"candidate_id", "candidate_ids"}:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_llm_result",
|
||
"LLM feedback must reference candidates by display_code only",
|
||
{"field": f"{path}.{key}", "reason": "llm_candidate_id_forbidden"},
|
||
)
|
||
_reject_llm_candidate_ids(item, path=f"{path}.{key}")
|
||
elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||
for index, item in enumerate(value):
|
||
_reject_llm_candidate_ids(item, path=f"{path}[{index}]")
|
||
|
||
|
||
def _optional_text(value: Any, *, field: str) -> str | None:
|
||
if value is None:
|
||
return None
|
||
if not isinstance(value, str):
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"invalid_llm_result",
|
||
f"LLM field {field!r} must be text or null",
|
||
{"field": field},
|
||
)
|
||
normalized = value.strip()
|
||
return normalized or None
|
||
|
||
|
||
def _validate_action_conflicts(
|
||
actions: Mapping[FeedbackAction, Sequence[WorkflowFeedbackCandidateRef]],
|
||
) -> None:
|
||
feedback_kinds = ("liked", "rejected", "strict_rejected")
|
||
ownership: dict[str, str] = {}
|
||
for kind in feedback_kinds:
|
||
for candidate in actions[kind]:
|
||
previous = ownership.get(candidate.candidate_id)
|
||
if previous and previous != kind:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"conflicting_candidate_feedback",
|
||
"one candidate cannot receive conflicting feedback actions",
|
||
{
|
||
"candidate_id": candidate.candidate_id,
|
||
"display_code": candidate.display_code,
|
||
"actions": [previous, kind],
|
||
},
|
||
)
|
||
ownership[candidate.candidate_id] = kind
|
||
rejected_ids = {
|
||
candidate.candidate_id
|
||
for kind in ("rejected", "strict_rejected")
|
||
for candidate in actions[kind]
|
||
}
|
||
for candidate in actions["final"]:
|
||
if candidate.candidate_id in rejected_ids:
|
||
raise WorkflowFeedbackInterpretationError(
|
||
"final_candidate_rejected",
|
||
"a rejected candidate cannot be marked final",
|
||
{
|
||
"candidate_id": candidate.candidate_id,
|
||
"display_code": candidate.display_code,
|
||
},
|
||
)
|