diff --git a/docs/运维陪伴错题本.md b/docs/运维陪伴错题本.md index 84e6149..925199d 100644 --- a/docs/运维陪伴错题本.md +++ b/docs/运维陪伴错题本.md @@ -601,3 +601,13 @@ docs\运维陪伴错题本.md - 修复或规避:删掉两个 `_update_*` 里的无条件写入,把锚点所有权交还给 `_decay_*`。不是加「命中分支才推进」的条件——`apology`/`comfort` 这些分支本身在**降**情绪,更没有理由重置衰减时钟。核对过消费面:全仓只有两个 `_decay_*` 读这个字段,页面摘要与前端都不读它,受伤余波的硬闸门走的是另外三个独立时间戳,不受影响。 - 预防规则:一个时间戳字段只能有一种语义。「最后修改时间」和「结算锚点」看起来都叫 updated_ts,但前者可以随便推进,后者推进一次就丢掉一段没被消耗的时间。共享锚点的多维状态尤其危险:任一维的写入都会连累其余维度,而症状只在「某几维不衰减」上体现,很难联想到锚点。 - 验证证据:`tests/test_audit_batch1.py::B2EmotionUpdateTimestampTests` 4 项(高频对话仍衰减、分步与静默一小时严格等价、正向事件不冻结负向维度、单次事件不吞掉不足一格的时间)。反证:临时还原成缺陷版,这 4 项全红。全量 235 项通过。 + +## 2026-07-27 — 清理写在 return 旁边,就一定漏掉异常和取消 + +- 阶段:P0 生产缺陷清理(B7) +- 类型:资源泄漏 +- 现象:识图会把 http(s) 图片源下载落盘到 `private_inbound_images//`,唯一的删除入口是 `_cleanup_prepared_image_sources`。私聊侧六处清理调用**全部包在 `if group_mode:` 里**(`group_mode = clean_cache_scope == "group_image"`),只有群聊识图会清;forward 侧复用了准备函数却没复制配套的清理调用,四条 return 全泄漏。单文件上限 12 MB,随消息量无界增长。 +- 根因:清理责任被绑在「各条 return 语句旁边」,而不是绑在临时文件的生命周期上。于是出现三重漏洞:被业务开关误当作条件、复用者只抄了申请没抄释放、逐 return 点手工清理天生覆盖不到异常与取消路径——识图协程被 `asyncio.wait_for` 超时取消时抛的是 `CancelledError`,`except Exception` 捕不到,六处清理全部落空(群聊也一样漏)。 +- 修复或规避:三条防线各管一段。①去掉六处的 `group_mode` 门控——清理是临时文件的生命周期,不是群聊功能的一部分;②forward 侧拆出内层函数、外层 `try/finally` 收口,正常与异常路径都清;③**准备新图之前先按存活时长清扫该 namespace**(`PREPARED_IMAGE_MAX_AGE_SECONDS = 30 分钟`),这一条不依赖任何一条 return 走到,覆盖异常与取消。没有采用「把 260 行函数整体包进 try/finally」的方案:那需要整体改缩进,风险大于收益,而按时长清扫对取消路径的覆盖反而更彻底。 +- 预防规则:临时资源的释放要绑在申请点的同一个作用域(`try/finally`),不能散落在下游各个 return 分支。只要清理逻辑出现在 `if <业务条件>:` 里面,就该警觉——释放不该有业务条件。此外,凡是可能被 `wait_for` 取消的协程,`except Exception` 都不算兜底,必须另有一条不依赖控制流的回收路径。 +- 验证证据:`tests/test_b7_image_temp_files.py` 8 项(按时长清扫、准备时先清扫、清理点不再被门控、forward 侧正常/异常/取消三条路径都清理)。全量 243 项通过;联合 E2E 146/146。 diff --git a/forward_message.py b/forward_message.py index c08d1ac..221d9a9 100644 --- a/forward_message.py +++ b/forward_message.py @@ -929,6 +929,18 @@ class ForwardMessageMixin: original_sources, namespace="forward_vision", ) + # 这条链路此前一次也没清理过下载下来的临时图:作者复用了 _prepare_... + # 却没有复制配套的清理调用,四条 return 全部泄漏。 + # 正常路径在 finally 里立刻清;异常与取消路径由准备阶段的按时长清扫兜住。 + try: + return await self._transcribe_forward_message_images_inner( + event, sources, original_sources=original_sources) + finally: + self._cleanup_prepared_image_sources(sources, namespace="forward_vision") + + async def _transcribe_forward_message_images_inner( + self, event, sources: list[str], *, original_sources: list[str] + ) -> str: if not sources: logger.info( "[PrivateCompanion] 合并/引用图片视觉跳过: 图片源无法转为模型可读源 original=%s", diff --git a/private_image.py b/private_image.py index 66fb454..fd23418 100644 --- a/private_image.py +++ b/private_image.py @@ -32,6 +32,10 @@ from astrbot.core.utils.astrbot_path import get_astrbot_data_path from .helpers import _missing_optional_model_dependency, _now_ts, _safe_float, _safe_int, _single_line, _strip_internal_message_blocks, _today_key, _url_host_is_public from .segmented_message import split_plain_component_chain +# 识图临时文件的存活上限。单次识图(含重试与多 provider 回退)远低于这个数量级, +# 超过还留着的一定是被异常或取消绕过了清理的残留。 +PREPARED_IMAGE_MAX_AGE_SECONDS = 30 * 60 + class _PublicOnlyRedirectHandler(urllib.request.HTTPRedirectHandler): """跟随重定向前重新校验目标地址,避免公网 URL 302 跳到内网(SSRF)。""" @@ -344,6 +348,10 @@ class PrivateImageMixin: target_dir.mkdir(parents=True, exist_ok=True) except Exception: return [] + # 逐个 return 点手工清理天生覆盖不到异常与取消路径(识图协程被 asyncio.wait_for + # 超时取消时,except Exception 也捕不到 CancelledError)。所以每次准备新图之前, + # 先按存活时长扫掉这个 namespace 下的陈旧残留——不依赖任何一条 return 走到。 + self._sweep_stale_prepared_image_files(target_dir) prepared: list[str] = [] now_ms = int(_now_ts() * 1000) for index, source in enumerate([str(item).strip() for item in (image_sources or []) if str(item or "").strip()][:12], 1): @@ -356,6 +364,29 @@ class PrivateImageMixin: prepared.append(source) return prepared + def _sweep_stale_prepared_image_files(self, target_dir: Path) -> int: + """扫掉 namespace 目录下超过存活上限的临时图片,返回删除条数。 + + 识图用的临时文件只在一次调用内有意义:模型侧拿到的是 data: 内联 URL + (`_private_image_source_to_model_url`),缓存预览另存在别处,所以任何 + 超过上限还留在这里的文件都已经没有读者。上限给得比单次识图耗时宽得多。 + """ + removed = 0 + try: + deadline = _now_ts() - PREPARED_IMAGE_MAX_AGE_SECONDS + for path in target_dir.iterdir(): + try: + if path.is_file() and path.stat().st_mtime < deadline: + path.unlink(missing_ok=True) + removed += 1 + except Exception: + continue + except Exception: + return removed + if removed: + logger.info("[PrivateCompanion] 已清理识图临时文件残留: dir=%s removed=%s", target_dir.name, removed) + return removed + def _cleanup_prepared_image_sources(self, sources: list[str], *, namespace: str) -> None: """Remove only temporary files downloaded into this plugin's vision namespace.""" try: @@ -2339,17 +2370,14 @@ class PrivateImageMixin: missing, _single_line(exc, 160), ) - if group_mode: - self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) - return "" - if group_mode: self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) + return "" + self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) raise image_keys = [key for key, _ in image_items] image_urls = [url for _, url in image_items] if not image_urls: - if group_mode: - self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) + self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) return "" refresher = getattr(self, "_refresh_default_persona_prompt", None) if not group_mode and callable(refresher): @@ -2469,8 +2497,7 @@ class PrivateImageMixin: ownership_line or "无", _single_line(cached_text, 220), ) - if group_mode: - self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) + self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) return cached_text if not self._can_run_llm_task(provider_id, task=clean_task_name): self._record_llm_budget_skip(provider_id=provider_id, task=clean_task_name, prompt=prompt) @@ -2552,8 +2579,7 @@ class PrivateImageMixin: scope=scope, preview=self._private_image_cache_preview_from_sources(cache_key, [*original_sources, *sources]), ) - if group_mode: - self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) + self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) return cleaned_text except asyncio.TimeoutError: elapsed_ms = int((time.time() - start) * 1000) if "start" in locals() else 0 @@ -2594,8 +2620,7 @@ class PrivateImageMixin: ) self._mark_private_image_provider_failure(provider_id, provider_source, exc, task=clean_task_name) continue - if group_mode: - self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) + self._cleanup_prepared_image_sources(sources, namespace=clean_namespace) logger.info("[PrivateCompanion] %s视觉转述失败: 所有候选 provider 均不可用或失败 attempts=%s", clean_log_subject, attempts) return "" diff --git a/tests/test_b7_image_temp_files.py b/tests/test_b7_image_temp_files.py new file mode 100644 index 0000000..391b47f --- /dev/null +++ b/tests/test_b7_image_temp_files.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +"""B7 识图临时文件泄漏回归。 + +`_prepare_private_image_sources_for_model` 会把 http(s) 图片源下载落盘到 +`/private_inbound_images//`。这些是纯临时文件,模型侧拿到的是 +data: 内联 URL,缓存预览另存在别处,所以它们没有任何长期读者。 + +三重泄漏: +1. 私聊侧六处清理调用全部包在 `if group_mode:` 里,只有群聊识图会清; +2. forward 侧复用了准备函数却没有复制配套的清理调用,四条 return 全泄漏; +3. 就算走到清理,逐 return 点手工清理也覆盖不到异常与取消路径——识图协程被 + `asyncio.wait_for` 超时取消时抛的是 `CancelledError`,`except Exception` 捕不到。 + +对应三条防线:去掉门控、forward 侧 try/finally 收口、准备阶段按存活时长清扫。 +""" +from __future__ import annotations + +import asyncio +import os +import pathlib +import re +import sys +import tempfile +import time +import types +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import _host_stub # noqa: E402 + +_host_stub.install() + +_package_name = "_b7_companion" +_package = types.ModuleType(_package_name) +_package.__path__ = [str(pathlib.Path(__file__).resolve().parents[1])] +sys.modules.setdefault(_package_name, _package) + +private_image = __import__(f"{_package_name}.private_image", fromlist=["private_image"]) +forward_message = __import__(f"{_package_name}.forward_message", fromlist=["forward_message"]) + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +class Host(private_image.PrivateImageMixin, forward_message.ForwardMessageMixin): + def __init__(self, data_dir): + self.data_dir = str(data_dir) + + +class SweepStaleFilesTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.host = Host(self.tmp.name) + self.dir = pathlib.Path(self.tmp.name) / "private_inbound_images" / "private_vision" + self.dir.mkdir(parents=True) + + def make(self, name: str, age_seconds: float) -> pathlib.Path: + path = self.dir / name + path.write_bytes(b"x") + stamp = time.time() - age_seconds + os.utime(path, (stamp, stamp)) + return path + + def test_stale_files_are_removed_and_fresh_ones_kept(self): + limit = private_image.PREPARED_IMAGE_MAX_AGE_SECONDS + stale = self.make("old.png", limit + 60) + fresh = self.make("new.png", 5) + removed = self.host._sweep_stale_prepared_image_files(self.dir) + self.assertEqual(removed, 1) + self.assertFalse(stale.exists()) + self.assertTrue(fresh.exists(), "还在用的临时文件被误删了") + + def test_sweeping_an_empty_or_missing_dir_is_harmless(self): + self.assertEqual(self.host._sweep_stale_prepared_image_files(self.dir), 0) + self.assertEqual(self.host._sweep_stale_prepared_image_files(self.dir / "不存在"), 0) + + def test_preparing_new_images_sweeps_leftovers_first(self): + """这是覆盖异常与取消路径的那道防线:不依赖任何一条 return 走到。""" + stale = self.make("leaked.png", private_image.PREPARED_IMAGE_MAX_AGE_SECONDS + 10) + asyncio.run(self.host._prepare_private_image_sources_for_model([], namespace="private_vision")) + self.assertFalse(stale.exists(), "准备新图时没有清扫上一轮泄漏的残留") + + +class CleanupIsNotGatedTests(unittest.TestCase): + def test_cleanup_calls_are_no_longer_behind_group_mode(self): + """清理是临时文件的生命周期,不是群聊功能的一部分。""" + source = (ROOT / "private_image.py").read_text(encoding="utf-8") + offenders = [] + lines = source.splitlines() + for index, line in enumerate(lines): + if "_cleanup_prepared_image_sources(sources" not in line: + continue + previous = lines[index - 1].strip() if index else "" + if previous.startswith("if group_mode"): + offenders.append(index + 1) + self.assertEqual(offenders, [], f"仍有清理点被 group_mode 门控: {offenders}") + + def test_every_cleanup_site_still_exists(self): + source = (ROOT / "private_image.py").read_text(encoding="utf-8") + self.assertGreaterEqual(source.count("_cleanup_prepared_image_sources(sources"), 6) + + +class ForwardPathCleanupTests(unittest.IsolatedAsyncioTestCase): + def build(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + host = Host(tmp.name) + host.forward_message_image_vision = True + host.forward_message_image_limit = 4 + cleaned: list[tuple] = [] + + async def prepare(sources, *, namespace="vision"): + return ["/tmp/fake-a.png", "/tmp/fake-b.png"] + + def cleanup(sources, *, namespace): + cleaned.append((tuple(sources), namespace)) + + host._prepare_private_image_sources_for_model = prepare + host._cleanup_prepared_image_sources = cleanup + return host, cleaned + + async def test_cleanup_runs_on_the_normal_path(self): + host, cleaned = self.build() + + async def inner(event, sources, *, original_sources): + return "识图结果" + + host._transcribe_forward_message_images_inner = inner + result = await host._transcribe_forward_message_images(object(), ["https://x/a.png"]) + self.assertEqual(result, "识图结果") + self.assertEqual(cleaned, [(("/tmp/fake-a.png", "/tmp/fake-b.png"), "forward_vision")]) + + async def test_cleanup_runs_even_when_the_pipeline_raises(self): + host, cleaned = self.build() + + async def inner(event, sources, *, original_sources): + raise RuntimeError("provider 炸了") + + host._transcribe_forward_message_images_inner = inner + with self.assertRaises(RuntimeError): + await host._transcribe_forward_message_images(object(), ["https://x/a.png"]) + self.assertEqual(len(cleaned), 1, "异常路径没有清理临时文件") + + async def test_cleanup_runs_when_the_pipeline_is_cancelled(self): + """asyncio.wait_for 超时取消抛的是 CancelledError,except Exception 捕不到。""" + host, cleaned = self.build() + + async def inner(event, sources, *, original_sources): + raise asyncio.CancelledError() + + host._transcribe_forward_message_images_inner = inner + with self.assertRaises(asyncio.CancelledError): + await host._transcribe_forward_message_images(object(), ["https://x/a.png"]) + self.assertEqual(len(cleaned), 1, "取消路径没有清理临时文件") + + +if __name__ == "__main__": + unittest.main()