diff --git a/crates/desktop-protocol/src/astrbot.rs b/crates/desktop-protocol/src/astrbot.rs index 3456fe3..d777c94 100644 --- a/crates/desktop-protocol/src/astrbot.rs +++ b/crates/desktop-protocol/src/astrbot.rs @@ -269,6 +269,23 @@ struct PendingMedia { media_type: String, } +/// Longest run of media frames awaiting their `attachment_saved` partners. +/// +/// A bound rather than a design limit: the queue exists so two attachments in +/// one reply do not overwrite each other, and a server that sent media frames +/// without ever pairing them must not grow it without end. +const MAX_PENDING_MEDIA: usize = 16; + +/// The marker a message carries in place of an attachment. +/// +/// The alt text is the AstrBot frame type, so the renderer knows what kind of +/// attachment to expect before the bytes have been fetched. The bytes' own media +/// type decides the element in the end; this only shapes the placeholder and the +/// failure message. +fn media_marker(media_type: &str, reference: &str) -> String { + format!("![{media_type}](astrbot-attachment://{reference})") +} + /// Separates a message id from its reopen index. /// /// Content that arrives after `complete` reuses the server's message id, so @@ -292,7 +309,7 @@ pub struct AstrBotStreamMapper { /// and that is exactly the question a `complete` frame poses when the reply /// arrived as discrete segments. delivered: String, - pending_media: Option, + pending_media: Vec, /// True once the active message has been completed. /// /// AstrBot can deliver content *after* the `complete` frame — an observed @@ -408,59 +425,58 @@ impl AstrBotStreamMapper { } fn map_media(&mut self, frame: AstrBotStreamFrame) -> Vec { - let mut events = Vec::new(); let file_name = text_from_value(&frame.data) .as_deref() .and_then(attachment_file_name); - if let Some(message_id) = frame.message_id { - if self - .active_base_id() - .is_some_and(|current| current != message_id) - { - events = self.complete_if_needed(FinishReason::Stop); - self.accumulated.clear(); - self.delta_index = 0; - self.saw_assistant_delta = false; - self.active_message_completed = false; - } - self.active_message_id = Some(message_id); - } + // Shared with the text frames rather than a second copy of the same + // logic: the copy that used to live here also assigned the server's bare + // id back over an active id carrying a reopen suffix. + let mut events = self.adopt_message_id(frame.message_id.clone()); // The media frame carries the file name, and AstrBot's own dashboard - // fetches a chat image by name. The id in the following + // fetches a chat file by name. The id in the following // `attachment_saved` frame addresses a different resource, so waiting // for that frame both loses the name and produces an unusable marker. - if frame.frame_type == "image" - && let Some(file_name) = file_name - { - self.pending_media = None; - events.extend(self.emit_delta(format!("![image](astrbot-attachment://{file_name})"))); + // Applied to every media type, not only images: a voice reply arrives as + // a `record` frame, and gating this on `image` is what dropped it. + if let Some(file_name) = file_name { + events.extend(self.emit_delta(media_marker(&frame.frame_type, &file_name))); return events; } - self.pending_media = Some(PendingMedia { - media_type: frame.frame_type, - }); + // No usable name, so the marker has to wait for the id in + // `attachment_saved`. Queued rather than held in a single slot, which + // let a second media frame overwrite the first before it was paired. + if self.pending_media.len() < MAX_PENDING_MEDIA { + self.pending_media.push(PendingMedia { + media_type: frame.frame_type, + }); + } events } fn map_attachment_saved(&mut self, frame: AstrBotStreamFrame) -> Vec { - let Some(media) = self.pending_media.take() else { + if self.pending_media.is_empty() { return Vec::new(); - }; + } + // Read before the queue is touched, so a frame carrying no id does not + // consume the media frame that is still waiting for one. let Some(attachment_id) = frame .data .get("id") .or_else(|| frame.data.get("attachment_id")) .and_then(Value::as_str) .filter(|value| !value.trim().is_empty()) + .map(str::to_owned) else { return Vec::new(); }; - if media.media_type != "image" { - return Vec::new(); - } - self.emit_delta(format!("![image](astrbot-attachment://{attachment_id})")) + let media = self.pending_media.remove(0); + // The marker belongs on the message this frame names, which is not + // necessarily the one that happens to be open. + let mut events = self.adopt_message_id(frame.message_id.clone()); + events.extend(self.emit_delta(media_marker(&media.media_type, &attachment_id))); + events } fn map_plain(&mut self, frame: AstrBotStreamFrame) -> Vec { @@ -1003,6 +1019,93 @@ mod tests { } } + /// A voice reply reaches the client. Regression for F-33. + /// + /// `map_media` used to gate its only emitting branch on the frame being an + /// image, and the `attachment_saved` fallback rejected anything else + /// outright, so a `record` frame produced nothing anywhere — no marker, no + /// event, no error. + #[test] + fn a_record_frame_becomes_a_playable_marker() { + let events = drive(&[ + r#"data: {"type":"plain","data":"这是我的回答","streaming":true,"message_id":"m1"}"#, + r#"data: {"type":"record","data":"[RECORD]reply.wav","message_id":"m1"}"#, + ]); + + let done = completions(&events); + // One message: the attachment belongs inline with the text it came with, + // the same way an image does. A media frame arriving after `complete` + // opens its own message instead, which + // `content_after_complete_does_not_overwrite_the_finished_message` pins. + assert_eq!(done.len(), 1, "{done:?}"); + // The alt text is the frame type, which is what tells the renderer to + // expect audio before the bytes have been fetched. + assert_eq!( + done[0].1, + "这是我的回答![record](astrbot-attachment://reply.wav)" + ); + } + + #[test] + fn file_and_video_frames_are_carried_too() { + for (frame_type, reference) in [("file", "notes.pdf"), ("video", "clip.mp4")] { + let line = format!( + r#"data: {{"type":"{frame_type}","data":"[X]{reference}","message_id":"m1"}}"# + ); + let events = drive(&[line.as_str()]); + let done = completions(&events); + assert_eq!(done.len(), 1, "{frame_type}: {done:?}"); + assert_eq!( + done[0].1, + format!("![{frame_type}](astrbot-attachment://{reference})") + ); + } + } + + /// Two attachments in one reply both survive. + /// + /// `pending_media` was a single slot, so the second media frame overwrote + /// the first before its `attachment_saved` partner arrived and the first + /// attachment was lost. + #[test] + fn two_unnamed_media_frames_are_both_paired() { + let events = drive(&[ + r#"data: {"type":"image","data":"","message_id":"m1"}"#, + r#"data: {"type":"record","data":"","message_id":"m1"}"#, + r#"data: {"type":"attachment_saved","data":{"id":"att-one"},"message_id":"m1"}"#, + r#"data: {"type":"attachment_saved","data":{"id":"att-two"},"message_id":"m1"}"#, + ]); + + let markers: Vec<_> = events + .iter() + .filter_map(|event| match &event.body { + GatewayEventBody::MessageDelta(payload) => Some(payload.delta.clone()), + _ => None, + }) + .collect(); + assert_eq!( + markers, + [ + "![image](astrbot-attachment://att-one)", + "![record](astrbot-attachment://att-two)", + ] + ); + } + + /// An `attachment_saved` with no id must not consume the queued media frame. + #[test] + fn an_attachment_saved_without_an_id_leaves_the_pairing_pending() { + let events = drive(&[ + r#"data: {"type":"record","data":"","message_id":"m1"}"#, + r#"data: {"type":"attachment_saved","data":{},"message_id":"m1"}"#, + r#"data: {"type":"attachment_saved","data":{"id":"att-late"},"message_id":"m1"}"#, + ]); + + let done = completions(&events); + assert_eq!(done.len(), 1, "{done:?}"); + assert_eq!(done[0].1, "![record](astrbot-attachment://att-late)"); + } + /// The media frame names the file; the id that follows addresses something /// else. AstrBot's dashboard fetches a chat image by name, and asking the /// by-id endpoint for the `attachment_saved` id returned the dashboard's own diff --git a/src-tauri/src/networking/astrbot_client.rs b/src-tauri/src/networking/astrbot_client.rs index bcc85ef..503fbe3 100644 --- a/src-tauri/src/networking/astrbot_client.rs +++ b/src-tauri/src/networking/astrbot_client.rs @@ -587,7 +587,7 @@ impl AstrBotClient { // webview `application/octet-stream` makes `` refuse bytes that are // a perfectly good PNG. Prefer a concrete image type from the server, // fall back to what the bytes themselves say. - let sniffed = sniff_image_mime(&bytes); + let sniffed = sniff_media_mime(&bytes); // When the body is not an image it is the server explaining why, which // is diagnostic text rather than anything the user wrote. let server_note = if sniffed.is_none() { @@ -599,12 +599,12 @@ impl AstrBotClient { target: FRAME_TRACE_TARGET, status = status.as_u16(), header_mime = header_mime.as_str(), - sniffed = sniffed.unwrap_or(""), + sniffed = sniffed.unwrap_or(""), bytes = bytes.len(), server_note = server_note.as_str(), "astrbot attachment download" ); - match (header_mime.starts_with("image/"), sniffed) { + match (declares_media(&header_mime), sniffed) { (true, _) => Ok(Ok((header_mime, bytes))), (false, Some(sniffed)) => Ok(Ok((sniffed.to_owned(), bytes))), (false, None) => Ok(Err(server_note)), @@ -1030,6 +1030,70 @@ fn trace_frame_fields(value: &Value) { /// Only used when the server does not name a concrete image type. Every /// signature here is a fixed format marker at a fixed offset, so this reads no /// user content. +/// Whether a Content-Type is a media type worth trusting over the bytes. +/// +/// Deliberately not "anything that is not an error": this server serves its own +/// dashboard HTML with HTTP 200 for an unknown path, and `application/json` for +/// an error envelope, so only an explicit media type may short-circuit the +/// sniffing below. +fn declares_media(header_mime: &str) -> bool { + let header_mime = header_mime.trim().to_ascii_lowercase(); + header_mime.starts_with("image/") + || header_mime.starts_with("audio/") + || header_mime.starts_with("video/") +} + +/// Recognises the media formats an AstrBot attachment can arrive as. +/// +/// Sniffing is not a nicety here: this server answers an unknown path with its +/// own "404 Not found" text under HTTP 200, so the bytes are the only reliable +/// evidence that a candidate path produced an attachment at all. Audio matters +/// as much as image now that voice replies arrive as `record` attachments. +fn sniff_media_mime(bytes: &[u8]) -> Option<&'static str> { + if let Some(image) = sniff_image_mime(bytes) { + return Some(image); + } + sniff_audio_mime(bytes) +} + +fn sniff_audio_mime(bytes: &[u8]) -> Option<&'static str> { + // RIFF is shared with WEBP, so the form tag at byte 8 decides. Checked here + // as well as in the image sniffer because either may run first. + if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WAVE" { + return Some("audio/wav"); + } + if bytes.starts_with(b"OggS") { + return Some("audio/ogg"); + } + if bytes.starts_with(b"fLaC") { + return Some("audio/flac"); + } + if bytes.starts_with(b"ID3") { + return Some("audio/mpeg"); + } + // MPEG audio frame sync: eleven set bits, then a valid layer. + if bytes.len() >= 2 && bytes[0] == 0xFF && matches!(bytes[1] & 0xF6, 0xF2 | 0xF4 | 0xF6) { + return Some("audio/mpeg"); + } + if bytes.starts_with(b"#!AMR") { + return Some("audio/amr"); + } + // SILK, with and without the single-byte prefix the QQ and WeChat platforms + // put in front of it. AstrBot relays voice from those platforms, and a + // browser cannot decode it — recognising it is what lets the UI say so + // rather than showing an empty player. + if bytes.starts_with(b"#!SILK") || (bytes.len() > 6 && bytes[1..].starts_with(b"#!SILK")) { + return Some("audio/silk"); + } + if bytes.len() >= 12 && &bytes[4..8] == b"ftyp" { + return match &bytes[8..12] { + b"M4A " | b"M4B " => Some("audio/mp4"), + _ => Some("video/mp4"), + }; + } + None +} + fn sniff_image_mime(bytes: &[u8]) -> Option<&'static str> { const PNG: &[u8] = &[0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A]; const JPEG: &[u8] = &[0xFF, 0xD8, 0xFF]; @@ -1252,6 +1316,54 @@ mod tests { assert_eq!(sniff_image_mime(b""), None); } + /// Audio has to be recognised for the same reason images do: this server + /// answers an unknown path with its own text under HTTP 200, so the bytes are + /// the only evidence a candidate path produced an attachment. + #[test] + fn audio_bytes_are_identified_when_the_server_names_no_type() { + assert_eq!(sniff_media_mime(b"RIFF\0\0\0\0WAVEfmt "), Some("audio/wav")); + assert_eq!(sniff_media_mime(b"OggS\0\x02"), Some("audio/ogg")); + assert_eq!(sniff_media_mime(b"fLaC\0\0"), Some("audio/flac")); + assert_eq!(sniff_media_mime(b"ID3\x03\0"), Some("audio/mpeg")); + assert_eq!( + sniff_media_mime(&[0xFF, 0xFB, 0x90, 0x00]), + Some("audio/mpeg") + ); + assert_eq!(sniff_media_mime(b"#!AMR\n"), Some("audio/amr")); + assert_eq!(sniff_media_mime(b"\0\0\0\x18ftypM4A "), Some("audio/mp4")); + assert_eq!(sniff_media_mime(b"\0\0\0\x18ftypisom"), Some("video/mp4")); + + // RIFF is shared with WEBP, so the form tag decides which it is. + assert_eq!( + sniff_media_mime(b"RIFF\0\0\0\0WEBPVP8 "), + Some("image/webp") + ); + + // SILK is what AstrBot relays for QQ and WeChat voice, with and without + // the platform's one-byte prefix. Recognising it is what lets the UI say + // no browser can play it instead of rendering a dead player. + assert_eq!(sniff_media_mime(b"#!SILK_V3"), Some("audio/silk")); + assert_eq!(sniff_media_mime(b"\x02#!SILK_V3"), Some("audio/silk")); + + assert_eq!(sniff_media_mime(b""), None); + assert_eq!(sniff_media_mime(b""), None); + } + + /// Only an explicit media type may skip the sniffing. + #[test] + fn a_declared_type_is_trusted_only_when_it_names_media() { + assert!(declares_media("image/png")); + assert!(declares_media("audio/wav")); + assert!(declares_media("AUDIO/OGG")); + assert!(declares_media("video/mp4")); + + // The shapes this server actually returns for "not here". + assert!(!declares_media("text/html")); + assert!(!declares_media("application/json")); + assert!(!declares_media("application/octet-stream")); + assert!(!declares_media("")); + } + /// A block that differs from the streamed text only in line breaks is the /// same content, and must not be counted as content the stream never /// carried. Exact containment says otherwise, which is why both are logged. diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index c41277a..7b81400 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -43,7 +43,7 @@ } ], "security": { - "csp": "default-src \u0027self\u0027; connect-src ipc: http://ipc.localhost http://127.0.0.1:*; img-src \u0027self\u0027 asset: http://asset.localhost data: blob:; style-src \u0027self\u0027 \u0027unsafe-inline\u0027; script-src \u0027self\u0027" + "csp": "default-src \u0027self\u0027; connect-src ipc: http://ipc.localhost http://127.0.0.1:*; img-src \u0027self\u0027 asset: http://asset.localhost data: blob:; media-src \u0027self\u0027 asset: http://asset.localhost data: blob:; style-src \u0027self\u0027 \u0027unsafe-inline\u0027; script-src \u0027self\u0027" } }, "bundle": { diff --git a/src/features/chat/MessageContent.test.tsx b/src/features/chat/MessageContent.test.tsx index 0b82d13..c0da7a8 100644 --- a/src/features/chat/MessageContent.test.tsx +++ b/src/features/chat/MessageContent.test.tsx @@ -1,5 +1,6 @@ -import { render, screen } from "@testing-library/react"; -import { describe, expect, it, vi } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import { invoke } from "@tauri-apps/api/core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { MessageContent } from "./MessageContent"; @@ -52,3 +53,81 @@ describe("MessageContent link safety", () => { expect(container.querySelector("a")).toBeNull(); }); }); + +describe("MessageContent attachment rendering", () => { + /// The element is chosen from the media type of the bytes that arrived, so + /// each case here has to go through a real download and blob URL. + const serve = (mimeType: string) => { + vi.mocked(invoke).mockImplementation(async () => ({ + mimeType, + bytes: [0, 1, 2, 3], + })); + }; + + beforeEach(() => { + // jsdom implements neither, and both are on the path from bytes to element. + URL.createObjectURL = vi.fn(() => "blob:mock-url"); + URL.revokeObjectURL = vi.fn(); + }); + + it("renders an audio player for a voice reply", async () => { + serve("audio/ogg"); + const { container } = render( + , + ); + + const player = await waitFor(() => { + const found = container.querySelector("audio"); + expect(found).not.toBeNull(); + return found; + }); + expect(player).toHaveAttribute("src", "blob:mock-url"); + expect(player).toHaveAttribute("controls"); + expect(container.querySelector("img")).toBeNull(); + }); + + it("renders an image for an image reply", async () => { + serve("image/png"); + const { container } = render( + , + ); + + await waitFor(() => expect(container.querySelector("img")).not.toBeNull()); + expect(container.querySelector("audio")).toBeNull(); + }); + + /// SILK is what AstrBot relays for QQ and WeChat voice. No browser decodes it, + /// so an