From 876fad30c11346de6c18495d3c934701c39dbcce Mon Sep 17 00:00:00 2001 From: YokiiroBW Date: Mon, 27 Jul 2026 06:33:48 +0800 Subject: [PATCH] feat(chat): record voice and send it for the server to transcribe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voice input could never have worked. `ChatPanel` reached for `SpeechRecognition`, which WebView2 does not provide, so the only branch ever taken was the one reporting "当前 WebView 不支持语音输入" — and the architecture had specified a different route all along (`01_Architecture_Design.md:1057-1063`): press to record, upload the audio, let AstrBot transcribe it. That is what this does. `MediaRecorder` captures to Opus in WebM, the audio is uploaded through the same attachment boundary as any other file, and it is sent with no text of its own — the audio *is* the message, and the server's speech-to-text stage turns it into one. The `record` segment this produces comes for free from the message-composition fix earlier today. The slot is fixed at the start of the recording, so a recording finished after the user switched tabs still reaches the conversation they were speaking to (`:1065` binds the microphone to the focused instance). Failures are mapped rather than reported raw. Refused permission, no microphone, and a microphone held by another program need three different actions from the user; the old code told them "check microphone permissions" for all of them. A WebView2 build with no audio codec is detected before the microphone is opened rather than by letting `MediaRecorder` throw. The microphone track is stopped on every exit path, including a failed encode and a recorder that cannot be constructed. Leaving it live is what keeps Windows' recording indicator on after the user has finished speaking, and each of those paths has a test that fails without it. One thing this cannot settle without a real build: WebView2 raises a `PermissionRequested` event for the microphone that the host application handles, and Tauri registers a handler only when its `clipboard` option is enabled, which this app does not enable. So WebView2's own default decides whether the prompt appears, is auto-granted, or is auto-denied. The failure mapping exists so that whichever it turns out to be, the user is told something they can act on. Frontend tests 47 -> 59. All five mutations of the new assertions were confirmed to fail. Co-Authored-By: Claude Opus 5 --- src/features/chat/ChatPanel.tsx | 110 +++++++++------- src/features/chat/voiceRecorder.test.ts | 164 ++++++++++++++++++++++++ src/features/chat/voiceRecorder.ts | 156 ++++++++++++++++++++++ 3 files changed, 383 insertions(+), 47 deletions(-) create mode 100644 src/features/chat/voiceRecorder.test.ts create mode 100644 src/features/chat/voiceRecorder.ts diff --git a/src/features/chat/ChatPanel.tsx b/src/features/chat/ChatPanel.tsx index 571ebb9..296a871 100644 --- a/src/features/chat/ChatPanel.tsx +++ b/src/features/chat/ChatPanel.tsx @@ -5,6 +5,12 @@ import { createPortal } from "react-dom"; import { WINDOW_CHROME_MOUNT_ID } from "../../App"; import { errorMessage } from "../../lib/errorMessage"; +import { + describeMicrophoneFailure, + isUsableRecording, + startRecording, + type VoiceRecording, +} from "./voiceRecorder"; import { MessageContent } from "./MessageContent"; import { @@ -45,16 +51,6 @@ export interface ChatPanelProps { fixedSlot?: ChatSlot; } -type SpeechRecognitionLike = { - lang: string; - interimResults: boolean; - onresult: ((event: unknown) => void) | null; - onerror: (() => void) | null; - onend: (() => void) | null; - start: () => void; - stop: () => void; -}; - export function ChatPanel({ fixedSlot }: ChatPanelProps) { const [status, setStatus] = useState(emptyStatus); const [messages, setMessages] = useState([]); @@ -121,10 +117,9 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) { /// stays true until the reader scrolls up — following unconditionally would /// yank someone out of the history they were reading. const followTail = useRef(true); - const speechRecognition = useRef<{ - start: () => void; - stop: () => void; - } | null>(null); + /// Stops the in-flight recording and yields its audio. Present only while a + /// recording is live, so the button cannot start a second one. + const stopRecording = useRef<(() => Promise) | null>(null); const [unreadCounts, setUnreadCounts] = useState>({ companion: 0, operations: 0, @@ -636,43 +631,64 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) { } }; - const toggleVoiceInput = () => { + /// Records, uploads, and sends the audio for AstrBot to transcribe. + /// + /// The architecture fixes this flow at `:1057-1063` as 按键录音 → 上传音频 → STT + /// → 文本回复 → 可选 TTS, and the microphone binds to the focused instance + /// (`:1065`). One recording therefore goes to one slot: the slot is fixed here, + /// at the start, so a recording finished after the user switched tabs still + /// reaches the conversation they were speaking to. + const toggleVoiceInput = async () => { const slot = selectedSlot; - if (voiceListening) { - speechRecognition.current?.stop(); + if (stopRecording.current) { + const stop = stopRecording.current; + stopRecording.current = null; setVoiceListening(false); + try { + const recording = await stop(); + if (!isUsableRecording(recording)) { + reportSlotError(slot, "录音太短,没有发送。"); + return; + } + const attachment = astrbotAttachmentSchema.parse( + await invoke("upload_connection_slot_file", { + request: { + slot, + // The extension follows the container so the server, and whichever + // speech provider it hands the file to, can tell what this is. + fileName: `voice-${Date.now()}.${recording.mimeType.includes("ogg") ? "ogg" : recording.mimeType.includes("mp4") ? "m4a" : "webm"}`, + mimeType: recording.mimeType, + bytes: recording.bytes, + }, + }), + ); + // No text of our own: the audio *is* the message, and AstrBot's + // speech-to-text stage is what turns it into one. + await invoke("send_connection_slot_message", { + request: { slot, message: "", attachments: [attachment] }, + }); + } catch (reason) { + reportSlotError(slot, errorMessage(reason)); + } return; } - const browserWindow = window as unknown as { - SpeechRecognition?: new () => SpeechRecognitionLike; - webkitSpeechRecognition?: new () => SpeechRecognitionLike; - }; - const Recognition = - browserWindow.SpeechRecognition ?? browserWindow.webkitSpeechRecognition; - if (!Recognition) { - reportSlotError(slot, "当前 WebView 不支持语音输入。"); - return; - } - const recognition = new Recognition(); - recognition.lang = "zh-CN"; - recognition.interimResults = true; - recognition.onresult = (event) => { - const result = event as { - results?: ArrayLike>; - }; - const transcript = Array.from(result.results ?? []) - .map((item) => item[0]?.transcript ?? "") - .join(""); - setDraft((current) => ({ ...current, [slot]: transcript })); - }; - recognition.onerror = () => { - reportSlotError(slot, "语音输入失败,请检查麦克风权限。"); + + try { + stopRecording.current = await startRecording( + navigator.mediaDevices, + (stream, mimeType) => new MediaRecorder(stream, { mimeType }), + (type) => MediaRecorder.isTypeSupported(type), + () => Date.now(), + ); + setVoiceListening(true); + } catch (reason) { + // Mapped rather than reported raw: refused permission, no microphone, and + // a microphone held by another program need three different actions from + // the user, and the old code told them "check microphone permissions" for + // all of them. + reportSlotError(slot, describeMicrophoneFailure(reason)); setVoiceListening(false); - }; - recognition.onend = () => setVoiceListening(false); - speechRecognition.current = recognition; - recognition.start(); - setVoiceListening(true); + } }; const speakMessage = (content: string) => { @@ -950,7 +966,7 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) { {uploading ? 上传中… : null}