feat(chat): record voice and send it for the server to transcribe
一些检测仍在等待运行
ci / Portable quality checks (push) Waiting to run
ci / Windows Tauri build (push) Waiting to run

Voice input could never have worked. `ChatPanel` reached for
`SpeechRecognition`, which WebView2 does not provide, so the only branch
ever taken was the one reporting "当前 WebView 不支持语音输入" — and the
architecture had specified a different route all along
(`01_Architecture_Design.md:1057-1063`): press to record, upload the
audio, let AstrBot transcribe it.

That is what this does. `MediaRecorder` captures to Opus in WebM, the
audio is uploaded through the same attachment boundary as any other file,
and it is sent with no text of its own — the audio *is* the message, and
the server's speech-to-text stage turns it into one. The `record` segment
this produces comes for free from the message-composition fix earlier
today.

The slot is fixed at the start of the recording, so a recording finished
after the user switched tabs still reaches the conversation they were
speaking to (`:1065` binds the microphone to the focused instance).

Failures are mapped rather than reported raw. Refused permission, no
microphone, and a microphone held by another program need three different
actions from the user; the old code told them "check microphone
permissions" for all of them. A WebView2 build with no audio codec is
detected before the microphone is opened rather than by letting
`MediaRecorder` throw.

The microphone track is stopped on every exit path, including a failed
encode and a recorder that cannot be constructed. Leaving it live is what
keeps Windows' recording indicator on after the user has finished
speaking, and each of those paths has a test that fails without it.

One thing this cannot settle without a real build: WebView2 raises a
`PermissionRequested` event for the microphone that the host application
handles, and Tauri registers a handler only when its `clipboard` option is
enabled, which this app does not enable. So WebView2's own default decides
whether the prompt appears, is auto-granted, or is auto-denied. The
failure mapping exists so that whichever it turns out to be, the user is
told something they can act on.

Frontend tests 47 -> 59. All five mutations of the new assertions were
confirmed to fail.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
这个提交存在于:
YokiiroBW 2026-07-27 06:33:48 +08:00
父节点 b6a21a70ec
当前提交 876fad30c1
共有 3 个文件被更改,包括 383 次插入 和 47 次删除

查看文件

@ -5,6 +5,12 @@ import { createPortal } from "react-dom";
import { WINDOW_CHROME_MOUNT_ID } from "../../App"; import { WINDOW_CHROME_MOUNT_ID } from "../../App";
import { errorMessage } from "../../lib/errorMessage"; import { errorMessage } from "../../lib/errorMessage";
import {
describeMicrophoneFailure,
isUsableRecording,
startRecording,
type VoiceRecording,
} from "./voiceRecorder";
import { MessageContent } from "./MessageContent"; import { MessageContent } from "./MessageContent";
import { import {
@ -45,16 +51,6 @@ export interface ChatPanelProps {
fixedSlot?: ChatSlot; fixedSlot?: ChatSlot;
} }
type SpeechRecognitionLike = {
lang: string;
interimResults: boolean;
onresult: ((event: unknown) => void) | null;
onerror: (() => void) | null;
onend: (() => void) | null;
start: () => void;
stop: () => void;
};
export function ChatPanel({ fixedSlot }: ChatPanelProps) { export function ChatPanel({ fixedSlot }: ChatPanelProps) {
const [status, setStatus] = useState(emptyStatus); const [status, setStatus] = useState(emptyStatus);
const [messages, setMessages] = useState<ChatMessage[]>([]); const [messages, setMessages] = useState<ChatMessage[]>([]);
@ -121,10 +117,9 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) {
/// stays true until the reader scrolls up — following unconditionally would /// stays true until the reader scrolls up — following unconditionally would
/// yank someone out of the history they were reading. /// yank someone out of the history they were reading.
const followTail = useRef(true); const followTail = useRef(true);
const speechRecognition = useRef<{ /// Stops the in-flight recording and yields its audio. Present only while a
start: () => void; /// recording is live, so the button cannot start a second one.
stop: () => void; const stopRecording = useRef<(() => Promise<VoiceRecording>) | null>(null);
} | null>(null);
const [unreadCounts, setUnreadCounts] = useState<Record<ChatSlot, number>>({ const [unreadCounts, setUnreadCounts] = useState<Record<ChatSlot, number>>({
companion: 0, companion: 0,
operations: 0, operations: 0,
@ -636,43 +631,64 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) {
} }
}; };
const toggleVoiceInput = () => { /// Records, uploads, and sends the audio for AstrBot to transcribe.
///
/// The architecture fixes this flow at `:1057-1063` as 按键录音 → 上传音频 → STT
/// → 文本回复 → 可选 TTS, and the microphone binds to the focused instance
/// (`:1065`). One recording therefore goes to one slot: the slot is fixed here,
/// at the start, so a recording finished after the user switched tabs still
/// reaches the conversation they were speaking to.
const toggleVoiceInput = async () => {
const slot = selectedSlot; const slot = selectedSlot;
if (voiceListening) { if (stopRecording.current) {
speechRecognition.current?.stop(); const stop = stopRecording.current;
stopRecording.current = null;
setVoiceListening(false); setVoiceListening(false);
try {
const recording = await stop();
if (!isUsableRecording(recording)) {
reportSlotError(slot, "录音太短,没有发送。");
return;
}
const attachment = astrbotAttachmentSchema.parse(
await invoke<unknown>("upload_connection_slot_file", {
request: {
slot,
// The extension follows the container so the server, and whichever
// speech provider it hands the file to, can tell what this is.
fileName: `voice-${Date.now()}.${recording.mimeType.includes("ogg") ? "ogg" : recording.mimeType.includes("mp4") ? "m4a" : "webm"}`,
mimeType: recording.mimeType,
bytes: recording.bytes,
},
}),
);
// No text of our own: the audio *is* the message, and AstrBot's
// speech-to-text stage is what turns it into one.
await invoke<unknown>("send_connection_slot_message", {
request: { slot, message: "", attachments: [attachment] },
});
} catch (reason) {
reportSlotError(slot, errorMessage(reason));
}
return; return;
} }
const browserWindow = window as unknown as {
SpeechRecognition?: new () => SpeechRecognitionLike; try {
webkitSpeechRecognition?: new () => SpeechRecognitionLike; stopRecording.current = await startRecording(
}; navigator.mediaDevices,
const Recognition = (stream, mimeType) => new MediaRecorder(stream, { mimeType }),
browserWindow.SpeechRecognition ?? browserWindow.webkitSpeechRecognition; (type) => MediaRecorder.isTypeSupported(type),
if (!Recognition) { () => Date.now(),
reportSlotError(slot, "当前 WebView 不支持语音输入。"); );
return; setVoiceListening(true);
} } catch (reason) {
const recognition = new Recognition(); // Mapped rather than reported raw: refused permission, no microphone, and
recognition.lang = "zh-CN"; // a microphone held by another program need three different actions from
recognition.interimResults = true; // the user, and the old code told them "check microphone permissions" for
recognition.onresult = (event) => { // all of them.
const result = event as { reportSlotError(slot, describeMicrophoneFailure(reason));
results?: ArrayLike<ArrayLike<{ transcript?: string }>>;
};
const transcript = Array.from(result.results ?? [])
.map((item) => item[0]?.transcript ?? "")
.join("");
setDraft((current) => ({ ...current, [slot]: transcript }));
};
recognition.onerror = () => {
reportSlotError(slot, "语音输入失败,请检查麦克风权限。");
setVoiceListening(false); setVoiceListening(false);
}; }
recognition.onend = () => setVoiceListening(false);
speechRecognition.current = recognition;
recognition.start();
setVoiceListening(true);
}; };
const speakMessage = (content: string) => { const speakMessage = (content: string) => {
@ -950,7 +966,7 @@ export function ChatPanel({ fixedSlot }: ChatPanelProps) {
{uploading ? <span className="toolbar-note">上传中…</span> : null} {uploading ? <span className="toolbar-note">上传中…</span> : null}
<button <button
type="button" type="button"
onClick={toggleVoiceInput} onClick={() => void toggleVoiceInput()}
className={voiceListening ? "voice-button active" : "voice-button"} className={voiceListening ? "voice-button active" : "voice-button"}
aria-pressed={voiceListening} aria-pressed={voiceListening}
> >

查看文件

@ -0,0 +1,164 @@
import { describe, expect, it, vi } from "vitest";
import {
MINIMUM_RECORDING_MS,
PREFERRED_RECORDING_TYPES,
describeMicrophoneFailure,
isUsableRecording,
pickRecordingType,
startRecording,
} from "./voiceRecorder";
describe("pickRecordingType", () => {
it("prefers opus in webm, which is what speech providers accept most reliably", () => {
expect(pickRecordingType(() => true)).toBe("audio/webm;codecs=opus");
});
it("falls through to whatever this WebView2 build does support", () => {
expect(pickRecordingType((type) => type === "audio/mp4")).toBe("audio/mp4");
});
/// A WebView2 without the media codec pack supports no audio type at all.
it("reports no format rather than letting MediaRecorder throw", () => {
expect(pickRecordingType(() => false)).toBeNull();
});
it("only ever offers audio types", () => {
for (const type of PREFERRED_RECORDING_TYPES) {
expect(type.startsWith("audio/")).toBe(true);
}
});
});
describe("describeMicrophoneFailure", () => {
/// Each of these needs a different action from the user, so they must not
/// collapse into one message — which is what the old code did.
it("distinguishes the cases that need different actions", () => {
const denied = describeMicrophoneFailure({ name: "NotAllowedError" });
const missing = describeMicrophoneFailure({ name: "NotFoundError" });
const busy = describeMicrophoneFailure({ name: "NotReadableError" });
expect(denied).toMatch(/权限/);
expect(missing).toMatch(/没有找到/);
expect(busy).toMatch(/占用/);
expect(new Set([denied, missing, busy]).size).toBe(3);
});
it("still says something usable for an unrecognised failure", () => {
expect(describeMicrophoneFailure(undefined)).not.toBe("");
expect(describeMicrophoneFailure(new Error("boom"))).toMatch(/WebView2/);
});
});
describe("isUsableRecording", () => {
it("discards a tap on the button", () => {
expect(
isUsableRecording({ mimeType: "audio/webm", bytes: [1], durationMs: 40 }),
).toBe(false);
});
it("discards a recording that produced no audio", () => {
expect(
isUsableRecording({
mimeType: "audio/webm",
bytes: [],
durationMs: 5000,
}),
).toBe(false);
});
it("keeps a recording long enough to be speech", () => {
expect(
isUsableRecording({
mimeType: "audio/webm",
bytes: [1, 2],
durationMs: MINIMUM_RECORDING_MS,
}),
).toBe(true);
});
});
describe("startRecording", () => {
const fakeStream = () => {
const stop = vi.fn();
return {
stream: { getTracks: () => [{ stop }] } as unknown as MediaStream,
stop,
};
};
const fakeRecorder = () => {
const recorder = {
ondataavailable: null as ((event: { data: Blob }) => void) | null,
onstop: null as (() => void) | null,
onerror: null as (() => void) | null,
start: vi.fn(),
stop: vi.fn(),
};
// A real MediaRecorder fires onstop asynchronously after stop().
recorder.stop.mockImplementation(() => recorder.onstop?.());
return recorder;
};
it("releases the microphone once the recording ends", async () => {
const { stream, stop: stopTrack } = fakeStream();
const recorder = fakeRecorder();
let clock = 1000;
const finish = await startRecording(
{ getUserMedia: async () => stream } as unknown as MediaDevices,
() => recorder as unknown as MediaRecorder,
() => true,
() => clock,
);
expect(recorder.start).toHaveBeenCalled();
// The track must still be live while recording, or nothing is captured.
expect(stopTrack).not.toHaveBeenCalled();
recorder.ondataavailable?.({ data: new Blob([new Uint8Array([7, 8])]) });
clock = 3000;
const recording = await finish();
// Leaving the track live is what keeps Windows' recording indicator on after
// the user has finished speaking.
expect(stopTrack).toHaveBeenCalled();
expect(recording.durationMs).toBe(2000);
expect(recording.mimeType).toBe("audio/webm;codecs=opus");
expect(recording.bytes).toEqual([7, 8]);
});
it("releases the microphone when no codec is available", async () => {
const { stream, stop: stopTrack } = fakeStream();
const getUserMedia = vi.fn(async () => stream);
await expect(
startRecording(
{ getUserMedia } as unknown as MediaDevices,
() => fakeRecorder() as unknown as MediaRecorder,
() => false,
() => 0,
),
).rejects.toThrow(/编码器/);
// Refused before the microphone was even opened.
expect(getUserMedia).not.toHaveBeenCalled();
expect(stopTrack).not.toHaveBeenCalled();
});
it("releases the microphone when the recorder cannot be constructed", async () => {
const { stream, stop: stopTrack } = fakeStream();
await expect(
startRecording(
{ getUserMedia: async () => stream } as unknown as MediaDevices,
() => {
throw new Error("unsupported");
},
() => true,
() => 0,
),
).rejects.toThrow("unsupported");
expect(stopTrack).toHaveBeenCalled();
});
});

查看文件

@ -0,0 +1,156 @@
/**
* Microphone capture for the voice flow in ADR-0008 and
* `01_Architecture_Design.md:1057-1063`: press to record, upload the audio, let
* AstrBot transcribe it.
*
* The client runs no speech recognition of its own. `ChatPanel` used to reach for
* `SpeechRecognition`, which WebView2 does not provide — so voice input could
* never have worked, and the "当前 WebView 不支持语音输入" branch was the only one
* ever taken.
*
* Microphone access in WebView2 raises a `PermissionRequested` event that the
* host application handles. Tauri registers a handler for that event only when
* its `clipboard` option is enabled, which this application does not enable, so
* what happens here is decided by WebView2's own default and cannot be
* established without a real build. That is why the failure mapping below is
* careful: if permission is refused, the user has to be told which of several
* quite different things went wrong.
*/
/// Formats this application prefers, best first.
///
/// Opus in WebM is what Chromium produces natively and what speech-to-text
/// providers most reliably accept. The others are fallbacks for a WebView2
/// build with a different codec set.
export const PREFERRED_RECORDING_TYPES = [
"audio/webm;codecs=opus",
"audio/webm",
"audio/ogg;codecs=opus",
"audio/ogg",
"audio/mp4",
] as const;
/**
* Chooses a recording format, or `null` when none of them is available.
*
* `null` is a real outcome rather than a defensive branch: a WebView2 build
* without the media codecs installed supports no audio type at all, and calling
* `MediaRecorder` anyway throws with a message no user could act on.
*/
export function pickRecordingType(
isTypeSupported: (type: string) => boolean,
): string | null {
for (const candidate of PREFERRED_RECORDING_TYPES) {
if (isTypeSupported(candidate)) return candidate;
}
return null;
}
/**
* Turns a `getUserMedia` failure into something the user can act on.
*
* These cases lead to different actions — grant a permission, plug in a
* microphone, close the application holding it — so collapsing them into "语音
* 输入失败,请检查麦克风权限" (which is what the old code did for every failure)
* sends the user to the wrong place most of the time.
*/
export function describeMicrophoneFailure(reason: unknown): string {
const name =
typeof reason === "object" && reason !== null && "name" in reason
? String((reason as { name: unknown }).name)
: "";
switch (name) {
case "NotAllowedError":
case "SecurityError":
return "麦克风权限被拒绝。请在 Windows 设置 › 隐私和安全性 › 麦克风中允许桌面应用使用麦克风。";
case "NotFoundError":
case "OverconstrainedError":
return "没有找到可用的麦克风。";
case "NotReadableError":
return "麦克风被其他程序占用,无法读取。";
case "AbortError":
return "麦克风初始化被中断,请重试。";
default:
return "无法开始录音。这台机器上的 WebView2 可能未提供麦克风访问。";
}
}
/// The result of one recording, ready for the attachment upload boundary.
export type VoiceRecording = {
mimeType: string;
bytes: number[];
durationMs: number;
};
/**
* Discards a recording that is too short to be speech.
*
* A tap on the button produces a few tens of milliseconds of audio, which costs
* an upload and a transcription round trip to come back as an empty string.
*/
export const MINIMUM_RECORDING_MS = 350;
export function isUsableRecording(recording: VoiceRecording): boolean {
return (
recording.durationMs >= MINIMUM_RECORDING_MS && recording.bytes.length > 0
);
}
/**
* Starts recording and resolves with a function that stops it.
*
* Kept as a single call returning its own stopper so a caller cannot hold a
* half-started recorder: either there is a live stream and a way to end it, or
* the promise rejected and there is nothing to clean up. The microphone track is
* stopped on every exit path, including a failed encode — leaving it live is what
* keeps the operating system's recording indicator on after the user has
* finished.
*/
export async function startRecording(
mediaDevices: MediaDevices,
createRecorder: (stream: MediaStream, mimeType: string) => MediaRecorder,
isTypeSupported: (type: string) => boolean,
now: () => number,
): Promise<() => Promise<VoiceRecording>> {
const mimeType = pickRecordingType(isTypeSupported);
if (!mimeType) {
throw new Error("这台机器上的 WebView2 没有可用的音频编码器。");
}
const stream = await mediaDevices.getUserMedia({ audio: true });
const startedAt = now();
let recorder: MediaRecorder;
try {
recorder = createRecorder(stream, mimeType);
} catch (reason) {
stream.getTracks().forEach((track) => track.stop());
throw reason;
}
const chunks: Blob[] = [];
recorder.ondataavailable = (event) => {
if (event.data && event.data.size > 0) chunks.push(event.data);
};
recorder.start();
return () =>
new Promise<VoiceRecording>((resolve, reject) => {
recorder.onstop = () => {
stream.getTracks().forEach((track) => track.stop());
const durationMs = now() - startedAt;
void new Blob(chunks, { type: mimeType })
.arrayBuffer()
.then((buffer) => {
resolve({
mimeType,
bytes: Array.from(new Uint8Array(buffer)),
durationMs,
});
})
.catch(reject);
};
recorder.onerror = () => {
stream.getTracks().forEach((track) => track.stop());
reject(new Error("录音过程中出错。"));
};
recorder.stop();
});
}