// 语音会话客户端(JARVIS):一条 WebSocket 承载上行麦克风音频 + 下行转写 + 下行 TTS 音频。 // 协议与网关 internal/voice/protocol.go 对齐: // - 二进制帧:上行=麦克风 PCM 16k;下行=TTS PCM 24k // - 文本帧(JSON):控制/事件(ClientMsg / ServerMsg) // // 麦克风采集用 ScriptProcessorNode(WKWebView 通吃,无需单独 worklet 文件),48k→16k 降采样、 // Float32→Int16。下行播放用 AudioBufferSourceNode 排队调度,做无缝连续朗读。 import { GATEWAY, getToken } from "./api"; export type VoiceState = "idle" | "connecting" | "ready" | "listening" | "thinking" | "speaking"; export interface VoiceCallbacks { onState?: (s: VoiceState) => void; onTranscript?: (text: string, final: boolean) => void; onTask?: (taskId: string) => void; onReply?: (deltaText: string) => void; // Agent 回答增量文本(打字机,逐 token) onError?: (msg: string) => void; } const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率(与网关 voice.AudioSampleRate 一致) const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率(与网关 voice.TTSSampleRate 一致) function wsURL(): string { const base = GATEWAY.replace(/^http/, "ws"); const t = getToken(); const q = t ? `?token=${encodeURIComponent(t)}` : ""; return `${base}/api/v1/voice/stream${q}`; } // float32 → int16 PCM(小端)。 function floatToPCM16(f32: Float32Array): ArrayBuffer { const out = new DataView(new ArrayBuffer(f32.length * 2)); for (let i = 0; i < f32.length; i++) { let s = Math.max(-1, Math.min(1, f32[i])); out.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); } return out.buffer; } // 线性降采样到 16k(浏览器 AudioContext 常跑 48k,火山 ASR 要 16k)。 function downsampleTo16k(input: Float32Array, inRate: number): Float32Array { if (inRate === UP_SAMPLE_RATE) return input; const ratio = inRate / UP_SAMPLE_RATE; const outLen = Math.floor(input.length / ratio); const out = new Float32Array(outLen); for (let i = 0; i < outLen; i++) out[i] = input[Math.floor(i * ratio)]; return out; } export class VoiceClient { private ws: WebSocket | null = null; private cb: VoiceCallbacks; private state: VoiceState = "idle"; // 上行采集 private micCtx: AudioContext | null = null; private micStream: MediaStream | null = null; private micNode: ScriptProcessorNode | null = null; private micAnalyser: AnalyserNode | null = null; // 下行播放 private playCtx: AudioContext | null = null; private nextStart = 0; // 下一段音频的调度起点(连续朗读用) private sources: AudioBufferSourceNode[] = []; private playAnalyser: AnalyserNode | null = null; private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲(level() 每帧读,别每帧新建) constructor(cb: VoiceCallbacks) { this.cb = cb; } private setState(s: VoiceState) { this.state = s; this.cb.onState?.(s); } getState(): VoiceState { return this.state; } // connect 建立 WS,等待服务端 ready。 async connect(): Promise { if (this.ws && this.ws.readyState === WebSocket.OPEN) return; this.setState("connecting"); await new Promise((resolve, reject) => { const ws = new WebSocket(wsURL()); ws.binaryType = "arraybuffer"; this.ws = ws; ws.onmessage = (e) => this.onMessage(e); ws.onerror = () => { this.cb.onError?.("语音连接出错"); reject(new Error("ws error")); }; ws.onclose = () => { this.stopMic(); if (this.state !== "idle") this.setState("idle"); }; ws.onopen = () => resolve(); }); } private onMessage(e: MessageEvent) { if (typeof e.data === "string") { let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string }; try { m = JSON.parse(e.data); } catch { return; } switch (m.type) { case "ready": this.setState("ready"); break; case "transcript": this.cb.onTranscript?.(m.text ?? "", !!m.final); if (m.final) this.setState("thinking"); break; case "task": if (m.task_id) this.cb.onTask?.(m.task_id); break; case "reply": if (m.text) this.cb.onReply?.(m.text); // 打字机:回答增量文本 break; case "speaking": this.setState("speaking"); break; case "tts_end": this.resetPlayback(); this.setState("ready"); break; case "error": this.cb.onError?.(m.msg ?? "语音出错"); break; } return; } // 二进制帧 = 下行 TTS 音频(PCM 24k int16) this.enqueueAudio(e.data as ArrayBuffer); } // ---- 上行:麦克风 ---- // startListening 开一轮:连接(若需)→ 若正在朗读先打断 → 采麦克风 → 发 start。 async startListening(graph?: string): Promise { await this.connect(); // 关键:在**用户手势**(点麦克风)里就把播放 AudioContext 建好并 resume——否则它在 WS 回调里 // 惰性创建会处于 suspended 态,TTS 音频静默播不出(Chrome/WKWebView 的自动播放策略)。 this.ensurePlayCtx(); if (this.state === "speaking") this.bargeIn(); this.send({ type: "start", graph }); await this.startMic(); this.setState("listening"); } // ensurePlayCtx 建/复用下行播放 AudioContext,并在 suspended 时 resume(自动播放策略要求手势内唤醒)。 private ensurePlayCtx(): AudioContext { if (!this.playCtx) this.playCtx = new AudioContext(); if (!this.playAnalyser) { // 下行电平探针:所有 TTS 音频经它再到扬声器(驱动 HUD 的"说话"频谱)。 const an = this.playCtx.createAnalyser(); an.fftSize = 512; an.connect(this.playCtx.destination); this.playAnalyser = an; } if (this.playCtx.state === "suspended") void this.playCtx.resume(); return this.playCtx; } // level 返回当前"活跃"音频电平 0..1:说话读下行 TTS,其余读上行麦克风;无探针则 0。 // 供 JARVIS HUD 每帧读取,让可视化随真实声音起伏。 level(): number { const a = this.state === "speaking" ? this.playAnalyser : this.micAnalyser; if (!a) return 0; a.getByteTimeDomainData(this.lvlBuf); let s = 0; for (let i = 0; i < this.lvlBuf.length; i++) { const v = (this.lvlBuf[i] - 128) / 128; s += v * v; } return Math.min(1, Math.sqrt(s / this.lvlBuf.length) * 3.2); } // stopListening 结束本轮说话:停麦克风 + 发 end(服务端拿最终转写→提交任务)。 stopListening(): void { this.stopMic(); this.send({ type: "end" }); if (this.state === "listening") this.setState("thinking"); } // bargeIn 打断当前朗读(用户又开口)。 bargeIn(): void { this.send({ type: "barge_in" }); this.resetPlayback(); } private async startMic(): Promise { if (this.micNode) return; // WKWebView(Wails 原生壳)等非安全上下文里 navigator.mediaDevices 可能未暴露—— // 直接调用会抛 "undefined is not an object" 崩掉。先探测,给出可读提示而非崩溃。 if (!navigator.mediaDevices?.getUserMedia) { throw new Error("此窗口暂不支持麦克风(原生壳需授权/安全上下文)。用浏览器打开 localhost:5173 可直接说话。"); } // 先试带回声消除等约束(浏览器更好),WKWebView(原生壳)不认高级约束会抛 "Invalid constraint", // 退回最简 {audio:true}(最兼容;多数实现默认已开回声消除)。 let stream: MediaStream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 }, }); } catch { stream = await navigator.mediaDevices.getUserMedia({ audio: true }); } this.micStream = stream; const ctx = new AudioContext(); this.micCtx = ctx; if (ctx.state === "suspended") await ctx.resume(); // 防采集上下文挂起(无回调=不上行音频) const src = ctx.createMediaStreamSource(stream); const an = ctx.createAnalyser(); // 上行电平探针(驱动 JARVIS HUD 的"聆听"反应) an.fftSize = 512; src.connect(an); this.micAnalyser = an; const node = ctx.createScriptProcessor(4096, 1, 1); node.onaudioprocess = (ev) => { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; const f32 = ev.inputBuffer.getChannelData(0); const ds = downsampleTo16k(f32, ctx.sampleRate); this.ws.send(floatToPCM16(ds)); }; src.connect(node); node.connect(ctx.destination); // ScriptProcessor 需接上目的地才触发回调(下游静音) this.micNode = node; } private stopMic(): void { this.micNode?.disconnect(); this.micNode = null; this.micAnalyser = null; this.micStream?.getTracks().forEach((t) => t.stop()); this.micStream = null; this.micCtx?.close().catch(() => {}); this.micCtx = null; } // ---- 下行:TTS 音频播放(排队调度,无缝连续)---- private enqueueAudio(buf: ArrayBuffer): void { if (buf.byteLength === 0) return; const ctx = this.ensurePlayCtx(); // 建/复用并 resume(防 WS 回调里上下文仍 suspended → 静默) const i16 = new Int16Array(buf); const f32 = new Float32Array(i16.length); for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 0x8000; const audioBuf = ctx.createBuffer(1, f32.length, DOWN_SAMPLE_RATE); audioBuf.getChannelData(0).set(f32); const node = ctx.createBufferSource(); node.buffer = audioBuf; node.connect(this.playAnalyser ?? ctx.destination); // 经探针再到扬声器(读得到电平) const now = ctx.currentTime; if (this.nextStart < now) this.nextStart = now; node.start(this.nextStart); this.nextStart += audioBuf.duration; node.onended = () => { this.sources = this.sources.filter((s) => s !== node); }; this.sources.push(node); } private resetPlayback(): void { this.sources.forEach((s) => { try { s.stop(); } catch { /* 已停忽略 */ } }); this.sources = []; this.nextStart = 0; } private send(m: { type: string; graph?: string }): void { if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(m)); } // close 彻底关闭会话(发 bye、停采集/播放、断 WS)。 close(): void { this.send({ type: "bye" }); this.stopMic(); this.resetPlayback(); this.playCtx?.close().catch(() => {}); this.playCtx = null; this.playAnalyser = null; this.ws?.close(); this.ws = null; this.setState("idle"); } }