Blizzard
fe5c4215f7
原生壳(Wails WKWebView)的 getUserMedia 不认 channelCount/echoCancellation 等高级约束,
抛 "Invalid constraint"。先试高级约束(浏览器更好),失败退回最简 {audio:true}(最兼容)。
配合上一提交的 .app 打包+NSMicrophoneUsageDescription,原生壳麦克风打通。
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
299 lines
11 KiB
TypeScript
299 lines
11 KiB
TypeScript
// 语音会话客户端（JARVIS）：一条 WebSocket 承载上行麦克风音频 + 下行转写 + 下行 TTS 音频。
|
||
// 协议与网关 internal/voice/protocol.go 对齐：
|
||
// - 二进制帧：上行=麦克风 PCM 16k；下行=TTS PCM 24k
|
||
// - 文本帧(JSON)：控制/事件（ClientMsg / ServerMsg）
|
||
//
|
||
// 麦克风采集用 ScriptProcessorNode（WKWebView 通吃，无需单独 worklet 文件），48k→16k 降采样、
|
||
// Float32→Int16。下行播放用 AudioBufferSourceNode 排队调度，做无缝连续朗读。
|
||
|
||
import { GATEWAY, getToken } from "./api";
|
||
|
||
export type VoiceState = "idle" | "connecting" | "ready" | "listening" | "thinking" | "speaking";
|
||
|
||
export interface VoiceCallbacks {
|
||
onState?: (s: VoiceState) => void;
|
||
onTranscript?: (text: string, final: boolean) => void;
|
||
onTask?: (taskId: string) => void;
|
||
onReply?: (deltaText: string) => void; // Agent 回答增量文本（打字机，逐 token）
|
||
onError?: (msg: string) => void;
|
||
}
|
||
|
||
const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率（与网关 voice.AudioSampleRate 一致）
|
||
const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率（与网关 voice.TTSSampleRate 一致）
|
||
|
||
function wsURL(): string {
|
||
const base = GATEWAY.replace(/^http/, "ws");
|
||
const t = getToken();
|
||
const q = t ? `?token=${encodeURIComponent(t)}` : "";
|
||
return `${base}/api/v1/voice/stream${q}`;
|
||
}
|
||
|
||
// float32 → int16 PCM（小端）。
|
||
function floatToPCM16(f32: Float32Array): ArrayBuffer {
|
||
const out = new DataView(new ArrayBuffer(f32.length * 2));
|
||
for (let i = 0; i < f32.length; i++) {
|
||
let s = Math.max(-1, Math.min(1, f32[i]));
|
||
out.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||
}
|
||
return out.buffer;
|
||
}
|
||
|
||
// 线性降采样到 16k（浏览器 AudioContext 常跑 48k，火山 ASR 要 16k）。
|
||
function downsampleTo16k(input: Float32Array, inRate: number): Float32Array {
|
||
if (inRate === UP_SAMPLE_RATE) return input;
|
||
const ratio = inRate / UP_SAMPLE_RATE;
|
||
const outLen = Math.floor(input.length / ratio);
|
||
const out = new Float32Array(outLen);
|
||
for (let i = 0; i < outLen; i++) out[i] = input[Math.floor(i * ratio)];
|
||
return out;
|
||
}
|
||
|
||
export class VoiceClient {
|
||
private ws: WebSocket | null = null;
|
||
private cb: VoiceCallbacks;
|
||
private state: VoiceState = "idle";
|
||
|
||
// 上行采集
|
||
private micCtx: AudioContext | null = null;
|
||
private micStream: MediaStream | null = null;
|
||
private micNode: ScriptProcessorNode | null = null;
|
||
private micAnalyser: AnalyserNode | null = null;
|
||
|
||
// 下行播放
|
||
private playCtx: AudioContext | null = null;
|
||
private nextStart = 0; // 下一段音频的调度起点（连续朗读用）
|
||
private sources: AudioBufferSourceNode[] = [];
|
||
private playAnalyser: AnalyserNode | null = null;
|
||
private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲（level() 每帧读，别每帧新建）
|
||
|
||
constructor(cb: VoiceCallbacks) {
|
||
this.cb = cb;
|
||
}
|
||
|
||
private setState(s: VoiceState) {
|
||
this.state = s;
|
||
this.cb.onState?.(s);
|
||
}
|
||
|
||
getState(): VoiceState {
|
||
return this.state;
|
||
}
|
||
|
||
// connect 建立 WS，等待服务端 ready。
|
||
async connect(): Promise<void> {
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) return;
|
||
this.setState("connecting");
|
||
await new Promise<void>((resolve, reject) => {
|
||
const ws = new WebSocket(wsURL());
|
||
ws.binaryType = "arraybuffer";
|
||
this.ws = ws;
|
||
ws.onmessage = (e) => this.onMessage(e);
|
||
ws.onerror = () => {
|
||
this.cb.onError?.("语音连接出错");
|
||
reject(new Error("ws error"));
|
||
};
|
||
ws.onclose = () => {
|
||
this.stopMic();
|
||
if (this.state !== "idle") this.setState("idle");
|
||
};
|
||
ws.onopen = () => resolve();
|
||
});
|
||
}
|
||
|
||
private onMessage(e: MessageEvent) {
|
||
if (typeof e.data === "string") {
|
||
let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string };
|
||
try {
|
||
m = JSON.parse(e.data);
|
||
} catch {
|
||
return;
|
||
}
|
||
switch (m.type) {
|
||
case "ready":
|
||
this.setState("ready");
|
||
break;
|
||
case "transcript":
|
||
this.cb.onTranscript?.(m.text ?? "", !!m.final);
|
||
if (m.final) this.setState("thinking");
|
||
break;
|
||
case "task":
|
||
if (m.task_id) this.cb.onTask?.(m.task_id);
|
||
break;
|
||
case "reply":
|
||
if (m.text) this.cb.onReply?.(m.text); // 打字机：回答增量文本
|
||
break;
|
||
case "speaking":
|
||
this.setState("speaking");
|
||
break;
|
||
case "tts_end":
|
||
this.resetPlayback();
|
||
this.setState("ready");
|
||
break;
|
||
case "error":
|
||
this.cb.onError?.(m.msg ?? "语音出错");
|
||
break;
|
||
}
|
||
return;
|
||
}
|
||
// 二进制帧 = 下行 TTS 音频（PCM 24k int16）
|
||
this.enqueueAudio(e.data as ArrayBuffer);
|
||
}
|
||
|
||
// ---- 上行：麦克风 ----
|
||
|
||
// startListening 开一轮：连接（若需）→ 若正在朗读先打断 → 采麦克风 → 发 start。
|
||
async startListening(graph?: string): Promise<void> {
|
||
await this.connect();
|
||
// 关键：在**用户手势**（点麦克风）里就把播放 AudioContext 建好并 resume——否则它在 WS 回调里
|
||
// 惰性创建会处于 suspended 态，TTS 音频静默播不出（Chrome/WKWebView 的自动播放策略）。
|
||
this.ensurePlayCtx();
|
||
if (this.state === "speaking") this.bargeIn();
|
||
this.send({ type: "start", graph });
|
||
await this.startMic();
|
||
this.setState("listening");
|
||
}
|
||
|
||
// ensurePlayCtx 建/复用下行播放 AudioContext，并在 suspended 时 resume（自动播放策略要求手势内唤醒）。
|
||
private ensurePlayCtx(): AudioContext {
|
||
if (!this.playCtx) this.playCtx = new AudioContext();
|
||
if (!this.playAnalyser) {
|
||
// 下行电平探针：所有 TTS 音频经它再到扬声器（驱动 HUD 的"说话"频谱）。
|
||
const an = this.playCtx.createAnalyser();
|
||
an.fftSize = 512;
|
||
an.connect(this.playCtx.destination);
|
||
this.playAnalyser = an;
|
||
}
|
||
if (this.playCtx.state === "suspended") void this.playCtx.resume();
|
||
return this.playCtx;
|
||
}
|
||
|
||
// level 返回当前"活跃"音频电平 0..1：说话读下行 TTS，其余读上行麦克风；无探针则 0。
|
||
// 供 JARVIS HUD 每帧读取，让可视化随真实声音起伏。
|
||
level(): number {
|
||
const a = this.state === "speaking" ? this.playAnalyser : this.micAnalyser;
|
||
if (!a) return 0;
|
||
a.getByteTimeDomainData(this.lvlBuf);
|
||
let s = 0;
|
||
for (let i = 0; i < this.lvlBuf.length; i++) {
|
||
const v = (this.lvlBuf[i] - 128) / 128;
|
||
s += v * v;
|
||
}
|
||
return Math.min(1, Math.sqrt(s / this.lvlBuf.length) * 3.2);
|
||
}
|
||
|
||
// stopListening 结束本轮说话：停麦克风 + 发 end（服务端拿最终转写→提交任务）。
|
||
stopListening(): void {
|
||
this.stopMic();
|
||
this.send({ type: "end" });
|
||
if (this.state === "listening") this.setState("thinking");
|
||
}
|
||
|
||
// bargeIn 打断当前朗读（用户又开口）。
|
||
bargeIn(): void {
|
||
this.send({ type: "barge_in" });
|
||
this.resetPlayback();
|
||
}
|
||
|
||
private async startMic(): Promise<void> {
|
||
if (this.micNode) return;
|
||
// WKWebView（Wails 原生壳）等非安全上下文里 navigator.mediaDevices 可能未暴露——
|
||
// 直接调用会抛 "undefined is not an object" 崩掉。先探测，给出可读提示而非崩溃。
|
||
if (!navigator.mediaDevices?.getUserMedia) {
|
||
throw new Error("此窗口暂不支持麦克风（原生壳需授权/安全上下文）。用浏览器打开 localhost:5173 可直接说话。");
|
||
}
|
||
// 先试带回声消除等约束（浏览器更好），WKWebView（原生壳）不认高级约束会抛 "Invalid constraint"，
|
||
// 退回最简 {audio:true}（最兼容；多数实现默认已开回声消除）。
|
||
let stream: MediaStream;
|
||
try {
|
||
stream = await navigator.mediaDevices.getUserMedia({
|
||
audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 },
|
||
});
|
||
} catch {
|
||
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||
}
|
||
this.micStream = stream;
|
||
const ctx = new AudioContext();
|
||
this.micCtx = ctx;
|
||
if (ctx.state === "suspended") await ctx.resume(); // 防采集上下文挂起（无回调=不上行音频）
|
||
const src = ctx.createMediaStreamSource(stream);
|
||
const an = ctx.createAnalyser(); // 上行电平探针（驱动 JARVIS HUD 的"聆听"反应）
|
||
an.fftSize = 512;
|
||
src.connect(an);
|
||
this.micAnalyser = an;
|
||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||
node.onaudioprocess = (ev) => {
|
||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||
const f32 = ev.inputBuffer.getChannelData(0);
|
||
const ds = downsampleTo16k(f32, ctx.sampleRate);
|
||
this.ws.send(floatToPCM16(ds));
|
||
};
|
||
src.connect(node);
|
||
node.connect(ctx.destination); // ScriptProcessor 需接上目的地才触发回调（下游静音）
|
||
this.micNode = node;
|
||
}
|
||
|
||
private stopMic(): void {
|
||
this.micNode?.disconnect();
|
||
this.micNode = null;
|
||
this.micAnalyser = null;
|
||
this.micStream?.getTracks().forEach((t) => t.stop());
|
||
this.micStream = null;
|
||
this.micCtx?.close().catch(() => {});
|
||
this.micCtx = null;
|
||
}
|
||
|
||
// ---- 下行：TTS 音频播放（排队调度，无缝连续）----
|
||
|
||
private enqueueAudio(buf: ArrayBuffer): void {
|
||
if (buf.byteLength === 0) return;
|
||
const ctx = this.ensurePlayCtx(); // 建/复用并 resume（防 WS 回调里上下文仍 suspended → 静默）
|
||
|
||
const i16 = new Int16Array(buf);
|
||
const f32 = new Float32Array(i16.length);
|
||
for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 0x8000;
|
||
const audioBuf = ctx.createBuffer(1, f32.length, DOWN_SAMPLE_RATE);
|
||
audioBuf.getChannelData(0).set(f32);
|
||
|
||
const node = ctx.createBufferSource();
|
||
node.buffer = audioBuf;
|
||
node.connect(this.playAnalyser ?? ctx.destination); // 经探针再到扬声器（读得到电平）
|
||
const now = ctx.currentTime;
|
||
if (this.nextStart < now) this.nextStart = now;
|
||
node.start(this.nextStart);
|
||
this.nextStart += audioBuf.duration;
|
||
node.onended = () => {
|
||
this.sources = this.sources.filter((s) => s !== node);
|
||
};
|
||
this.sources.push(node);
|
||
}
|
||
|
||
private resetPlayback(): void {
|
||
this.sources.forEach((s) => {
|
||
try {
|
||
s.stop();
|
||
} catch {
|
||
/* 已停忽略 */
|
||
}
|
||
});
|
||
this.sources = [];
|
||
this.nextStart = 0;
|
||
}
|
||
|
||
private send(m: { type: string; graph?: string }): void {
|
||
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(m));
|
||
}
|
||
|
||
// close 彻底关闭会话（发 bye、停采集/播放、断 WS）。
|
||
close(): void {
|
||
this.send({ type: "bye" });
|
||
this.stopMic();
|
||
this.resetPlayback();
|
||
this.playCtx?.close().catch(() => {});
|
||
this.playCtx = null;
|
||
this.playAnalyser = null;
|
||
this.ws?.close();
|
||
this.ws = null;
|
||
this.setState("idle");
|
||
}
|
||
}
|