feat(voice): 桌面端 JARVIS 语音坞——麦克风上行 + TTS 播放队列(Phase 1 收口)
右下角悬浮麦克风：点按说话→转写→提交任务→跳运行页→朗读回答，接入既有运行·观测流。 - lib/voice.ts: VoiceClient——一条 WS 承载上行 PCM16k/下行转写/下行 TTS PCM24k； ScriptProcessorNode 采麦(48k→16k 降采样+Float32→Int16)；AudioBufferSourceNode 排队调度做无缝连续朗读；start/end/barge_in/bye 控制；ready/transcript/task/speaking/tts_end - shell/VoiceDock.tsx: 悬浮麦克风 UI(聆听/思考/朗读态 + 转写气泡 + 打断)，懒建客户端(用户手势启 AudioContext) - App.tsx: onVoiceTask 把语音 task 挂回 attachRun(后端已提交,前端只 attach)；挂载 VoiceDock 真识别/合成需部署联调(真连火山+admin录入语音配置)；本地过 tsc/build/68 测试。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Blizzard
@@ -0,0 +1,241 @@
|
||||
// 语音会话客户端（JARVIS）：一条 WebSocket 承载上行麦克风音频 + 下行转写 + 下行 TTS 音频。
|
||||
// 协议与网关 internal/voice/protocol.go 对齐：
|
||||
// - 二进制帧：上行=麦克风 PCM 16k；下行=TTS PCM 24k
|
||||
// - 文本帧(JSON)：控制/事件（ClientMsg / ServerMsg）
|
||||
//
|
||||
// 麦克风采集用 ScriptProcessorNode（WKWebView 通吃，无需单独 worklet 文件），48k→16k 降采样、
|
||||
// Float32→Int16。下行播放用 AudioBufferSourceNode 排队调度，做无缝连续朗读。
|
||||
|
||||
import { GATEWAY, getToken } from "./api";
|
||||
|
||||
export type VoiceState = "idle" | "connecting" | "ready" | "listening" | "thinking" | "speaking";
|
||||
|
||||
export interface VoiceCallbacks {
|
||||
onState?: (s: VoiceState) => void;
|
||||
onTranscript?: (text: string, final: boolean) => void;
|
||||
onTask?: (taskId: string) => void;
|
||||
onError?: (msg: string) => void;
|
||||
}
|
||||
|
||||
const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率（与网关 voice.AudioSampleRate 一致）
|
||||
const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率（与网关 voice.TTSSampleRate 一致）
|
||||
|
||||
function wsURL(): string {
|
||||
const base = GATEWAY.replace(/^http/, "ws");
|
||||
const t = getToken();
|
||||
const q = t ? `?token=${encodeURIComponent(t)}` : "";
|
||||
return `${base}/api/v1/voice/stream${q}`;
|
||||
}
|
||||
|
||||
// float32 → int16 PCM（小端）。
|
||||
function floatToPCM16(f32: Float32Array): ArrayBuffer {
|
||||
const out = new DataView(new ArrayBuffer(f32.length * 2));
|
||||
for (let i = 0; i < f32.length; i++) {
|
||||
let s = Math.max(-1, Math.min(1, f32[i]));
|
||||
out.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
return out.buffer;
|
||||
}
|
||||
|
||||
// 线性降采样到 16k（浏览器 AudioContext 常跑 48k，火山 ASR 要 16k）。
|
||||
function downsampleTo16k(input: Float32Array, inRate: number): Float32Array {
|
||||
if (inRate === UP_SAMPLE_RATE) return input;
|
||||
const ratio = inRate / UP_SAMPLE_RATE;
|
||||
const outLen = Math.floor(input.length / ratio);
|
||||
const out = new Float32Array(outLen);
|
||||
for (let i = 0; i < outLen; i++) out[i] = input[Math.floor(i * ratio)];
|
||||
return out;
|
||||
}
|
||||
|
||||
export class VoiceClient {
|
||||
private ws: WebSocket | null = null;
|
||||
private cb: VoiceCallbacks;
|
||||
private state: VoiceState = "idle";
|
||||
|
||||
// 上行采集
|
||||
private micCtx: AudioContext | null = null;
|
||||
private micStream: MediaStream | null = null;
|
||||
private micNode: ScriptProcessorNode | null = null;
|
||||
|
||||
// 下行播放
|
||||
private playCtx: AudioContext | null = null;
|
||||
private nextStart = 0; // 下一段音频的调度起点（连续朗读用）
|
||||
private sources: AudioBufferSourceNode[] = [];
|
||||
|
||||
constructor(cb: VoiceCallbacks) {
|
||||
this.cb = cb;
|
||||
}
|
||||
|
||||
private setState(s: VoiceState) {
|
||||
this.state = s;
|
||||
this.cb.onState?.(s);
|
||||
}
|
||||
|
||||
getState(): VoiceState {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
// connect 建立 WS，等待服务端 ready。
|
||||
async connect(): Promise<void> {
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) return;
|
||||
this.setState("connecting");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ws = new WebSocket(wsURL());
|
||||
ws.binaryType = "arraybuffer";
|
||||
this.ws = ws;
|
||||
ws.onmessage = (e) => this.onMessage(e);
|
||||
ws.onerror = () => {
|
||||
this.cb.onError?.("语音连接出错");
|
||||
reject(new Error("ws error"));
|
||||
};
|
||||
ws.onclose = () => {
|
||||
this.stopMic();
|
||||
if (this.state !== "idle") this.setState("idle");
|
||||
};
|
||||
ws.onopen = () => resolve();
|
||||
});
|
||||
}
|
||||
|
||||
private onMessage(e: MessageEvent) {
|
||||
if (typeof e.data === "string") {
|
||||
let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string };
|
||||
try {
|
||||
m = JSON.parse(e.data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
switch (m.type) {
|
||||
case "ready":
|
||||
this.setState("ready");
|
||||
break;
|
||||
case "transcript":
|
||||
this.cb.onTranscript?.(m.text ?? "", !!m.final);
|
||||
if (m.final) this.setState("thinking");
|
||||
break;
|
||||
case "task":
|
||||
if (m.task_id) this.cb.onTask?.(m.task_id);
|
||||
break;
|
||||
case "speaking":
|
||||
this.setState("speaking");
|
||||
break;
|
||||
case "tts_end":
|
||||
this.resetPlayback();
|
||||
this.setState("ready");
|
||||
break;
|
||||
case "error":
|
||||
this.cb.onError?.(m.msg ?? "语音出错");
|
||||
break;
|
||||
}
|
||||
return;
|
||||
}
|
||||
// 二进制帧 = 下行 TTS 音频（PCM 24k int16）
|
||||
this.enqueueAudio(e.data as ArrayBuffer);
|
||||
}
|
||||
|
||||
// ---- 上行：麦克风 ----
|
||||
|
||||
// startListening 开一轮：连接（若需）→ 若正在朗读先打断 → 采麦克风 → 发 start。
|
||||
async startListening(graph?: string): Promise<void> {
|
||||
await this.connect();
|
||||
if (this.state === "speaking") this.bargeIn();
|
||||
this.send({ type: "start", graph });
|
||||
await this.startMic();
|
||||
this.setState("listening");
|
||||
}
|
||||
|
||||
// stopListening 结束本轮说话：停麦克风 + 发 end（服务端拿最终转写→提交任务）。
|
||||
stopListening(): void {
|
||||
this.stopMic();
|
||||
this.send({ type: "end" });
|
||||
if (this.state === "listening") this.setState("thinking");
|
||||
}
|
||||
|
||||
// bargeIn 打断当前朗读（用户又开口）。
|
||||
bargeIn(): void {
|
||||
this.send({ type: "barge_in" });
|
||||
this.resetPlayback();
|
||||
}
|
||||
|
||||
private async startMic(): Promise<void> {
|
||||
if (this.micNode) return;
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
||||
});
|
||||
this.micStream = stream;
|
||||
const ctx = new AudioContext();
|
||||
this.micCtx = ctx;
|
||||
const src = ctx.createMediaStreamSource(stream);
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
node.onaudioprocess = (ev) => {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||||
const f32 = ev.inputBuffer.getChannelData(0);
|
||||
const ds = downsampleTo16k(f32, ctx.sampleRate);
|
||||
this.ws.send(floatToPCM16(ds));
|
||||
};
|
||||
src.connect(node);
|
||||
node.connect(ctx.destination); // ScriptProcessor 需接上目的地才触发回调（下游静音）
|
||||
this.micNode = node;
|
||||
}
|
||||
|
||||
private stopMic(): void {
|
||||
this.micNode?.disconnect();
|
||||
this.micNode = null;
|
||||
this.micStream?.getTracks().forEach((t) => t.stop());
|
||||
this.micStream = null;
|
||||
this.micCtx?.close().catch(() => {});
|
||||
this.micCtx = null;
|
||||
}
|
||||
|
||||
// ---- 下行：TTS 音频播放（排队调度，无缝连续）----
|
||||
|
||||
private enqueueAudio(buf: ArrayBuffer): void {
|
||||
if (buf.byteLength === 0) return;
|
||||
if (!this.playCtx) this.playCtx = new AudioContext();
|
||||
const ctx = this.playCtx;
|
||||
const i16 = new Int16Array(buf);
|
||||
const f32 = new Float32Array(i16.length);
|
||||
for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 0x8000;
|
||||
const audioBuf = ctx.createBuffer(1, f32.length, DOWN_SAMPLE_RATE);
|
||||
audioBuf.getChannelData(0).set(f32);
|
||||
|
||||
const node = ctx.createBufferSource();
|
||||
node.buffer = audioBuf;
|
||||
node.connect(ctx.destination);
|
||||
const now = ctx.currentTime;
|
||||
if (this.nextStart < now) this.nextStart = now;
|
||||
node.start(this.nextStart);
|
||||
this.nextStart += audioBuf.duration;
|
||||
node.onended = () => {
|
||||
this.sources = this.sources.filter((s) => s !== node);
|
||||
};
|
||||
this.sources.push(node);
|
||||
}
|
||||
|
||||
private resetPlayback(): void {
|
||||
this.sources.forEach((s) => {
|
||||
try {
|
||||
s.stop();
|
||||
} catch {
|
||||
/* 已停忽略 */
|
||||
}
|
||||
});
|
||||
this.sources = [];
|
||||
this.nextStart = 0;
|
||||
}
|
||||
|
||||
private send(m: { type: string; graph?: string }): void {
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(m));
|
||||
}
|
||||
|
||||
// close 彻底关闭会话（发 bye、停采集/播放、断 WS）。
|
||||
close(): void {
|
||||
this.send({ type: "bye" });
|
||||
this.stopMic();
|
||||
this.resetPlayback();
|
||||
this.playCtx?.close().catch(() => {});
|
||||
this.playCtx = null;
|
||||
this.ws?.close();
|
||||
this.ws = null;
|
||||
this.setState("idle");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user