From 9458bf21f36865f2dce2fb8de269197c217eded9 Mon Sep 17 00:00:00 2001 From: Blizzard Date: Wed, 22 Jul 2026 09:29:47 +0800 Subject: [PATCH] =?UTF-8?q?feat(voice):=20=E6=A1=8C=E9=9D=A2=E7=AB=AF=20JA?= =?UTF-8?q?RVIS=20=E8=AF=AD=E9=9F=B3=E5=9D=9E=E2=80=94=E2=80=94=E9=BA=A6?= =?UTF-8?q?=E5=85=8B=E9=A3=8E=E4=B8=8A=E8=A1=8C=20+=20TTS=20=E6=92=AD?= =?UTF-8?q?=E6=94=BE=E9=98=9F=E5=88=97(Phase=201=20=E6=94=B6=E5=8F=A3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 右下角悬浮麦克风：点按说话→转写→提交任务→跳运行页→朗读回答，接入既有运行·观测流。 - lib/voice.ts: VoiceClient——一条 WS 承载上行 PCM16k/下行转写/下行 TTS PCM24k； ScriptProcessorNode 采麦(48k→16k 降采样+Float32→Int16)；AudioBufferSourceNode 排队调度做无缝连续朗读；start/end/barge_in/bye 控制；ready/transcript/task/speaking/tts_end - shell/VoiceDock.tsx: 悬浮麦克风 UI(聆听/思考/朗读态 + 转写气泡 + 打断)，懒建客户端(用户手势启 AudioContext) - App.tsx: onVoiceTask 把语音 task 挂回 attachRun(后端已提交,前端只 attach)；挂载 VoiceDock 真识别/合成需部署联调(真连火山+admin录入语音配置)；本地过 tsc/build/68 测试。 Co-Authored-By: Claude Opus 4.8 --- sundynix-desktop/frontend/src/App.tsx | 18 ++ sundynix-desktop/frontend/src/lib/voice.ts | 241 ++++++++++++++++++ .../frontend/src/shell/VoiceDock.tsx | 116 +++++++++ 3 files changed, 375 insertions(+) create mode 100644 sundynix-desktop/frontend/src/lib/voice.ts create mode 100644 sundynix-desktop/frontend/src/shell/VoiceDock.tsx diff --git a/sundynix-desktop/frontend/src/App.tsx b/sundynix-desktop/frontend/src/App.tsx index cf26d2f..824edfb 100644 --- a/sundynix-desktop/frontend/src/App.tsx +++ b/sundynix-desktop/frontend/src/App.tsx @@ -16,6 +16,7 @@ import { Home } from "./views/Home"; import { CommandPalette, type Command } from "./components/CommandPalette"; import { UpdateBanner } from "./components/UpdateBanner"; import { Login } from "./views/Login"; +import { VoiceDock } from "./shell/VoiceDock"; import { submitTask, generateReport, streamTokens, streamExec, taskStatus, listRuns, authMe, logout, tenantCurrent, myTenants, switchTenant, spaceCurrent, mySpaces, switchSpace, createSpace, type Identity, type AuthUser, type TenantCtx, type MyTenant, type SpaceCtx, type MySpace } from "./lib/api"; import type { TaskDsl } from "./lib/dsl"; import { emptyRun, type RunState } from "./lib/run"; @@ -283,6 +284,22 @@ export default function App() { [identity, attachRun], ); + // 语音任务：后端已在 ASR 转写后自行提交（走 SubmitTask 同一关卡），前端只需挂回它的运行流。 + // 与 onRun 的区别：不再 submit，直接 attach（task 已在后端跑）——跳「运行·观测」看轨迹/输出。 + const onVoiceTask = useCallback( + (taskId: string) => { + closeRef.current?.(); + execCloseRef.current?.(); + stopPoll(); + const t0 = Date.now(); + setRun({ phase: "streaming", taskId, output: "", events: [{ t: 0, label: `语音任务 ${taskId}` }], exec: [] }); + setFocusRun(null); + setView("runs"); + attachRun(taskId, t0, "语音任务已跑完"); + }, + [attachRun], + ); + // 恢复在途待审任务：登录后若存在 waiting 任务且当前无 live run，挂回它 → 全局审批条重现， // 用户刷新页面/重开 app 也能继续批准（HITL 持久化中断后审批可跨重启、可等数小时）。 const restoredRef = useRef(false); @@ -360,6 +377,7 @@ export default function App() { selfUserId={user?.id ?? ""} /> setInviteOpen(false)} tenantName={tenant?.tenant?.name ?? ""} /> + ); diff --git a/sundynix-desktop/frontend/src/lib/voice.ts b/sundynix-desktop/frontend/src/lib/voice.ts new file mode 100644 index 0000000..3e35011 --- /dev/null +++ b/sundynix-desktop/frontend/src/lib/voice.ts @@ -0,0 +1,241 @@ +// 语音会话客户端（JARVIS）：一条 WebSocket 承载上行麦克风音频 + 下行转写 + 下行 TTS 音频。 +// 协议与网关 internal/voice/protocol.go 对齐： +// - 二进制帧：上行=麦克风 PCM 16k；下行=TTS PCM 24k +// - 文本帧(JSON)：控制/事件（ClientMsg / ServerMsg） +// +// 麦克风采集用 ScriptProcessorNode（WKWebView 通吃，无需单独 worklet 文件），48k→16k 降采样、 +// Float32→Int16。下行播放用 AudioBufferSourceNode 排队调度，做无缝连续朗读。 + +import { GATEWAY, getToken } from "./api"; + +export type VoiceState = "idle" | "connecting" | "ready" | "listening" | "thinking" | "speaking"; + +export interface VoiceCallbacks { + onState?: (s: VoiceState) => void; + onTranscript?: (text: string, final: boolean) => void; + onTask?: (taskId: string) => void; + onError?: (msg: string) => void; +} + +const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率（与网关 voice.AudioSampleRate 一致） +const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率（与网关 voice.TTSSampleRate 一致） + +function wsURL(): string { + const base = GATEWAY.replace(/^http/, "ws"); + const t = getToken(); + const q = t ? `?token=${encodeURIComponent(t)}` : ""; + return `${base}/api/v1/voice/stream${q}`; +} + +// float32 → int16 PCM（小端）。 +function floatToPCM16(f32: Float32Array): ArrayBuffer { + const out = new DataView(new ArrayBuffer(f32.length * 2)); + for (let i = 0; i < f32.length; i++) { + let s = Math.max(-1, Math.min(1, f32[i])); + out.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } + return out.buffer; +} + +// 线性降采样到 16k（浏览器 AudioContext 常跑 48k，火山 ASR 要 16k）。 +function downsampleTo16k(input: Float32Array, inRate: number): Float32Array { + if (inRate === UP_SAMPLE_RATE) return input; + const ratio = inRate / UP_SAMPLE_RATE; + const outLen = Math.floor(input.length / ratio); + const out = new Float32Array(outLen); + for (let i = 0; i < outLen; i++) out[i] = input[Math.floor(i * ratio)]; + return out; +} + +export class VoiceClient { + private ws: WebSocket | null = null; + private cb: VoiceCallbacks; + private state: VoiceState = "idle"; + + // 上行采集 + private micCtx: AudioContext | null = null; + private micStream: MediaStream | null = null; + private micNode: ScriptProcessorNode | null = null; + + // 下行播放 + private playCtx: AudioContext | null = null; + private nextStart = 0; // 下一段音频的调度起点（连续朗读用） + private sources: AudioBufferSourceNode[] = []; + + constructor(cb: VoiceCallbacks) { + this.cb = cb; + } + + private setState(s: VoiceState) { + this.state = s; + this.cb.onState?.(s); + } + + getState(): VoiceState { + return this.state; + } + + // connect 建立 WS，等待服务端 ready。 + async connect(): Promise { + if (this.ws && this.ws.readyState === WebSocket.OPEN) return; + this.setState("connecting"); + await new Promise((resolve, reject) => { + const ws = new WebSocket(wsURL()); + ws.binaryType = "arraybuffer"; + this.ws = ws; + ws.onmessage = (e) => this.onMessage(e); + ws.onerror = () => { + this.cb.onError?.("语音连接出错"); + reject(new Error("ws error")); + }; + ws.onclose = () => { + this.stopMic(); + if (this.state !== "idle") this.setState("idle"); + }; + ws.onopen = () => resolve(); + }); + } + + private onMessage(e: MessageEvent) { + if (typeof e.data === "string") { + let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string }; + try { + m = JSON.parse(e.data); + } catch { + return; + } + switch (m.type) { + case "ready": + this.setState("ready"); + break; + case "transcript": + this.cb.onTranscript?.(m.text ?? "", !!m.final); + if (m.final) this.setState("thinking"); + break; + case "task": + if (m.task_id) this.cb.onTask?.(m.task_id); + break; + case "speaking": + this.setState("speaking"); + break; + case "tts_end": + this.resetPlayback(); + this.setState("ready"); + break; + case "error": + this.cb.onError?.(m.msg ?? "语音出错"); + break; + } + return; + } + // 二进制帧 = 下行 TTS 音频（PCM 24k int16） + this.enqueueAudio(e.data as ArrayBuffer); + } + + // ---- 上行：麦克风 ---- + + // startListening 开一轮：连接（若需）→ 若正在朗读先打断 → 采麦克风 → 发 start。 + async startListening(graph?: string): Promise { + await this.connect(); + if (this.state === "speaking") this.bargeIn(); + this.send({ type: "start", graph }); + await this.startMic(); + this.setState("listening"); + } + + // stopListening 结束本轮说话：停麦克风 + 发 end（服务端拿最终转写→提交任务）。 + stopListening(): void { + this.stopMic(); + this.send({ type: "end" }); + if (this.state === "listening") this.setState("thinking"); + } + + // bargeIn 打断当前朗读（用户又开口）。 + bargeIn(): void { + this.send({ type: "barge_in" }); + this.resetPlayback(); + } + + private async startMic(): Promise { + if (this.micNode) return; + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, + }); + this.micStream = stream; + const ctx = new AudioContext(); + this.micCtx = ctx; + const src = ctx.createMediaStreamSource(stream); + const node = ctx.createScriptProcessor(4096, 1, 1); + node.onaudioprocess = (ev) => { + if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; + const f32 = ev.inputBuffer.getChannelData(0); + const ds = downsampleTo16k(f32, ctx.sampleRate); + this.ws.send(floatToPCM16(ds)); + }; + src.connect(node); + node.connect(ctx.destination); // ScriptProcessor 需接上目的地才触发回调（下游静音） + this.micNode = node; + } + + private stopMic(): void { + this.micNode?.disconnect(); + this.micNode = null; + this.micStream?.getTracks().forEach((t) => t.stop()); + this.micStream = null; + this.micCtx?.close().catch(() => {}); + this.micCtx = null; + } + + // ---- 下行：TTS 音频播放（排队调度，无缝连续）---- + + private enqueueAudio(buf: ArrayBuffer): void { + if (buf.byteLength === 0) return; + if (!this.playCtx) this.playCtx = new AudioContext(); + const ctx = this.playCtx; + const i16 = new Int16Array(buf); + const f32 = new Float32Array(i16.length); + for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 0x8000; + const audioBuf = ctx.createBuffer(1, f32.length, DOWN_SAMPLE_RATE); + audioBuf.getChannelData(0).set(f32); + + const node = ctx.createBufferSource(); + node.buffer = audioBuf; + node.connect(ctx.destination); + const now = ctx.currentTime; + if (this.nextStart < now) this.nextStart = now; + node.start(this.nextStart); + this.nextStart += audioBuf.duration; + node.onended = () => { + this.sources = this.sources.filter((s) => s !== node); + }; + this.sources.push(node); + } + + private resetPlayback(): void { + this.sources.forEach((s) => { + try { + s.stop(); + } catch { + /* 已停忽略 */ + } + }); + this.sources = []; + this.nextStart = 0; + } + + private send(m: { type: string; graph?: string }): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(m)); + } + + // close 彻底关闭会话（发 bye、停采集/播放、断 WS）。 + close(): void { + this.send({ type: "bye" }); + this.stopMic(); + this.resetPlayback(); + this.playCtx?.close().catch(() => {}); + this.playCtx = null; + this.ws?.close(); + this.ws = null; + this.setState("idle"); + } +} diff --git a/sundynix-desktop/frontend/src/shell/VoiceDock.tsx b/sundynix-desktop/frontend/src/shell/VoiceDock.tsx new file mode 100644 index 0000000..65d9df0 --- /dev/null +++ b/sundynix-desktop/frontend/src/shell/VoiceDock.tsx @@ -0,0 +1,116 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Mic, Square, Loader2, Volume2, X } from "lucide-react"; +import { VoiceClient, type VoiceState } from "../lib/voice"; +import { useToast } from "../ui/Toast"; +import { cn } from "../ui/cn"; + +// JARVIS 语音坞：右下角悬浮麦克风。点按说话 → 转写 → 提交任务 → 跳运行页 → 朗读回答。 +// onTask 把语音触发的 task_id 交回 App，接入既有运行·观测流（与键盘提交同一条路）。 + +interface Props { + onTask: (taskId: string) => void; +} + +const HINT: Record = { + idle: "点击说话", + connecting: "连接中…", + ready: "点击说话", + listening: "聆听中，再点结束", + thinking: "思考中…", + speaking: "朗读中，点击打断", +}; + +export function VoiceDock({ onTask }: Props) { + const toast = useToast(); + const clientRef = useRef(null); + const [state, setState] = useState("idle"); + const [transcript, setTranscript] = useState(""); + const [open, setOpen] = useState(false); // 是否展开转写气泡 + + // 懒建客户端（首次点按时，带上用户手势→AudioContext 才能启动）。 + const ensureClient = useCallback((): VoiceClient => { + if (!clientRef.current) { + clientRef.current = new VoiceClient({ + onState: setState, + onTranscript: (text, final) => { + setTranscript(text); + if (final) setOpen(true); + }, + onTask: (taskId) => { + onTask(taskId); + setOpen(false); + }, + onError: (msg) => toast.push("error", msg), + }); + } + return clientRef.current; + }, [onTask, toast]); + + useEffect(() => () => clientRef.current?.close(), []); + + const onMic = useCallback(async () => { + const c = ensureClient(); + try { + if (state === "listening") { + c.stopListening(); + } else { + setTranscript(""); + setOpen(true); + await c.startListening(); // speaking 中会先打断再开新一轮 + } + } catch (e) { + toast.push("error", (e as Error).message || "麦克风启动失败（检查权限）"); + } + }, [ensureClient, state, toast]); + + const active = state === "listening"; + const busy = state === "connecting" || state === "thinking"; + + return ( +
+ {/* 转写气泡 */} + {open && transcript && ( +
+
+ {transcript} + +
+
+ )} + + {/* 麦克风按钮 */} + + + {/* 状态提示 */} + + {HINT[state]} + +
+ ); +}