sundynix-desktop/frontend/src/App.tsx

@@ -16,6 +16,7 @@ import { Home } from "./views/Home";
 import { CommandPalette, type Command } from "./components/CommandPalette";
 import { UpdateBanner } from "./components/UpdateBanner";
 import { Login } from "./views/Login";
+import { VoiceDock } from "./shell/VoiceDock";
 import { submitTask, generateReport, streamTokens, streamExec, taskStatus, listRuns, authMe, logout, tenantCurrent, myTenants, switchTenant, spaceCurrent, mySpaces, switchSpace, createSpace, type Identity, type AuthUser, type TenantCtx, type MyTenant, type SpaceCtx, type MySpace } from "./lib/api";
 import type { TaskDsl } from "./lib/dsl";
 import { emptyRun, type RunState } from "./lib/run";
@@ -283,6 +284,22 @@ export default function App() {
     [identity, attachRun],
   );
 
+  // 语音任务：后端已在 ASR 转写后自行提交（走 SubmitTask 同一关卡），前端只需挂回它的运行流。
+  // 与 onRun 的区别：不再 submit，直接 attach（task 已在后端跑）——跳「运行·观测」看轨迹/输出。
+  const onVoiceTask = useCallback(
+    (taskId: string) => {
+      closeRef.current?.();
+      execCloseRef.current?.();
+      stopPoll();
+      const t0 = Date.now();
+      setRun({ phase: "streaming", taskId, output: "", events: [{ t: 0, label: `语音任务 ${taskId}` }], exec: [] });
+      setFocusRun(null);
+      setView("runs");
+      attachRun(taskId, t0, "语音任务已跑完");
+    },
+    [attachRun],
+  );
+
   // 恢复在途待审任务：登录后若存在 waiting 任务且当前无 live run，挂回它 → 全局审批条重现，
   // 用户刷新页面/重开 app 也能继续批准（HITL 持久化中断后审批可跨重启、可等数小时）。
   const restoredRef = useRef(false);
@@ -360,6 +377,7 @@ export default function App() {
         selfUserId={user?.id ?? ""}
       />
       <InviteMembers open={inviteOpen} onClose={() => setInviteOpen(false)} tenantName={tenant?.tenant?.name ?? ""} />
+      <VoiceDock onTask={onVoiceTask} />
     </div>
     </ToastProvider>
   );

sundynix-desktop/frontend/src/lib/voice.ts

@@ -0,0 +1,241 @@
+// 语音会话客户端（JARVIS）：一条 WebSocket 承载上行麦克风音频 + 下行转写 + 下行 TTS 音频。
+// 协议与网关 internal/voice/protocol.go 对齐：
+//   - 二进制帧：上行=麦克风 PCM 16k；下行=TTS PCM 24k
+//   - 文本帧(JSON)：控制/事件（ClientMsg / ServerMsg）
+//
+// 麦克风采集用 ScriptProcessorNode（WKWebView 通吃，无需单独 worklet 文件），48k→16k 降采样、
+// Float32→Int16。下行播放用 AudioBufferSourceNode 排队调度，做无缝连续朗读。
+
+import { GATEWAY, getToken } from "./api";
+
+export type VoiceState = "idle" | "connecting" | "ready" | "listening" | "thinking" | "speaking";
+
+export interface VoiceCallbacks {
+  onState?: (s: VoiceState) => void;
+  onTranscript?: (text: string, final: boolean) => void;
+  onTask?: (taskId: string) => void;
+  onError?: (msg: string) => void;
+}
+
+const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率（与网关 voice.AudioSampleRate 一致）
+const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率（与网关 voice.TTSSampleRate 一致）
+
+function wsURL(): string {
+  const base = GATEWAY.replace(/^http/, "ws");
+  const t = getToken();
+  const q = t ? `?token=${encodeURIComponent(t)}` : "";
+  return `${base}/api/v1/voice/stream${q}`;
+}
+
+// float32 → int16 PCM（小端）。
+function floatToPCM16(f32: Float32Array): ArrayBuffer {
+  const out = new DataView(new ArrayBuffer(f32.length * 2));
+  for (let i = 0; i < f32.length; i++) {
+    let s = Math.max(-1, Math.min(1, f32[i]));
+    out.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+  return out.buffer;
+}
+
+// 线性降采样到 16k（浏览器 AudioContext 常跑 48k，火山 ASR 要 16k）。
+function downsampleTo16k(input: Float32Array, inRate: number): Float32Array {
+  if (inRate === UP_SAMPLE_RATE) return input;
+  const ratio = inRate / UP_SAMPLE_RATE;
+  const outLen = Math.floor(input.length / ratio);
+  const out = new Float32Array(outLen);
+  for (let i = 0; i < outLen; i++) out[i] = input[Math.floor(i * ratio)];
+  return out;
+}
+
+export class VoiceClient {
+  private ws: WebSocket | null = null;
+  private cb: VoiceCallbacks;
+  private state: VoiceState = "idle";
+
+  // 上行采集
+  private micCtx: AudioContext | null = null;
+  private micStream: MediaStream | null = null;
+  private micNode: ScriptProcessorNode | null = null;
+
+  // 下行播放
+  private playCtx: AudioContext | null = null;
+  private nextStart = 0; // 下一段音频的调度起点（连续朗读用）
+  private sources: AudioBufferSourceNode[] = [];
+
+  constructor(cb: VoiceCallbacks) {
+    this.cb = cb;
+  }
+
+  private setState(s: VoiceState) {
+    this.state = s;
+    this.cb.onState?.(s);
+  }
+
+  getState(): VoiceState {
+    return this.state;
+  }
+
+  // connect 建立 WS，等待服务端 ready。
+  async connect(): Promise<void> {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) return;
+    this.setState("connecting");
+    await new Promise<void>((resolve, reject) => {
+      const ws = new WebSocket(wsURL());
+      ws.binaryType = "arraybuffer";
+      this.ws = ws;
+      ws.onmessage = (e) => this.onMessage(e);
+      ws.onerror = () => {
+        this.cb.onError?.("语音连接出错");
+        reject(new Error("ws error"));
+      };
+      ws.onclose = () => {
+        this.stopMic();
+        if (this.state !== "idle") this.setState("idle");
+      };
+      ws.onopen = () => resolve();
+    });
+  }
+
+  private onMessage(e: MessageEvent) {
+    if (typeof e.data === "string") {
+      let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string };
+      try {
+        m = JSON.parse(e.data);
+      } catch {
+        return;
+      }
+      switch (m.type) {
+        case "ready":
+          this.setState("ready");
+          break;
+        case "transcript":
+          this.cb.onTranscript?.(m.text ?? "", !!m.final);
+          if (m.final) this.setState("thinking");
+          break;
+        case "task":
+          if (m.task_id) this.cb.onTask?.(m.task_id);
+          break;
+        case "speaking":
+          this.setState("speaking");
+          break;
+        case "tts_end":
+          this.resetPlayback();
+          this.setState("ready");
+          break;
+        case "error":
+          this.cb.onError?.(m.msg ?? "语音出错");
+          break;
+      }
+      return;
+    }
+    // 二进制帧 = 下行 TTS 音频（PCM 24k int16）
+    this.enqueueAudio(e.data as ArrayBuffer);
+  }
+
+  // ---- 上行：麦克风 ----
+
+  // startListening 开一轮：连接（若需）→ 若正在朗读先打断 → 采麦克风 → 发 start。
+  async startListening(graph?: string): Promise<void> {
+    await this.connect();
+    if (this.state === "speaking") this.bargeIn();
+    this.send({ type: "start", graph });
+    await this.startMic();
+    this.setState("listening");
+  }
+
+  // stopListening 结束本轮说话：停麦克风 + 发 end（服务端拿最终转写→提交任务）。
+  stopListening(): void {
+    this.stopMic();
+    this.send({ type: "end" });
+    if (this.state === "listening") this.setState("thinking");
+  }
+
+  // bargeIn 打断当前朗读（用户又开口）。
+  bargeIn(): void {
+    this.send({ type: "barge_in" });
+    this.resetPlayback();
+  }
+
+  private async startMic(): Promise<void> {
+    if (this.micNode) return;
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
+    });
+    this.micStream = stream;
+    const ctx = new AudioContext();
+    this.micCtx = ctx;
+    const src = ctx.createMediaStreamSource(stream);
+    const node = ctx.createScriptProcessor(4096, 1, 1);
+    node.onaudioprocess = (ev) => {
+      if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+      const f32 = ev.inputBuffer.getChannelData(0);
+      const ds = downsampleTo16k(f32, ctx.sampleRate);
+      this.ws.send(floatToPCM16(ds));
+    };
+    src.connect(node);
+    node.connect(ctx.destination); // ScriptProcessor 需接上目的地才触发回调（下游静音）
+    this.micNode = node;
+  }
+
+  private stopMic(): void {
+    this.micNode?.disconnect();
+    this.micNode = null;
+    this.micStream?.getTracks().forEach((t) => t.stop());
+    this.micStream = null;
+    this.micCtx?.close().catch(() => {});
+    this.micCtx = null;
+  }
+
+  // ---- 下行：TTS 音频播放（排队调度，无缝连续）----
+
+  private enqueueAudio(buf: ArrayBuffer): void {
+    if (buf.byteLength === 0) return;
+    if (!this.playCtx) this.playCtx = new AudioContext();
+    const ctx = this.playCtx;
+    const i16 = new Int16Array(buf);
+    const f32 = new Float32Array(i16.length);
+    for (let i = 0; i < i16.length; i++) f32[i] = i16[i] / 0x8000;
+    const audioBuf = ctx.createBuffer(1, f32.length, DOWN_SAMPLE_RATE);
+    audioBuf.getChannelData(0).set(f32);
+
+    const node = ctx.createBufferSource();
+    node.buffer = audioBuf;
+    node.connect(ctx.destination);
+    const now = ctx.currentTime;
+    if (this.nextStart < now) this.nextStart = now;
+    node.start(this.nextStart);
+    this.nextStart += audioBuf.duration;
+    node.onended = () => {
+      this.sources = this.sources.filter((s) => s !== node);
+    };
+    this.sources.push(node);
+  }
+
+  private resetPlayback(): void {
+    this.sources.forEach((s) => {
+      try {
+        s.stop();
+      } catch {
+        /* 已停忽略 */
+      }
+    });
+    this.sources = [];
+    this.nextStart = 0;
+  }
+
+  private send(m: { type: string; graph?: string }): void {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) this.ws.send(JSON.stringify(m));
+  }
+
+  // close 彻底关闭会话（发 bye、停采集/播放、断 WS）。
+  close(): void {
+    this.send({ type: "bye" });
+    this.stopMic();
+    this.resetPlayback();
+    this.playCtx?.close().catch(() => {});
+    this.playCtx = null;
+    this.ws?.close();
+    this.ws = null;
+    this.setState("idle");
+  }
+}

sundynix-desktop/frontend/src/shell/VoiceDock.tsx

@@ -0,0 +1,116 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { Mic, Square, Loader2, Volume2, X } from "lucide-react";
+import { VoiceClient, type VoiceState } from "../lib/voice";
+import { useToast } from "../ui/Toast";
+import { cn } from "../ui/cn";
+
+// JARVIS 语音坞：右下角悬浮麦克风。点按说话 → 转写 → 提交任务 → 跳运行页 → 朗读回答。
+// onTask 把语音触发的 task_id 交回 App，接入既有运行·观测流（与键盘提交同一条路）。
+
+interface Props {
+  onTask: (taskId: string) => void;
+}
+
+const HINT: Record<VoiceState, string> = {
+  idle: "点击说话",
+  connecting: "连接中…",
+  ready: "点击说话",
+  listening: "聆听中，再点结束",
+  thinking: "思考中…",
+  speaking: "朗读中，点击打断",
+};
+
+export function VoiceDock({ onTask }: Props) {
+  const toast = useToast();
+  const clientRef = useRef<VoiceClient | null>(null);
+  const [state, setState] = useState<VoiceState>("idle");
+  const [transcript, setTranscript] = useState("");
+  const [open, setOpen] = useState(false); // 是否展开转写气泡
+
+  // 懒建客户端（首次点按时，带上用户手势→AudioContext 才能启动）。
+  const ensureClient = useCallback((): VoiceClient => {
+    if (!clientRef.current) {
+      clientRef.current = new VoiceClient({
+        onState: setState,
+        onTranscript: (text, final) => {
+          setTranscript(text);
+          if (final) setOpen(true);
+        },
+        onTask: (taskId) => {
+          onTask(taskId);
+          setOpen(false);
+        },
+        onError: (msg) => toast.push("error", msg),
+      });
+    }
+    return clientRef.current;
+  }, [onTask, toast]);
+
+  useEffect(() => () => clientRef.current?.close(), []);
+
+  const onMic = useCallback(async () => {
+    const c = ensureClient();
+    try {
+      if (state === "listening") {
+        c.stopListening();
+      } else {
+        setTranscript("");
+        setOpen(true);
+        await c.startListening(); // speaking 中会先打断再开新一轮
+      }
+    } catch (e) {
+      toast.push("error", (e as Error).message || "麦克风启动失败（检查权限）");
+    }
+  }, [ensureClient, state, toast]);
+
+  const active = state === "listening";
+  const busy = state === "connecting" || state === "thinking";
+
+  return (
+    <div className="pointer-events-none fixed bottom-6 right-6 z-40 flex flex-col items-end gap-2">
+      {/* 转写气泡 */}
+      {open && transcript && (
+        <div className="pointer-events-auto max-w-xs rounded-2xl border border-line bg-ink-850/95 px-4 py-2.5 text-sm text-slate-200 shadow-xl backdrop-blur">
+          <div className="flex items-start gap-2">
+            <span className="flex-1 leading-relaxed">{transcript}</span>
+            <button className="mt-0.5 text-slate-500 hover:text-slate-300" onClick={() => setOpen(false)} aria-label="收起">
+              <X className="h-3.5 w-3.5" />
+            </button>
+          </div>
+        </div>
+      )}
+
+      {/* 麦克风按钮 */}
+      <button
+        onClick={onMic}
+        title={HINT[state]}
+        aria-label={HINT[state]}
+        className={cn(
+          "pointer-events-auto relative flex h-14 w-14 items-center justify-center rounded-full shadow-xl transition",
+          "focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
+          active
+            ? "bg-danger text-white"
+            : state === "speaking"
+              ? "bg-brand text-white"
+              : "bg-brand text-white hover:bg-brand-500 active:scale-95",
+        )}
+      >
+        {active && <span className="absolute inset-0 animate-ping rounded-full bg-danger/40" />}
+        {busy ? (
+          <Loader2 className="h-6 w-6 animate-spin" />
+        ) : state === "speaking" ? (
+          <Volume2 className="h-6 w-6" />
+        ) : active ? (
+          <Square className="h-5 w-5" />
+        ) : (
+          <Mic className="h-6 w-6" />
+        )}
+      </button>
+
+      {/* 状态提示 */}
+      <span className="pointer-events-none rounded-full bg-ink-900/80 px-2.5 py-0.5 text-[11px] text-slate-400">
+        {HINT[state]}
+      </span>
+    </div>
+  );
+}