fix(voice): PTT 改为主交互——按住说话松开发送(修 PTT 形同虚设的 bug)

VAD 自动断句在真实环境不够稳(环境音/停顿都会误触发),改回按住说话。

修掉 PTT 之前根本不生效的三处:
- keydown 里 `if (!c) return`——用户没先点过麦克风按钮时 clientRef 为 null,
  按空格什么都不发生。改为懒建客户端(keydown 本身就是用户手势,可启 AudioContext)
- 被 `!inConversation()` 挡着:一旦点过麦进了连续对话,PTT 永久失效。改为 PTT 优先,
  必要时先退出对话模式
- 缺 e.repeat 守卫:按住不放会连发 keydown

交互统一为一套 PTT 语义:
- 按住空格 / 按住麦克风按钮(pointer 事件,覆盖鼠标触控) → 说话,松开发送
- 朗读中按按钮 = 打断
- 指针滑出/取消/窗口失焦都算松开,不会卡在录音态
- 录音中按钮变红缩小 + 电平条 + 呼吸环,一眼可见"正在听"

顺带:松开后标签改为"已停止收音"(准确说法——麦克风硬件仍开着以便快速重按,
只是不再上传音频,门控在 shouldSendMic);删掉已无人使用的 micTapAction 及其单测。

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Blizzard
2026-07-25 14:52:26 +08:00
parent 08c4d94187
commit e60679ed7b
4 changed files with 83 additions and 72 deletions
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { micTapAction, nextAfterDrain, shouldSendMic, type VoiceState } from "./voice"; import { nextAfterDrain, shouldSendMic, type VoiceState } from "./voice";
// 连续对话的纯决策函数:状态门控 / 播完去向 / 点按语义。 // 连续对话的纯决策函数:状态门控 / 播完去向 / 点按语义。
// VoiceClient 本体依赖 WebSocket/AudioContextjsdom 难直测),决策逻辑抽纯函数在这测。 // VoiceClient 本体依赖 WebSocket/AudioContextjsdom 难直测),决策逻辑抽纯函数在这测。
@@ -27,22 +27,3 @@ describe("nextAfterDrain 播完去向", () => {
}); });
}); });
describe("micTapAction 点按语义(对话开关)", () => {
it("聆听中点 → 退出对话", () => {
expect(micTapAction("listening", true)).toBe("stop");
expect(micTapAction("listening", false)).toBe("stop");
});
it("朗读中点(对话里)→ 打断但留在对话", () => {
expect(micTapAction("speaking", true)).toBe("interrupt");
});
it("朗读中点(非对话)→ 开始对话(startConversation 内部先打断)", () => {
expect(micTapAction("speaking", false)).toBe("start");
});
it.each<VoiceState>(["idle", "ready", "connecting", "thinking"])("%s 态点 → 开始对话", (s) => {
expect(micTapAction(s, false)).toBe("start");
expect(micTapAction(s, true)).toBe("start");
});
});
@@ -34,15 +34,6 @@ export function nextAfterDrain(conversation: boolean): VoiceState {
return conversation ? "listening" : "ready"; return conversation ? "listening" : "ready";
} }
// micTapAction:点按麦克风的语义(对话开关)。
// 聆听中点 → 退出对话;朗读中点(在对话里)→ 打断但留在对话;其余 → 开始对话。
export type MicTapAction = "start" | "stop" | "interrupt";
export function micTapAction(state: VoiceState, inConversation: boolean): MicTapAction {
if (state === "listening") return "stop";
if (state === "speaking" && inConversation) return "interrupt";
return "start";
}
// 健壮性定时器时长(仅对话模式生效) // 健壮性定时器时长(仅对话模式生效)
const IDLE_EXIT_MS = 30_000; // 聆听空转:30s 无任何转写 → 自动退出对话(防 ASR 长连接白烧计费) const IDLE_EXIT_MS = 30_000; // 聆听空转:30s 无任何转写 → 自动退出对话(防 ASR 长连接白烧计费)
const THINK_WATCHDOG_MS = 90_000; // 思考看门狗:final 后 90s 没等到朗读 → 回聆听继续对话 const THINK_WATCHDOG_MS = 90_000; // 思考看门狗:final 后 90s 没等到朗读 → 回聆听继续对话
@@ -13,7 +13,9 @@ interface Props {
transcript: string; transcript: string;
reply: string; reply: string;
hint: string; hint: string;
onMic: () => void; // 按住说话(与语音坞/空格键同一套 PTT 语义):按下开始听、松开发送。
onPttDown: () => void;
onPttUp: () => void;
onClose: () => void; onClose: () => void;
} }
@@ -30,7 +32,7 @@ const CONF: Record<VoiceState, Conf> = {
}; };
const GLYPH = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%@*<>/\\"; const GLYPH = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%@*<>/\\";
export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMic, onClose }: Props) { export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onPttDown, onPttUp, onClose }: Props) {
const canvasRef = useRef<HTMLCanvasElement>(null); const canvasRef = useRef<HTMLCanvasElement>(null);
const decRef = useRef<HTMLSpanElement>(null); const decRef = useRef<HTMLSpanElement>(null);
const sigRef = useRef<HTMLElement>(null); const sigRef = useRef<HTMLElement>(null);
@@ -210,7 +212,15 @@ export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMi
<X className="h-4 w-4" /> ESC <X className="h-4 w-4" /> ESC
</button> </button>
<button className="jhud-core" onClick={onMic} aria-label={hint} title={hint} /> <button
className="jhud-core"
onPointerDown={(e) => { e.preventDefault(); onPttDown(); }}
onPointerUp={onPttUp}
onPointerLeave={onPttUp}
onPointerCancel={onPttUp}
aria-label={hint}
title={hint}
/>
<div className={"jhud-word" + (amber ? " amber" : state === "listening" ? " live" : "")}>{hint}</div> <div className={"jhud-word" + (amber ? " amber" : state === "listening" ? " live" : "")}>{hint}</div>
<div className="jhud-decode"> <div className="jhud-decode">
@@ -1,13 +1,14 @@
import { useCallback, useEffect, useRef, useState } from "react"; import { useCallback, useEffect, useRef, useState } from "react";
import { ExternalLink, Loader2, Mic, MicOff, Settings2, Maximize2, Volume2, X } from "lucide-react"; import { ExternalLink, Loader2, Mic, MicOff, Settings2, Maximize2, Volume2, X } from "lucide-react";
import { VoiceClient, micTapAction, type VoiceState } from "../lib/voice"; import { VoiceClient, type VoiceState } from "../lib/voice";
import { JarvisSettings } from "./JarvisSettings"; import { JarvisSettings } from "./JarvisSettings";
import { JarvisHud } from "./JarvisHud"; import { JarvisHud } from "./JarvisHud";
import { getMyJarvis } from "../lib/api"; import { getMyJarvis } from "../lib/api";
import { useToast } from "../ui/Toast"; import { useToast } from "../ui/Toast";
import { cn } from "../ui/cn"; import { cn } from "../ui/cn";
// JARVIS 语音坞:右下角悬浮麦克风,连续对话模式(点一次进对话,VAD 自动断句、答完自动重听) // JARVIS 语音坞:右下角悬浮麦克风。**按住说话、松开发送(PTT)**——按住空格或按住按钮均可
// VAD 自动断句在真实环境不够稳(环境音/停顿误触发),故不再默认进连续对话模式。
// 气泡是「迷你对话流」——保留最近几轮,旧轮淡化,新轮追加(不清屏丢上下文)。 // 气泡是「迷你对话流」——保留最近几轮,旧轮淡化,新轮追加(不清屏丢上下文)。
// onTask 不再每轮强制跳运行页(连续对话会被拽走):任务收进对话流里的芯片,点击才跳。 // onTask 不再每轮强制跳运行页(连续对话会被拽走):任务收进对话流里的芯片,点击才跳。
@@ -30,18 +31,18 @@ const KEEP_TURNS = 3; // 对话流保留最近几轮(含当前轮)
// pttActive 时显示"松开发送"替代默认聆听提示。 // pttActive 时显示"松开发送"替代默认聆听提示。
function hintText(state: VoiceState, name: string, idleLeft: number | null, ptt?: boolean): string { function hintText(state: VoiceState, name: string, idleLeft: number | null, ptt?: boolean): string {
if (idleLeft !== null) return `${idleLeft}s 后自动退出 · 说话取消`; if (idleLeft !== null) return `${idleLeft}s 后自动退出 · 说话取消`;
if (ptt && state === "listening") return "按住说话中 · 松开发送"; if (ptt) return "正在听 · 松开发送";
switch (state) { switch (state) {
case "connecting": case "connecting":
return "连接中…"; return "连接中…";
case "listening": case "listening":
return "聆听中 · 说完自动发送 · 点击退出"; return "聆听中";
case "thinking": case "thinking":
return `${name} 正在思考`; return `${name} 正在思考`;
case "speaking": case "speaking":
return "朗读中 · 点击打断"; return "朗读中 · 点击打断";
default: default:
return "点击开始对话 · 空格键按住说话"; return "按住空格 或 按住这里说话";
} }
} }
@@ -185,7 +186,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
return () => window.removeEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey);
}, [fullscreen]); }, [fullscreen]);
// PTT:按住空格语音输入,松开发送(不在对话模式时生效) // PTT:按住空格说话,松开发送——**这是默认且主要的交互方式**
// VAD 自动断句在真实环境里不够稳(环境音/停顿都会误触发),所以不再自动进连续对话。
useEffect(() => { useEffect(() => {
const onKeyDown = (e: KeyboardEvent) => { const onKeyDown = (e: KeyboardEvent) => {
if (e.key !== " " && e.code !== "Space") return; if (e.key !== " " && e.code !== "Space") return;
@@ -193,9 +195,14 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
const tag = (e.target as HTMLElement).tagName; const tag = (e.target as HTMLElement).tagName;
if (tag === "INPUT" || tag === "TEXTAREA" || (e.target as HTMLElement).isContentEditable) return; if (tag === "INPUT" || tag === "TEXTAREA" || (e.target as HTMLElement).isContentEditable) return;
e.preventDefault(); e.preventDefault();
if (e.repeat) return; // 按住不放会连发 keydown,只认第一次
const c = clientRef.current; // 关键:懒建客户端。此前这里是 `if (!c) return`——用户没先点过麦克风按钮时
if (!c || c.inConversation() || c.getState() === "listening") return; // clientRef 为 null,按空格**什么都不会发生**(PTT 形同虚设)。
// keydown 本身就是用户手势,可以合法启动 AudioContext。
const c = ensureClient();
if (pttRef.current || c.getState() === "listening") return;
if (c.inConversation()) c.stopConversation(); // 如果之前进了连续对话,PTT 优先,先退出
pttRef.current = true; pttRef.current = true;
setPttActive(true); setPttActive(true);
@@ -238,30 +245,37 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
if (c && c.getState() === "listening") c.stopPTT(); if (c && c.getState() === "listening") c.stopPTT();
} }
}; };
}, [toast]); }, [ensureClient, toast]);
// 点按语义 = 对话开关:待命点 → 进入连续对话;聆听中点 → 退出;朗读中点 → 打断但留在对话。 // 麦克风按钮 = 按住说话(与空格键同义)。按下开始听、松开发送——
const onMic = useCallback(async () => { // 和 PTT 统一,别让按钮和空格键两套语义打架。
const pttDown = useCallback(() => {
const c = ensureClient(); const c = ensureClient();
try { if (pttRef.current || c.getState() === "listening") return;
switch (micTapAction(state, c.inConversation())) { if (c.inConversation()) c.stopConversation();
case "stop": pttRef.current = true;
c.stopConversation(); setPttActive(true);
break; setOpen(true);
case "interrupt": c.startListening().catch((err: unknown) => {
c.interruptAndListen(); pttRef.current = false;
break; setPttActive(false);
case "start": toast.push("error", (err as Error).message || "麦克风启动失败(检查权限)");
setOpen(true); // 清屏交给 onTurnStartstartConversation 里触发) });
await c.startConversation(); }, [ensureClient, toast]);
break;
} const pttUp = useCallback(() => {
} catch (e) { if (!pttRef.current) return;
toast.push("error", (e as Error).message || "麦克风启动失败(检查权限)"); pttRef.current = false;
} finally { setPttActive(false);
setConv(clientRef.current?.inConversation() ?? false); const c = clientRef.current;
} if (c && c.getState() === "listening") c.stopPTT();
}, [ensureClient, state, toast]); }, []);
// 朗读中点一下 = 打断(此时不进入录音,只是掐掉 TTS)。
const onInterrupt = useCallback(() => {
const c = clientRef.current;
if (c && c.getState() === "speaking") c.bargeIn();
}, []);
// 进全屏 JARVIS 模式:建好客户端(HUD 读实时电平),刷一次助手名。 // 进全屏 JARVIS 模式:建好客户端(HUD 读实时电平),刷一次助手名。
const openFullscreen = useCallback(() => { const openFullscreen = useCallback(() => {
@@ -275,7 +289,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
const busy = state === "connecting"; const busy = state === "connecting";
const hint = hintText(state, name, idleLeft, pttActive); const hint = hintText(state, name, idleLeft, pttActive);
const visibleTurns = turns.filter((t) => t.me || t.ai || t.taskId); const visibleTurns = turns.filter((t) => t.me || t.ai || t.taskId);
const ringDur = conv ? RING_DUR[state] : undefined; // 呼吸环:PTT 录音中、思考中、朗读中都亮(只变节奏);待命时熄灭。
const ringDur = pttActive ? RING_DUR.listening : RING_DUR[state];
// 倒计时环:SVG 周长 182.2r=29),剩余秒数映射到 dashoffset(收缩)。 // 倒计时环:SVG 周长 182.2r=29),剩余秒数映射到 dashoffset(收缩)。
const cdOffset = idleLeft !== null ? (182.2 * (10 - idleLeft)) / 10 : 0; const cdOffset = idleLeft !== null ? (182.2 * (10 - idleLeft)) / 10 : 0;
@@ -351,25 +366,38 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
{/* 电平/静默指示 + 麦克风按钮 */} {/* 电平/静默指示 + 麦克风按钮 */}
<div className="pointer-events-auto flex items-center gap-2.5"> <div className="pointer-events-auto flex items-center gap-2.5">
{conv && (state === "listening" || state === "speaking") && ( {(pttActive || state === "speaking") && (
<VuBars getLevel={() => clientRef.current?.level() ?? 0} active /> <VuBars getLevel={() => clientRef.current?.level() ?? 0} active />
)} )}
{conv && state === "thinking" && ( {state === "thinking" && (
<span className="flex items-center gap-1 rounded-full bg-ink-900/80 px-2.5 py-1 text-[10px] text-slate-400"> <span className="flex items-center gap-1 rounded-full bg-ink-900/80 px-2.5 py-1 text-[10px] text-slate-400">
<MicOff className="h-3 w-3" /> <MicOff className="h-3 w-3" />
</span> </span>
)} )}
<button <button
onClick={onMic} // 按住说话:pointer 事件覆盖鼠标/触控/触控笔;朗读中按一下则是打断。
onPointerDown={(e) => {
e.preventDefault();
if (state === "speaking") {
onInterrupt();
return;
}
pttDown();
}}
onPointerUp={pttUp}
onPointerLeave={pttUp} // 按住时指针滑出按钮也算松开,别卡在录音态
onPointerCancel={pttUp}
title={hint} title={hint}
aria-label={hint} aria-label={hint}
className={cn( className={cn(
"relative flex h-14 w-14 items-center justify-center rounded-full shadow-xl transition", "relative flex h-14 w-14 select-none items-center justify-center rounded-full shadow-xl transition",
"focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60", "focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
conv && state === "thinking" pttActive
? "bg-ink-800 text-accent-400" ? "scale-95 bg-danger text-white" // 录音中:红 + 缩,一眼可见"正在听"
: "bg-brand text-white hover:bg-brand-500 active:scale-95", : state === "thinking"
? "bg-ink-800 text-accent-400"
: "bg-brand text-white hover:bg-brand-500",
)} )}
> >
{/* 呼吸环:对话进行中常亮(三种状态只变节奏)——环在=点击是退出/打断,环灭=点击是开始 */} {/* 呼吸环:对话进行中常亮(三种状态只变节奏)——环在=点击是退出/打断,环灭=点击是开始 */}
@@ -420,7 +448,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
transcript={visibleTurns[visibleTurns.length - 1]?.me ?? ""} transcript={visibleTurns[visibleTurns.length - 1]?.me ?? ""}
reply={visibleTurns[visibleTurns.length - 1]?.ai ?? ""} reply={visibleTurns[visibleTurns.length - 1]?.ai ?? ""}
hint={hint} hint={hint}
onMic={onMic} onPttDown={pttDown}
onPttUp={pttUp}
onClose={() => setFullscreen(false)} onClose={() => setFullscreen(false)}
/> />
)} )}