fix(voice): PTT 改为主交互——按住说话松开发送(修 PTT 形同虚设的 bug)
VAD 自动断句在真实环境不够稳(环境音/停顿都会误触发),改回按住说话。 修掉 PTT 之前根本不生效的三处: - keydown 里 `if (!c) return`——用户没先点过麦克风按钮时 clientRef 为 null, 按空格什么都不发生。改为懒建客户端(keydown 本身就是用户手势,可启 AudioContext) - 被 `!inConversation()` 挡着:一旦点过麦进了连续对话,PTT 永久失效。改为 PTT 优先, 必要时先退出对话模式 - 缺 e.repeat 守卫:按住不放会连发 keydown 交互统一为一套 PTT 语义: - 按住空格 / 按住麦克风按钮(pointer 事件,覆盖鼠标触控) → 说话,松开发送 - 朗读中按按钮 = 打断 - 指针滑出/取消/窗口失焦都算松开,不会卡在录音态 - 录音中按钮变红缩小 + 电平条 + 呼吸环,一眼可见"正在听" 顺带:松开后标签改为"已停止收音"(准确说法——麦克风硬件仍开着以便快速重按, 只是不再上传音频,门控在 shouldSendMic);删掉已无人使用的 micTapAction 及其单测。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { micTapAction, nextAfterDrain, shouldSendMic, type VoiceState } from "./voice";
|
import { nextAfterDrain, shouldSendMic, type VoiceState } from "./voice";
|
||||||
|
|
||||||
// 连续对话的纯决策函数:状态门控 / 播完去向 / 点按语义。
|
// 连续对话的纯决策函数:状态门控 / 播完去向 / 点按语义。
|
||||||
// VoiceClient 本体依赖 WebSocket/AudioContext(jsdom 难直测),决策逻辑抽纯函数在这测。
|
// VoiceClient 本体依赖 WebSocket/AudioContext(jsdom 难直测),决策逻辑抽纯函数在这测。
|
||||||
@@ -27,22 +27,3 @@ describe("nextAfterDrain 播完去向", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("micTapAction 点按语义(对话开关)", () => {
|
|
||||||
it("聆听中点 → 退出对话", () => {
|
|
||||||
expect(micTapAction("listening", true)).toBe("stop");
|
|
||||||
expect(micTapAction("listening", false)).toBe("stop");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("朗读中点(对话里)→ 打断但留在对话", () => {
|
|
||||||
expect(micTapAction("speaking", true)).toBe("interrupt");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("朗读中点(非对话)→ 开始对话(startConversation 内部先打断)", () => {
|
|
||||||
expect(micTapAction("speaking", false)).toBe("start");
|
|
||||||
});
|
|
||||||
|
|
||||||
it.each<VoiceState>(["idle", "ready", "connecting", "thinking"])("%s 态点 → 开始对话", (s) => {
|
|
||||||
expect(micTapAction(s, false)).toBe("start");
|
|
||||||
expect(micTapAction(s, true)).toBe("start");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|||||||
@@ -34,15 +34,6 @@ export function nextAfterDrain(conversation: boolean): VoiceState {
|
|||||||
return conversation ? "listening" : "ready";
|
return conversation ? "listening" : "ready";
|
||||||
}
|
}
|
||||||
|
|
||||||
// micTapAction:点按麦克风的语义(对话开关)。
|
|
||||||
// 聆听中点 → 退出对话;朗读中点(在对话里)→ 打断但留在对话;其余 → 开始对话。
|
|
||||||
export type MicTapAction = "start" | "stop" | "interrupt";
|
|
||||||
export function micTapAction(state: VoiceState, inConversation: boolean): MicTapAction {
|
|
||||||
if (state === "listening") return "stop";
|
|
||||||
if (state === "speaking" && inConversation) return "interrupt";
|
|
||||||
return "start";
|
|
||||||
}
|
|
||||||
|
|
||||||
// 健壮性定时器时长(仅对话模式生效)
|
// 健壮性定时器时长(仅对话模式生效)
|
||||||
const IDLE_EXIT_MS = 30_000; // 聆听空转:30s 无任何转写 → 自动退出对话(防 ASR 长连接白烧计费)
|
const IDLE_EXIT_MS = 30_000; // 聆听空转:30s 无任何转写 → 自动退出对话(防 ASR 长连接白烧计费)
|
||||||
const THINK_WATCHDOG_MS = 90_000; // 思考看门狗:final 后 90s 没等到朗读 → 回聆听继续对话
|
const THINK_WATCHDOG_MS = 90_000; // 思考看门狗:final 后 90s 没等到朗读 → 回聆听继续对话
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ interface Props {
|
|||||||
transcript: string;
|
transcript: string;
|
||||||
reply: string;
|
reply: string;
|
||||||
hint: string;
|
hint: string;
|
||||||
onMic: () => void;
|
// 按住说话(与语音坞/空格键同一套 PTT 语义):按下开始听、松开发送。
|
||||||
|
onPttDown: () => void;
|
||||||
|
onPttUp: () => void;
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,7 +32,7 @@ const CONF: Record<VoiceState, Conf> = {
|
|||||||
};
|
};
|
||||||
const GLYPH = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%@*<>/\\";
|
const GLYPH = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%@*<>/\\";
|
||||||
|
|
||||||
export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMic, onClose }: Props) {
|
export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onPttDown, onPttUp, onClose }: Props) {
|
||||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||||
const decRef = useRef<HTMLSpanElement>(null);
|
const decRef = useRef<HTMLSpanElement>(null);
|
||||||
const sigRef = useRef<HTMLElement>(null);
|
const sigRef = useRef<HTMLElement>(null);
|
||||||
@@ -210,7 +212,15 @@ export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMi
|
|||||||
<X className="h-4 w-4" /> ESC
|
<X className="h-4 w-4" /> ESC
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
<button className="jhud-core" onClick={onMic} aria-label={hint} title={hint} />
|
<button
|
||||||
|
className="jhud-core"
|
||||||
|
onPointerDown={(e) => { e.preventDefault(); onPttDown(); }}
|
||||||
|
onPointerUp={onPttUp}
|
||||||
|
onPointerLeave={onPttUp}
|
||||||
|
onPointerCancel={onPttUp}
|
||||||
|
aria-label={hint}
|
||||||
|
title={hint}
|
||||||
|
/>
|
||||||
<div className={"jhud-word" + (amber ? " amber" : state === "listening" ? " live" : "")}>{hint}</div>
|
<div className={"jhud-word" + (amber ? " amber" : state === "listening" ? " live" : "")}>{hint}</div>
|
||||||
|
|
||||||
<div className="jhud-decode">
|
<div className="jhud-decode">
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
import { useCallback, useEffect, useRef, useState } from "react";
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
import { ExternalLink, Loader2, Mic, MicOff, Settings2, Maximize2, Volume2, X } from "lucide-react";
|
import { ExternalLink, Loader2, Mic, MicOff, Settings2, Maximize2, Volume2, X } from "lucide-react";
|
||||||
import { VoiceClient, micTapAction, type VoiceState } from "../lib/voice";
|
import { VoiceClient, type VoiceState } from "../lib/voice";
|
||||||
import { JarvisSettings } from "./JarvisSettings";
|
import { JarvisSettings } from "./JarvisSettings";
|
||||||
import { JarvisHud } from "./JarvisHud";
|
import { JarvisHud } from "./JarvisHud";
|
||||||
import { getMyJarvis } from "../lib/api";
|
import { getMyJarvis } from "../lib/api";
|
||||||
import { useToast } from "../ui/Toast";
|
import { useToast } from "../ui/Toast";
|
||||||
import { cn } from "../ui/cn";
|
import { cn } from "../ui/cn";
|
||||||
|
|
||||||
// JARVIS 语音坞:右下角悬浮麦克风,连续对话模式(点一次进对话,VAD 自动断句、答完自动重听)。
|
// JARVIS 语音坞:右下角悬浮麦克风。**按住说话、松开发送(PTT)**——按住空格或按住按钮均可。
|
||||||
|
// VAD 自动断句在真实环境不够稳(环境音/停顿误触发),故不再默认进连续对话模式。
|
||||||
// 气泡是「迷你对话流」——保留最近几轮,旧轮淡化,新轮追加(不清屏丢上下文)。
|
// 气泡是「迷你对话流」——保留最近几轮,旧轮淡化,新轮追加(不清屏丢上下文)。
|
||||||
// onTask 不再每轮强制跳运行页(连续对话会被拽走):任务收进对话流里的芯片,点击才跳。
|
// onTask 不再每轮强制跳运行页(连续对话会被拽走):任务收进对话流里的芯片,点击才跳。
|
||||||
|
|
||||||
@@ -30,18 +31,18 @@ const KEEP_TURNS = 3; // 对话流保留最近几轮(含当前轮)
|
|||||||
// pttActive 时显示"松开发送"替代默认聆听提示。
|
// pttActive 时显示"松开发送"替代默认聆听提示。
|
||||||
function hintText(state: VoiceState, name: string, idleLeft: number | null, ptt?: boolean): string {
|
function hintText(state: VoiceState, name: string, idleLeft: number | null, ptt?: boolean): string {
|
||||||
if (idleLeft !== null) return `${idleLeft}s 后自动退出 · 说话取消`;
|
if (idleLeft !== null) return `${idleLeft}s 后自动退出 · 说话取消`;
|
||||||
if (ptt && state === "listening") return "按住说话中 · 松开发送";
|
if (ptt) return "正在听 · 松开发送";
|
||||||
switch (state) {
|
switch (state) {
|
||||||
case "connecting":
|
case "connecting":
|
||||||
return "连接中…";
|
return "连接中…";
|
||||||
case "listening":
|
case "listening":
|
||||||
return "聆听中 · 说完自动发送 · 点击退出";
|
return "聆听中…";
|
||||||
case "thinking":
|
case "thinking":
|
||||||
return `${name} 正在思考`;
|
return `${name} 正在思考`;
|
||||||
case "speaking":
|
case "speaking":
|
||||||
return "朗读中 · 点击打断";
|
return "朗读中 · 点击打断";
|
||||||
default:
|
default:
|
||||||
return "点击开始对话 · 空格键按住说话";
|
return "按住空格 或 按住这里说话";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,7 +186,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
return () => window.removeEventListener("keydown", onKey);
|
return () => window.removeEventListener("keydown", onKey);
|
||||||
}, [fullscreen]);
|
}, [fullscreen]);
|
||||||
|
|
||||||
// PTT:按住空格语音输入,松开发送(不在对话模式时生效)。
|
// PTT:按住空格说话,松开发送——**这是默认且主要的交互方式**。
|
||||||
|
// VAD 自动断句在真实环境里不够稳(环境音/停顿都会误触发),所以不再自动进连续对话。
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const onKeyDown = (e: KeyboardEvent) => {
|
const onKeyDown = (e: KeyboardEvent) => {
|
||||||
if (e.key !== " " && e.code !== "Space") return;
|
if (e.key !== " " && e.code !== "Space") return;
|
||||||
@@ -193,9 +195,14 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
const tag = (e.target as HTMLElement).tagName;
|
const tag = (e.target as HTMLElement).tagName;
|
||||||
if (tag === "INPUT" || tag === "TEXTAREA" || (e.target as HTMLElement).isContentEditable) return;
|
if (tag === "INPUT" || tag === "TEXTAREA" || (e.target as HTMLElement).isContentEditable) return;
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
|
if (e.repeat) return; // 按住不放会连发 keydown,只认第一次
|
||||||
|
|
||||||
const c = clientRef.current;
|
// 关键:懒建客户端。此前这里是 `if (!c) return`——用户没先点过麦克风按钮时
|
||||||
if (!c || c.inConversation() || c.getState() === "listening") return;
|
// clientRef 为 null,按空格**什么都不会发生**(PTT 形同虚设)。
|
||||||
|
// keydown 本身就是用户手势,可以合法启动 AudioContext。
|
||||||
|
const c = ensureClient();
|
||||||
|
if (pttRef.current || c.getState() === "listening") return;
|
||||||
|
if (c.inConversation()) c.stopConversation(); // 如果之前进了连续对话,PTT 优先,先退出
|
||||||
|
|
||||||
pttRef.current = true;
|
pttRef.current = true;
|
||||||
setPttActive(true);
|
setPttActive(true);
|
||||||
@@ -238,30 +245,37 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
if (c && c.getState() === "listening") c.stopPTT();
|
if (c && c.getState() === "listening") c.stopPTT();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}, [toast]);
|
}, [ensureClient, toast]);
|
||||||
|
|
||||||
// 点按语义 = 对话开关:待命点 → 进入连续对话;聆听中点 → 退出;朗读中点 → 打断但留在对话。
|
// 麦克风按钮 = 按住说话(与空格键同义)。按下开始听、松开发送——
|
||||||
const onMic = useCallback(async () => {
|
// 和 PTT 统一,别让按钮和空格键两套语义打架。
|
||||||
|
const pttDown = useCallback(() => {
|
||||||
const c = ensureClient();
|
const c = ensureClient();
|
||||||
try {
|
if (pttRef.current || c.getState() === "listening") return;
|
||||||
switch (micTapAction(state, c.inConversation())) {
|
if (c.inConversation()) c.stopConversation();
|
||||||
case "stop":
|
pttRef.current = true;
|
||||||
c.stopConversation();
|
setPttActive(true);
|
||||||
break;
|
setOpen(true);
|
||||||
case "interrupt":
|
c.startListening().catch((err: unknown) => {
|
||||||
c.interruptAndListen();
|
pttRef.current = false;
|
||||||
break;
|
setPttActive(false);
|
||||||
case "start":
|
toast.push("error", (err as Error).message || "麦克风启动失败(检查权限)");
|
||||||
setOpen(true); // 清屏交给 onTurnStart(startConversation 里触发)
|
});
|
||||||
await c.startConversation();
|
}, [ensureClient, toast]);
|
||||||
break;
|
|
||||||
}
|
const pttUp = useCallback(() => {
|
||||||
} catch (e) {
|
if (!pttRef.current) return;
|
||||||
toast.push("error", (e as Error).message || "麦克风启动失败(检查权限)");
|
pttRef.current = false;
|
||||||
} finally {
|
setPttActive(false);
|
||||||
setConv(clientRef.current?.inConversation() ?? false);
|
const c = clientRef.current;
|
||||||
}
|
if (c && c.getState() === "listening") c.stopPTT();
|
||||||
}, [ensureClient, state, toast]);
|
}, []);
|
||||||
|
|
||||||
|
// 朗读中点一下 = 打断(此时不进入录音,只是掐掉 TTS)。
|
||||||
|
const onInterrupt = useCallback(() => {
|
||||||
|
const c = clientRef.current;
|
||||||
|
if (c && c.getState() === "speaking") c.bargeIn();
|
||||||
|
}, []);
|
||||||
|
|
||||||
// 进全屏 JARVIS 模式:建好客户端(HUD 读实时电平),刷一次助手名。
|
// 进全屏 JARVIS 模式:建好客户端(HUD 读实时电平),刷一次助手名。
|
||||||
const openFullscreen = useCallback(() => {
|
const openFullscreen = useCallback(() => {
|
||||||
@@ -275,7 +289,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
const busy = state === "connecting";
|
const busy = state === "connecting";
|
||||||
const hint = hintText(state, name, idleLeft, pttActive);
|
const hint = hintText(state, name, idleLeft, pttActive);
|
||||||
const visibleTurns = turns.filter((t) => t.me || t.ai || t.taskId);
|
const visibleTurns = turns.filter((t) => t.me || t.ai || t.taskId);
|
||||||
const ringDur = conv ? RING_DUR[state] : undefined;
|
// 呼吸环:PTT 录音中、思考中、朗读中都亮(只变节奏);待命时熄灭。
|
||||||
|
const ringDur = pttActive ? RING_DUR.listening : RING_DUR[state];
|
||||||
// 倒计时环:SVG 周长 182.2(r=29),剩余秒数映射到 dashoffset(收缩)。
|
// 倒计时环:SVG 周长 182.2(r=29),剩余秒数映射到 dashoffset(收缩)。
|
||||||
const cdOffset = idleLeft !== null ? (182.2 * (10 - idleLeft)) / 10 : 0;
|
const cdOffset = idleLeft !== null ? (182.2 * (10 - idleLeft)) / 10 : 0;
|
||||||
|
|
||||||
@@ -351,25 +366,38 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
|
|
||||||
{/* 电平/静默指示 + 麦克风按钮 */}
|
{/* 电平/静默指示 + 麦克风按钮 */}
|
||||||
<div className="pointer-events-auto flex items-center gap-2.5">
|
<div className="pointer-events-auto flex items-center gap-2.5">
|
||||||
{conv && (state === "listening" || state === "speaking") && (
|
{(pttActive || state === "speaking") && (
|
||||||
<VuBars getLevel={() => clientRef.current?.level() ?? 0} active />
|
<VuBars getLevel={() => clientRef.current?.level() ?? 0} active />
|
||||||
)}
|
)}
|
||||||
{conv && state === "thinking" && (
|
{state === "thinking" && (
|
||||||
<span className="flex items-center gap-1 rounded-full bg-ink-900/80 px-2.5 py-1 text-[10px] text-slate-400">
|
<span className="flex items-center gap-1 rounded-full bg-ink-900/80 px-2.5 py-1 text-[10px] text-slate-400">
|
||||||
<MicOff className="h-3 w-3" />
|
<MicOff className="h-3 w-3" />
|
||||||
麦克风已静默
|
已停止收音
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
<button
|
<button
|
||||||
onClick={onMic}
|
// 按住说话:pointer 事件覆盖鼠标/触控/触控笔;朗读中按一下则是打断。
|
||||||
|
onPointerDown={(e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
if (state === "speaking") {
|
||||||
|
onInterrupt();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pttDown();
|
||||||
|
}}
|
||||||
|
onPointerUp={pttUp}
|
||||||
|
onPointerLeave={pttUp} // 按住时指针滑出按钮也算松开,别卡在录音态
|
||||||
|
onPointerCancel={pttUp}
|
||||||
title={hint}
|
title={hint}
|
||||||
aria-label={hint}
|
aria-label={hint}
|
||||||
className={cn(
|
className={cn(
|
||||||
"relative flex h-14 w-14 items-center justify-center rounded-full shadow-xl transition",
|
"relative flex h-14 w-14 select-none items-center justify-center rounded-full shadow-xl transition",
|
||||||
"focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
|
"focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
|
||||||
conv && state === "thinking"
|
pttActive
|
||||||
? "bg-ink-800 text-accent-400"
|
? "scale-95 bg-danger text-white" // 录音中:红 + 缩,一眼可见"正在听"
|
||||||
: "bg-brand text-white hover:bg-brand-500 active:scale-95",
|
: state === "thinking"
|
||||||
|
? "bg-ink-800 text-accent-400"
|
||||||
|
: "bg-brand text-white hover:bg-brand-500",
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
{/* 呼吸环:对话进行中常亮(三种状态只变节奏)——环在=点击是退出/打断,环灭=点击是开始 */}
|
{/* 呼吸环:对话进行中常亮(三种状态只变节奏)——环在=点击是退出/打断,环灭=点击是开始 */}
|
||||||
@@ -420,7 +448,8 @@ export function VoiceDock({ onTask, onNavigate }: Props) {
|
|||||||
transcript={visibleTurns[visibleTurns.length - 1]?.me ?? ""}
|
transcript={visibleTurns[visibleTurns.length - 1]?.me ?? ""}
|
||||||
reply={visibleTurns[visibleTurns.length - 1]?.ai ?? ""}
|
reply={visibleTurns[visibleTurns.length - 1]?.ai ?? ""}
|
||||||
hint={hint}
|
hint={hint}
|
||||||
onMic={onMic}
|
onPttDown={pttDown}
|
||||||
|
onPttUp={pttUp}
|
||||||
onClose={() => setFullscreen(false)}
|
onClose={() => setFullscreen(false)}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|||||||
Reference in New Issue
Block a user