feat(desktop): 全屏「JARVIS 模式」HUD——钢铁侠风,接真实语音音频

弧反应堆核心 + 同心旋转环 + 随真实声音反应的环形频谱 + 状态编排 + HUD 边框/扫描线/网格。
可视化数据全来自真实语音会话(不是模拟)。

- voice.ts: micCtx/playCtx 各挂 AnalyserNode + level()——说话读下行TTS、其余读麦克风,
  实时电平 0..1(HUD 每帧读,复用缓冲)
- JarvisHud.tsx: Canvas 2D HUD,按 VoiceState 编排(待命呼吸/聆听炸开/思考散粒子+雷达/说话频谱),
  单一青色强调+思考态琥珀;显示助手名(星期五)+ 我说的 + 打字机回答;尊重 prefers-reduced-motion
- VoiceDock: 加"全屏 JARVIS 模式"按钮(建客户端+拉名字);Esc 退出

真机验证:全屏 HUD 正常渲染,STANDBY 呼吸,品牌位显示自定义名"星期五"。晚上点中心说话即随真声反应。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Blizzard
2026-07-22 14:28:39 +08:00
parent 24351014e0
commit 2a743a33f6
3 changed files with 318 additions and 11 deletions
+31 -1
View File
@@ -57,11 +57,14 @@ export class VoiceClient {
private micCtx: AudioContext | null = null;
private micStream: MediaStream | null = null;
private micNode: ScriptProcessorNode | null = null;
private micAnalyser: AnalyserNode | null = null;
// 下行播放
private playCtx: AudioContext | null = null;
private nextStart = 0; // 下一段音频的调度起点(连续朗读用)
private sources: AudioBufferSourceNode[] = [];
private playAnalyser: AnalyserNode | null = null;
private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲(level() 每帧读,别每帧新建)
constructor(cb: VoiceCallbacks) {
this.cb = cb;
@@ -153,10 +156,31 @@ export class VoiceClient {
// ensurePlayCtx 建/复用下行播放 AudioContext,并在 suspended 时 resume(自动播放策略要求手势内唤醒)。
private ensurePlayCtx(): AudioContext {
if (!this.playCtx) this.playCtx = new AudioContext();
if (!this.playAnalyser) {
// 下行电平探针:所有 TTS 音频经它再到扬声器(驱动 HUD 的"说话"频谱)。
const an = this.playCtx.createAnalyser();
an.fftSize = 512;
an.connect(this.playCtx.destination);
this.playAnalyser = an;
}
if (this.playCtx.state === "suspended") void this.playCtx.resume();
return this.playCtx;
}
// level 返回当前"活跃"音频电平 0..1:说话读下行 TTS,其余读上行麦克风;无探针则 0。
// 供 JARVIS HUD 每帧读取,让可视化随真实声音起伏。
level(): number {
const a = this.state === "speaking" ? this.playAnalyser : this.micAnalyser;
if (!a) return 0;
a.getByteTimeDomainData(this.lvlBuf);
let s = 0;
for (let i = 0; i < this.lvlBuf.length; i++) {
const v = (this.lvlBuf[i] - 128) / 128;
s += v * v;
}
return Math.min(1, Math.sqrt(s / this.lvlBuf.length) * 3.2);
}
// stopListening 结束本轮说话:停麦克风 + 发 end(服务端拿最终转写→提交任务)。
stopListening(): void {
this.stopMic();
@@ -180,6 +204,10 @@ export class VoiceClient {
this.micCtx = ctx;
if (ctx.state === "suspended") await ctx.resume(); // 防采集上下文挂起(无回调=不上行音频)
const src = ctx.createMediaStreamSource(stream);
const an = ctx.createAnalyser(); // 上行电平探针(驱动 JARVIS HUD 的"聆听"反应)
an.fftSize = 512;
src.connect(an);
this.micAnalyser = an;
const node = ctx.createScriptProcessor(4096, 1, 1);
node.onaudioprocess = (ev) => {
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
@@ -195,6 +223,7 @@ export class VoiceClient {
private stopMic(): void {
this.micNode?.disconnect();
this.micNode = null;
this.micAnalyser = null;
this.micStream?.getTracks().forEach((t) => t.stop());
this.micStream = null;
this.micCtx?.close().catch(() => {});
@@ -215,7 +244,7 @@ export class VoiceClient {
const node = ctx.createBufferSource();
node.buffer = audioBuf;
node.connect(ctx.destination);
node.connect(this.playAnalyser ?? ctx.destination); // 经探针再到扬声器(读得到电平)
const now = ctx.currentTime;
if (this.nextStart < now) this.nextStart = now;
node.start(this.nextStart);
@@ -249,6 +278,7 @@ export class VoiceClient {
this.resetPlayback();
this.playCtx?.close().catch(() => {});
this.playCtx = null;
this.playAnalyser = null;
this.ws?.close();
this.ws = null;
this.setState("idle");
@@ -0,0 +1,234 @@
import { useEffect, useRef } from "react";
import { X } from "lucide-react";
import type { VoiceState } from "../lib/voice";
// 全屏「JARVIS 模式」HUD:弧反应堆核心 + 随真实声音反应的环形频谱 + 状态编排。
// 视觉数据全来自真实语音会话:level 由 VoiceClient 的麦克风/TTS 探针实时给出(getLevel),
// state 是会话状态机;transcript/reply 是当轮问答。点中心说话,右上角退出。
interface Props {
name: string; // 助手名(HUD 品牌位)
state: VoiceState;
getLevel: () => number; // 0..1 实时电平
transcript: string; // 我说的
reply: string; // JARVIS 回答(流式)
hint: string; // 当前状态提示
onMic: () => void; // 点中心:开始/结束说话
onClose: () => void;
}
type Conf = { accent: number[]; activity: number; ringSpin: number; label: string };
const CY = [56, 225, 255];
const AM = [255, 182, 56];
const CONF: Record<VoiceState, Conf> = {
idle: { accent: CY, activity: 0.18, ringSpin: 0.1, label: "STANDBY" },
connecting: { accent: CY, activity: 0.22, ringSpin: 0.5, label: "LINKING" },
ready: { accent: CY, activity: 0.18, ringSpin: 0.1, label: "READY" },
listening: { accent: CY, activity: 0.6, ringSpin: 0.35, label: "LISTENING" },
thinking: { accent: AM, activity: 0.3, ringSpin: 0.8, label: "PROCESSING" },
speaking: { accent: CY, activity: 0.85, ringSpin: 0.22, label: "SPEAKING" },
};
export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMic, onClose }: Props) {
const canvasRef = useRef<HTMLCanvasElement>(null);
// 用 ref 让 RAF 闭包读到最新 props,无需重启动画。
const P = useRef({ state, getLevel, transcript, reply });
P.current = { state, getLevel, transcript, reply };
useEffect(() => {
const cv = canvasRef.current!;
const ctx = cv.getContext("2d")!;
const reduce = matchMedia("(prefers-reduced-motion: reduce)").matches;
let W = 0, H = 0, raf = 0;
const resize = () => {
const r = cv.getBoundingClientRect();
const dpr = Math.min(2, devicePixelRatio || 1);
W = r.width; H = r.height;
cv.width = W * dpr; cv.height = H * dpr;
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
};
resize();
const ro = new ResizeObserver(resize);
ro.observe(cv);
const rgba = (c: number[], a: number) => `rgba(${c[0]},${c[1]},${c[2]},${a})`;
const lerp = (a: number, b: number, t: number) => a + (b - a) * t;
const accent = CY.slice();
let level = 0.18, think = 0, t = 0, last = performance.now();
const parts = Array.from({ length: 64 }, () => ({ a: Math.random() * 6.28, r: Math.random(), s: 0.3 + Math.random() * 0.9 }));
const synth = (s: VoiceState) =>
s === "thinking" ? 0.2 + 0.06 * Math.sin(t * 3) : 0.16 + 0.05 * Math.sin(t * 1.6);
const draw = (now: number) => {
const dt = Math.min(0.05, (now - last) / 1000); last = now; t += dt;
const st = P.current.state;
const c = CONF[st] ?? CONF.idle;
const active = st === "listening" || st === "speaking";
const tgt = active ? Math.max(P.current.getLevel(), 0) : synth(st);
level += (tgt - level) * Math.min(1, dt * (active ? 16 : 6));
for (let i = 0; i < 3; i++) accent[i] = lerp(accent[i], c.accent[i], Math.min(1, dt * 4));
think += ((st === "thinking" ? 1 : 0) - think) * Math.min(1, dt * 3);
const cx = W / 2, cy = H / 2, R = Math.min(W, H) * 0.16;
ctx.clearRect(0, 0, W, H);
ctx.globalCompositeOperation = "lighter";
let g = ctx.createRadialGradient(cx, cy, R * 0.2, cx, cy, R * 3.4);
g.addColorStop(0, rgba(accent, 0.1 + level * 0.1));
g.addColorStop(1, rgba(accent, 0));
ctx.fillStyle = g; ctx.fillRect(cx - R * 4, cy - R * 4, R * 8, R * 8);
const spin = reduce ? 0 : t * c.ringSpin;
[1.35, 1.7, 2.05, 2.5, 3.0].forEach((rr, i) => {
const rad = R * rr, dir = i % 2 ? -1 : 1, a0 = spin * dir * (1 + i * 0.25);
ctx.lineWidth = i === 0 ? 2.2 : 1.2;
ctx.strokeStyle = rgba(accent, (0.5 - i * 0.06) * (0.6 + level * 0.5));
ctx.shadowColor = rgba(accent, 0.8); ctx.shadowBlur = 12;
const segs: [number, number][] = i === 0 ? [[0, 5.0]] : [[a0, 1.3], [a0 + 2.3, 2.0], [a0 + 5.0, 0.8]];
segs.forEach(([s, len]) => { ctx.beginPath(); ctx.arc(cx, cy, rad, s, s + len); ctx.stroke(); });
});
ctx.shadowBlur = 0;
const NB = 104, inner = R * 1.12;
ctx.lineWidth = 2;
for (let i = 0; i < NB; i++) {
const ang = (i / NB) * 6.28 - Math.PI / 2 + spin * 0.15;
const seed = Math.sin(i * 12.9898) * 43758.5453; const nz = seed - Math.floor(seed);
const wob = 0.5 + 0.5 * Math.sin(t * 3 + i * 0.5);
const h = R * (0.06 + (0.1 + c.activity * 0.9 * level) * (0.35 + 0.65 * wob) * (0.5 + nz));
const x1 = cx + Math.cos(ang) * inner, y1 = cy + Math.sin(ang) * inner;
const x2 = cx + Math.cos(ang) * (inner + h), y2 = cy + Math.sin(ang) * (inner + h);
const bright = 0.35 + 0.65 * (h / (R * 0.9));
ctx.strokeStyle = rgba(accent, 0.25 + bright * 0.6);
ctx.beginPath(); ctx.moveTo(x1, y1); ctx.lineTo(x2, y2); ctx.stroke();
}
if (think > 0.02) {
parts.forEach((p) => {
if (!reduce) p.a += dt * p.s * 1.2;
const rad = R * (1.15 + p.r * 1.7), x = cx + Math.cos(p.a) * rad, y = cy + Math.sin(p.a) * rad;
ctx.fillStyle = rgba(AM, 0.7 * think);
ctx.shadowColor = rgba(AM, 0.9); ctx.shadowBlur = 8;
ctx.beginPath(); ctx.arc(x, y, 1.6, 0, 6.28); ctx.fill();
});
const sa = t * 2.2;
ctx.save(); ctx.beginPath(); ctx.moveTo(cx, cy); ctx.arc(cx, cy, R * 2.9, sa, sa + 0.5); ctx.closePath();
ctx.fillStyle = rgba(AM, 0.1 * think); ctx.fill(); ctx.restore();
ctx.shadowBlur = 0;
}
const cr = R * (0.62 + level * 0.3);
const cg = ctx.createRadialGradient(cx, cy, 0, cx, cy, cr);
cg.addColorStop(0, rgba([255, 255, 255], 0.95));
cg.addColorStop(0.25, rgba(accent, 0.95));
cg.addColorStop(0.7, rgba(accent, 0.28));
cg.addColorStop(1, rgba(accent, 0));
ctx.fillStyle = cg; ctx.beginPath(); ctx.arc(cx, cy, cr, 0, 6.28); ctx.fill();
ctx.globalCompositeOperation = "source-over";
for (let k = 0; k < 3; k++) {
ctx.lineWidth = 1; ctx.strokeStyle = rgba([255, 255, 255], 0.5 - k * 0.13);
ctx.beginPath(); ctx.arc(cx, cy, R * (0.3 + k * 0.14), 0, 6.28); ctx.stroke();
}
ctx.globalCompositeOperation = "lighter";
ctx.strokeStyle = rgba([255, 255, 255], 0.85); ctx.lineWidth = 1.5; ctx.beginPath();
for (let k = 0; k < 3; k++) {
const a = spin * 2 + k * 2.094, rr = R * 0.22;
const x = cx + Math.cos(a) * rr, y = cy + Math.sin(a) * rr;
k ? ctx.lineTo(x, y) : ctx.moveTo(x, y);
}
ctx.closePath(); ctx.stroke();
ctx.globalCompositeOperation = "source-over";
raf = requestAnimationFrame(draw);
};
raf = requestAnimationFrame(draw);
return () => { cancelAnimationFrame(raf); ro.disconnect(); };
}, []);
const c = CONF[state] ?? CONF.idle;
const amber = state === "thinking";
return (
<div className="jhud" role="dialog" aria-label="JARVIS 全屏模式">
<style>{HUD_CSS}</style>
<canvas ref={canvasRef} className="jhud-canvas" />
<span className="jhud-corner tl" /><span className="jhud-corner tr" />
<span className="jhud-corner bl" /><span className="jhud-corner br" />
<div className="jhud-status">
<div className="jhud-brand">{(name || "JARVIS").toUpperCase()}</div>
<div className="jhud-row">
<span className={"jhud-dot" + (state === "listening" ? " live" : amber ? " amber" : "")} />
<b>{c.label}</b>
</div>
</div>
<button className="jhud-close" onClick={onClose} aria-label="退出 JARVIS 模式">
<X className="h-4 w-4" /> ESC
</button>
{/* 点中心说话 */}
<button className="jhud-core" onClick={onMic} aria-label={hint} title={hint} />
<div className={"jhud-word" + (amber ? " amber" : state === "listening" ? " live" : "")}>{hint}</div>
<div className="jhud-decode">
{transcript && <p className="jhud-me"><span></span>{transcript}</p>}
{reply && (
<p className="jhud-reply">
<span>{(name || "JARVIS").toUpperCase()}</span>{reply}
{(state === "thinking" || state === "speaking") && <i className="jhud-cursor" />}
</p>
)}
</div>
</div>
);
}
const HUD_CSS = `
.jhud{position:fixed;inset:0;z-index:60;overflow:hidden;color:#d6f2fc;user-select:none;
background:radial-gradient(120% 80% at 50% 44%,#08131d 0%,#04070d 62%,#02040a 100%);
font-family:ui-monospace,"SF Mono","JetBrains Mono",Menlo,Consolas,monospace;
animation:jhudIn .5s ease}
@keyframes jhudIn{from{opacity:0}to{opacity:1}}
.jhud::before{content:"";position:absolute;inset:0;pointer-events:none;opacity:.5;
background-image:linear-gradient(transparent 0 3px,rgba(56,225,255,.035) 3px 4px),linear-gradient(90deg,transparent 0 3px,rgba(56,225,255,.028) 3px 4px);
background-size:100% 4px,4px 100%;mask-image:radial-gradient(100% 100% at 50% 45%,#000 55%,transparent 100%)}
.jhud::after{content:"";position:absolute;inset:0;pointer-events:none;z-index:3;
background:linear-gradient(rgba(56,225,255,0),rgba(56,225,255,.05) 50%,rgba(56,225,255,0));height:26%;animation:jhudSweep 6.5s linear infinite;opacity:.55}
@keyframes jhudSweep{0%{transform:translateY(-30%)}100%{transform:translateY(430%)}}
.jhud-canvas{position:absolute;inset:0;width:100%;height:100%;z-index:1}
.jhud-corner{position:absolute;width:44px;height:44px;border:1.5px solid #0e5a70;z-index:4;opacity:.8}
.jhud-corner.tl{top:20px;left:20px;border-right:0;border-bottom:0}
.jhud-corner.tr{top:20px;right:20px;border-left:0;border-bottom:0}
.jhud-corner.bl{bottom:20px;left:20px;border-right:0;border-top:0}
.jhud-corner.br{bottom:20px;right:20px;border-left:0;border-top:0}
.jhud-status{position:absolute;top:30px;left:38px;z-index:5;letter-spacing:.14em;font-size:12px;line-height:1.9}
.jhud-brand{color:#38e1ff;font-size:15px;letter-spacing:.28em;text-shadow:0 0 14px rgba(56,225,255,.55)}
.jhud-row{display:flex;gap:8px;align-items:center;color:#5f8496}
.jhud-row b{color:#d6f2fc;font-weight:500}
.jhud-dot{width:7px;height:7px;border-radius:50%;background:#38e1ff;box-shadow:0 0 10px #38e1ff}
.jhud-dot.live{background:#ff4d5e;box-shadow:0 0 10px #ff4d5e;animation:jhudBlink 1s steps(2) infinite}
.jhud-dot.amber{background:#ffb638;box-shadow:0 0 10px #ffb638}
@keyframes jhudBlink{50%{opacity:.25}}
.jhud-close{position:absolute;top:26px;right:34px;z-index:6;display:flex;align-items:center;gap:6px;
background:rgba(8,22,32,.6);border:1px solid #0e2a38;color:#7fa6b6;border-radius:999px;padding:7px 13px;
font-family:inherit;font-size:11px;letter-spacing:.14em;cursor:pointer;backdrop-filter:blur(6px);transition:.18s}
.jhud-close:hover{border-color:#38e1ff;color:#d6f2fc}
.jhud-core{position:absolute;left:50%;top:50%;transform:translate(-50%,-50%);width:26vmin;height:26vmin;
border-radius:50%;background:transparent;border:0;cursor:pointer;z-index:5}
.jhud-word{position:absolute;left:50%;top:calc(50% + 20vmin);transform:translateX(-50%);z-index:5;
font-size:13px;letter-spacing:.5em;color:#38e1ff;text-shadow:0 0 16px rgba(56,225,255,.5);text-transform:uppercase;white-space:nowrap}
.jhud-word.amber{color:#ffb638;text-shadow:0 0 16px rgba(255,182,56,.5)}
.jhud-word.live{color:#ff4d5e;text-shadow:0 0 16px rgba(255,77,94,.5)}
.jhud-decode{position:absolute;left:50%;bottom:56px;transform:translateX(-50%);z-index:5;
width:min(760px,88vw);text-align:center;line-height:1.55}
.jhud-me{color:#5f8496;font-size:13px;margin:0 0 10px}
.jhud-me span,.jhud-reply span{font-size:10px;letter-spacing:.24em;margin-right:8px;opacity:.75}
.jhud-reply{color:#eaf8ff;font-size:16px;margin:0;text-shadow:0 0 10px rgba(56,225,255,.25)}
.jhud-reply span{color:#38e1ff}
.jhud-cursor{display:inline-block;width:8px;height:2px;background:#38e1ff;margin-left:3px;vertical-align:middle;box-shadow:0 0 8px #38e1ff;animation:jhudBlink .8s steps(2) infinite}
@media (prefers-reduced-motion: reduce){.jhud::after{display:none}.jhud-dot.live,.jhud-cursor{animation:none}}
`;
@@ -1,7 +1,9 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { Mic, Square, Loader2, Volume2, X, Settings2 } from "lucide-react";
import { Mic, Square, Loader2, Volume2, X, Settings2, Maximize2 } from "lucide-react";
import { VoiceClient, type VoiceState } from "../lib/voice";
import { JarvisSettings } from "./JarvisSettings";
import { JarvisHud } from "./JarvisHud";
import { getMyJarvis } from "../lib/api";
import { useToast } from "../ui/Toast";
import { cn } from "../ui/cn";
@@ -29,6 +31,8 @@ export function VoiceDock({ onTask }: Props) {
const [reply, setReply] = useState(""); // JARVIS 回答(打字机,逐 token 累加)
const [open, setOpen] = useState(false); // 是否展开对话气泡
const [settingsOpen, setSettingsOpen] = useState(false); // JARVIS 设置弹窗
const [fullscreen, setFullscreen] = useState(false); // 全屏 JARVIS 模式
const [name, setName] = useState("JARVIS"); // 助手名(HUD 品牌位)
// 懒建客户端(首次点按时,带上用户手势→AudioContext 才能启动)。
const ensureClient = useCallback((): VoiceClient => {
@@ -55,6 +59,14 @@ export function VoiceDock({ onTask }: Props) {
useEffect(() => () => clientRef.current?.close(), []);
// Esc 退出全屏 JARVIS 模式。
useEffect(() => {
if (!fullscreen) return;
const onKey = (e: KeyboardEvent) => e.key === "Escape" && setFullscreen(false);
window.addEventListener("keydown", onKey);
return () => window.removeEventListener("keydown", onKey);
}, [fullscreen]);
const onMic = useCallback(async () => {
const c = ensureClient();
try {
@@ -71,21 +83,40 @@ export function VoiceDock({ onTask }: Props) {
}
}, [ensureClient, state, toast]);
// 进全屏 JARVIS 模式:建好客户端(这样 HUD 能读实时电平),拉一次助手名做品牌位。
const openFullscreen = useCallback(() => {
ensureClient();
getMyJarvis()
.then((j) => setName(j.name || "JARVIS"))
.catch(() => {});
setFullscreen(true);
}, [ensureClient]);
const active = state === "listening";
const busy = state === "connecting" || state === "thinking";
return (
<>
<div className="pointer-events-none fixed bottom-6 right-6 z-40 flex flex-col items-end gap-2">
{/* 设置齿轮:名字 / 人设 / 我的豆包配置 */}
<button
onClick={() => setSettingsOpen(true)}
title="JARVIS 设置(名字 / 人设 / 我的豆包)"
aria-label="JARVIS 设置"
className="pointer-events-auto flex h-8 w-8 items-center justify-center rounded-full border border-line bg-ink-850/90 text-slate-400 shadow-md backdrop-blur transition hover:border-ink-600 hover:text-slate-200"
>
<Settings2 className="h-4 w-4" />
</button>
{/* 小工具:全屏 JARVIS 模式 + 设置 */}
<div className="pointer-events-auto flex items-center gap-2">
<button
onClick={openFullscreen}
title="全屏 JARVIS 模式"
aria-label="全屏 JARVIS 模式"
className="flex h-8 w-8 items-center justify-center rounded-full border border-line bg-ink-850/90 text-slate-400 shadow-md backdrop-blur transition hover:border-brand hover:text-brand-300"
>
<Maximize2 className="h-4 w-4" />
</button>
<button
onClick={() => setSettingsOpen(true)}
title="JARVIS 设置(名字 / 人设 / 我的豆包)"
aria-label="JARVIS 设置"
className="flex h-8 w-8 items-center justify-center rounded-full border border-line bg-ink-850/90 text-slate-400 shadow-md backdrop-blur transition hover:border-ink-600 hover:text-slate-200"
>
<Settings2 className="h-4 w-4" />
</button>
</div>
{/* 对话气泡:我说的(转写)+ JARVIS 回答(打字机) */}
{open && (transcript || reply) && (
@@ -146,6 +177,18 @@ export function VoiceDock({ onTask }: Props) {
</span>
</div>
<JarvisSettings open={settingsOpen} onClose={() => setSettingsOpen(false)} />
{fullscreen && (
<JarvisHud
name={name}
state={state}
getLevel={() => clientRef.current?.level() ?? 0}
transcript={transcript}
reply={reply}
hint={HINT[state]}
onMic={onMic}
onClose={() => setFullscreen(false)}
/>
)}
</>
);
}