feat(desktop): 全屏「JARVIS 模式」HUD——钢铁侠风,接真实语音音频
弧反应堆核心 + 同心旋转环 + 随真实声音反应的环形频谱 + 状态编排 + HUD 边框/扫描线/网格。 可视化数据全来自真实语音会话(不是模拟)。 - voice.ts: micCtx/playCtx 各挂 AnalyserNode + level()——说话读下行TTS、其余读麦克风, 实时电平 0..1(HUD 每帧读,复用缓冲) - JarvisHud.tsx: Canvas 2D HUD,按 VoiceState 编排(待命呼吸/聆听炸开/思考散粒子+雷达/说话频谱), 单一青色强调+思考态琥珀;显示助手名(星期五)+ 我说的 + 打字机回答;尊重 prefers-reduced-motion - VoiceDock: 加"全屏 JARVIS 模式"按钮(建客户端+拉名字);Esc 退出 真机验证:全屏 HUD 正常渲染,STANDBY 呼吸,品牌位显示自定义名"星期五"。晚上点中心说话即随真声反应。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -57,11 +57,14 @@ export class VoiceClient {
|
||||
private micCtx: AudioContext | null = null;
|
||||
private micStream: MediaStream | null = null;
|
||||
private micNode: ScriptProcessorNode | null = null;
|
||||
private micAnalyser: AnalyserNode | null = null;
|
||||
|
||||
// 下行播放
|
||||
private playCtx: AudioContext | null = null;
|
||||
private nextStart = 0; // 下一段音频的调度起点(连续朗读用)
|
||||
private sources: AudioBufferSourceNode[] = [];
|
||||
private playAnalyser: AnalyserNode | null = null;
|
||||
private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲(level() 每帧读,别每帧新建)
|
||||
|
||||
constructor(cb: VoiceCallbacks) {
|
||||
this.cb = cb;
|
||||
@@ -153,10 +156,31 @@ export class VoiceClient {
|
||||
// ensurePlayCtx 建/复用下行播放 AudioContext,并在 suspended 时 resume(自动播放策略要求手势内唤醒)。
|
||||
private ensurePlayCtx(): AudioContext {
|
||||
if (!this.playCtx) this.playCtx = new AudioContext();
|
||||
if (!this.playAnalyser) {
|
||||
// 下行电平探针:所有 TTS 音频经它再到扬声器(驱动 HUD 的"说话"频谱)。
|
||||
const an = this.playCtx.createAnalyser();
|
||||
an.fftSize = 512;
|
||||
an.connect(this.playCtx.destination);
|
||||
this.playAnalyser = an;
|
||||
}
|
||||
if (this.playCtx.state === "suspended") void this.playCtx.resume();
|
||||
return this.playCtx;
|
||||
}
|
||||
|
||||
// level 返回当前"活跃"音频电平 0..1:说话读下行 TTS,其余读上行麦克风;无探针则 0。
|
||||
// 供 JARVIS HUD 每帧读取,让可视化随真实声音起伏。
|
||||
level(): number {
|
||||
const a = this.state === "speaking" ? this.playAnalyser : this.micAnalyser;
|
||||
if (!a) return 0;
|
||||
a.getByteTimeDomainData(this.lvlBuf);
|
||||
let s = 0;
|
||||
for (let i = 0; i < this.lvlBuf.length; i++) {
|
||||
const v = (this.lvlBuf[i] - 128) / 128;
|
||||
s += v * v;
|
||||
}
|
||||
return Math.min(1, Math.sqrt(s / this.lvlBuf.length) * 3.2);
|
||||
}
|
||||
|
||||
// stopListening 结束本轮说话:停麦克风 + 发 end(服务端拿最终转写→提交任务)。
|
||||
stopListening(): void {
|
||||
this.stopMic();
|
||||
@@ -180,6 +204,10 @@ export class VoiceClient {
|
||||
this.micCtx = ctx;
|
||||
if (ctx.state === "suspended") await ctx.resume(); // 防采集上下文挂起(无回调=不上行音频)
|
||||
const src = ctx.createMediaStreamSource(stream);
|
||||
const an = ctx.createAnalyser(); // 上行电平探针(驱动 JARVIS HUD 的"聆听"反应)
|
||||
an.fftSize = 512;
|
||||
src.connect(an);
|
||||
this.micAnalyser = an;
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
node.onaudioprocess = (ev) => {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||||
@@ -195,6 +223,7 @@ export class VoiceClient {
|
||||
private stopMic(): void {
|
||||
this.micNode?.disconnect();
|
||||
this.micNode = null;
|
||||
this.micAnalyser = null;
|
||||
this.micStream?.getTracks().forEach((t) => t.stop());
|
||||
this.micStream = null;
|
||||
this.micCtx?.close().catch(() => {});
|
||||
@@ -215,7 +244,7 @@ export class VoiceClient {
|
||||
|
||||
const node = ctx.createBufferSource();
|
||||
node.buffer = audioBuf;
|
||||
node.connect(ctx.destination);
|
||||
node.connect(this.playAnalyser ?? ctx.destination); // 经探针再到扬声器(读得到电平)
|
||||
const now = ctx.currentTime;
|
||||
if (this.nextStart < now) this.nextStart = now;
|
||||
node.start(this.nextStart);
|
||||
@@ -249,6 +278,7 @@ export class VoiceClient {
|
||||
this.resetPlayback();
|
||||
this.playCtx?.close().catch(() => {});
|
||||
this.playCtx = null;
|
||||
this.playAnalyser = null;
|
||||
this.ws?.close();
|
||||
this.ws = null;
|
||||
this.setState("idle");
|
||||
|
||||
Reference in New Issue
Block a user