diff --git a/sundynix-desktop/frontend/src/lib/voice.ts b/sundynix-desktop/frontend/src/lib/voice.ts index 295f1ae..0e0d54d 100644 --- a/sundynix-desktop/frontend/src/lib/voice.ts +++ b/sundynix-desktop/frontend/src/lib/voice.ts @@ -57,11 +57,14 @@ export class VoiceClient { private micCtx: AudioContext | null = null; private micStream: MediaStream | null = null; private micNode: ScriptProcessorNode | null = null; + private micAnalyser: AnalyserNode | null = null; // 下行播放 private playCtx: AudioContext | null = null; private nextStart = 0; // 下一段音频的调度起点（连续朗读用） private sources: AudioBufferSourceNode[] = []; + private playAnalyser: AnalyserNode | null = null; + private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲（level() 每帧读，别每帧新建） constructor(cb: VoiceCallbacks) { this.cb = cb; @@ -153,10 +156,31 @@ export class VoiceClient { // ensurePlayCtx 建/复用下行播放 AudioContext，并在 suspended 时 resume（自动播放策略要求手势内唤醒）。 private ensurePlayCtx(): AudioContext { if (!this.playCtx) this.playCtx = new AudioContext(); + if (!this.playAnalyser) { + // 下行电平探针：所有 TTS 音频经它再到扬声器（驱动 HUD 的"说话"频谱）。 + const an = this.playCtx.createAnalyser(); + an.fftSize = 512; + an.connect(this.playCtx.destination); + this.playAnalyser = an; + } if (this.playCtx.state === "suspended") void this.playCtx.resume(); return this.playCtx; } + // level 返回当前"活跃"音频电平 0..1：说话读下行 TTS，其余读上行麦克风；无探针则 0。 + // 供 JARVIS HUD 每帧读取，让可视化随真实声音起伏。 + level(): number { + const a = this.state === "speaking" ? this.playAnalyser : this.micAnalyser; + if (!a) return 0; + a.getByteTimeDomainData(this.lvlBuf); + let s = 0; + for (let i = 0; i < this.lvlBuf.length; i++) { + const v = (this.lvlBuf[i] - 128) / 128; + s += v * v; + } + return Math.min(1, Math.sqrt(s / this.lvlBuf.length) * 3.2); + } + // stopListening 结束本轮说话：停麦克风 + 发 end（服务端拿最终转写→提交任务）。 stopListening(): void { this.stopMic(); @@ -180,6 +204,10 @@ export class VoiceClient { this.micCtx = ctx; if (ctx.state === "suspended") await ctx.resume(); // 防采集上下文挂起（无回调=不上行音频） const src = ctx.createMediaStreamSource(stream); + const an = ctx.createAnalyser(); // 上行电平探针（驱动 JARVIS HUD 的"聆听"反应） + an.fftSize = 512; + src.connect(an); + this.micAnalyser = an; const node = ctx.createScriptProcessor(4096, 1, 1); node.onaudioprocess = (ev) => { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; @@ -195,6 +223,7 @@ export class VoiceClient { private stopMic(): void { this.micNode?.disconnect(); this.micNode = null; + this.micAnalyser = null; this.micStream?.getTracks().forEach((t) => t.stop()); this.micStream = null; this.micCtx?.close().catch(() => {}); @@ -215,7 +244,7 @@ export class VoiceClient { const node = ctx.createBufferSource(); node.buffer = audioBuf; - node.connect(ctx.destination); + node.connect(this.playAnalyser ?? ctx.destination); // 经探针再到扬声器（读得到电平） const now = ctx.currentTime; if (this.nextStart < now) this.nextStart = now; node.start(this.nextStart); @@ -249,6 +278,7 @@ export class VoiceClient { this.resetPlayback(); this.playCtx?.close().catch(() => {}); this.playCtx = null; + this.playAnalyser = null; this.ws?.close(); this.ws = null; this.setState("idle"); diff --git a/sundynix-desktop/frontend/src/shell/JarvisHud.tsx b/sundynix-desktop/frontend/src/shell/JarvisHud.tsx new file mode 100644 index 0000000..a3a2012 --- /dev/null +++ b/sundynix-desktop/frontend/src/shell/JarvisHud.tsx @@ -0,0 +1,234 @@ +import { useEffect, useRef } from "react"; +import { X } from "lucide-react"; +import type { VoiceState } from "../lib/voice"; + +// 全屏「JARVIS 模式」HUD：弧反应堆核心 + 随真实声音反应的环形频谱 + 状态编排。 +// 视觉数据全来自真实语音会话：level 由 VoiceClient 的麦克风/TTS 探针实时给出（getLevel）， +// state 是会话状态机；transcript/reply 是当轮问答。点中心说话，右上角退出。 + +interface Props { + name: string; // 助手名（HUD 品牌位） + state: VoiceState; + getLevel: () => number; // 0..1 实时电平 + transcript: string; // 我说的 + reply: string; // JARVIS 回答（流式） + hint: string; // 当前状态提示 + onMic: () => void; // 点中心：开始/结束说话 + onClose: () => void; +} + +type Conf = { accent: number[]; activity: number; ringSpin: number; label: string }; +const CY = [56, 225, 255]; +const AM = [255, 182, 56]; +const CONF: Record = { + idle: { accent: CY, activity: 0.18, ringSpin: 0.1, label: "STANDBY" }, + connecting: { accent: CY, activity: 0.22, ringSpin: 0.5, label: "LINKING" }, + ready: { accent: CY, activity: 0.18, ringSpin: 0.1, label: "READY" }, + listening: { accent: CY, activity: 0.6, ringSpin: 0.35, label: "LISTENING" }, + thinking: { accent: AM, activity: 0.3, ringSpin: 0.8, label: "PROCESSING" }, + speaking: { accent: CY, activity: 0.85, ringSpin: 0.22, label: "SPEAKING" }, +}; + +export function JarvisHud({ name, state, getLevel, transcript, reply, hint, onMic, onClose }: Props) { + const canvasRef = useRef(null); + // 用 ref 让 RAF 闭包读到最新 props，无需重启动画。 + const P = useRef({ state, getLevel, transcript, reply }); + P.current = { state, getLevel, transcript, reply }; + + useEffect(() => { + const cv = canvasRef.current!; + const ctx = cv.getContext("2d")!; + const reduce = matchMedia("(prefers-reduced-motion: reduce)").matches; + let W = 0, H = 0, raf = 0; + const resize = () => { + const r = cv.getBoundingClientRect(); + const dpr = Math.min(2, devicePixelRatio || 1); + W = r.width; H = r.height; + cv.width = W * dpr; cv.height = H * dpr; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + }; + resize(); + const ro = new ResizeObserver(resize); + ro.observe(cv); + + const rgba = (c: number[], a: number) => `rgba(${c[0]},${c[1]},${c[2]},${a})`; + const lerp = (a: number, b: number, t: number) => a + (b - a) * t; + const accent = CY.slice(); + let level = 0.18, think = 0, t = 0, last = performance.now(); + const parts = Array.from({ length: 64 }, () => ({ a: Math.random() * 6.28, r: Math.random(), s: 0.3 + Math.random() * 0.9 })); + + const synth = (s: VoiceState) => + s === "thinking" ? 0.2 + 0.06 * Math.sin(t * 3) : 0.16 + 0.05 * Math.sin(t * 1.6); + + const draw = (now: number) => { + const dt = Math.min(0.05, (now - last) / 1000); last = now; t += dt; + const st = P.current.state; + const c = CONF[st] ?? CONF.idle; + const active = st === "listening" || st === "speaking"; + const tgt = active ? Math.max(P.current.getLevel(), 0) : synth(st); + level += (tgt - level) * Math.min(1, dt * (active ? 16 : 6)); + for (let i = 0; i < 3; i++) accent[i] = lerp(accent[i], c.accent[i], Math.min(1, dt * 4)); + think += ((st === "thinking" ? 1 : 0) - think) * Math.min(1, dt * 3); + + const cx = W / 2, cy = H / 2, R = Math.min(W, H) * 0.16; + ctx.clearRect(0, 0, W, H); + ctx.globalCompositeOperation = "lighter"; + + let g = ctx.createRadialGradient(cx, cy, R * 0.2, cx, cy, R * 3.4); + g.addColorStop(0, rgba(accent, 0.1 + level * 0.1)); + g.addColorStop(1, rgba(accent, 0)); + ctx.fillStyle = g; ctx.fillRect(cx - R * 4, cy - R * 4, R * 8, R * 8); + + const spin = reduce ? 0 : t * c.ringSpin; + [1.35, 1.7, 2.05, 2.5, 3.0].forEach((rr, i) => { + const rad = R * rr, dir = i % 2 ? -1 : 1, a0 = spin * dir * (1 + i * 0.25); + ctx.lineWidth = i === 0 ? 2.2 : 1.2; + ctx.strokeStyle = rgba(accent, (0.5 - i * 0.06) * (0.6 + level * 0.5)); + ctx.shadowColor = rgba(accent, 0.8); ctx.shadowBlur = 12; + const segs: [number, number][] = i === 0 ? [[0, 5.0]] : [[a0, 1.3], [a0 + 2.3, 2.0], [a0 + 5.0, 0.8]]; + segs.forEach(([s, len]) => { ctx.beginPath(); ctx.arc(cx, cy, rad, s, s + len); ctx.stroke(); }); + }); + ctx.shadowBlur = 0; + + const NB = 104, inner = R * 1.12; + ctx.lineWidth = 2; + for (let i = 0; i < NB; i++) { + const ang = (i / NB) * 6.28 - Math.PI / 2 + spin * 0.15; + const seed = Math.sin(i * 12.9898) * 43758.5453; const nz = seed - Math.floor(seed); + const wob = 0.5 + 0.5 * Math.sin(t * 3 + i * 0.5); + const h = R * (0.06 + (0.1 + c.activity * 0.9 * level) * (0.35 + 0.65 * wob) * (0.5 + nz)); + const x1 = cx + Math.cos(ang) * inner, y1 = cy + Math.sin(ang) * inner; + const x2 = cx + Math.cos(ang) * (inner + h), y2 = cy + Math.sin(ang) * (inner + h); + const bright = 0.35 + 0.65 * (h / (R * 0.9)); + ctx.strokeStyle = rgba(accent, 0.25 + bright * 0.6); + ctx.beginPath(); ctx.moveTo(x1, y1); ctx.lineTo(x2, y2); ctx.stroke(); + } + + if (think > 0.02) { + parts.forEach((p) => { + if (!reduce) p.a += dt * p.s * 1.2; + const rad = R * (1.15 + p.r * 1.7), x = cx + Math.cos(p.a) * rad, y = cy + Math.sin(p.a) * rad; + ctx.fillStyle = rgba(AM, 0.7 * think); + ctx.shadowColor = rgba(AM, 0.9); ctx.shadowBlur = 8; + ctx.beginPath(); ctx.arc(x, y, 1.6, 0, 6.28); ctx.fill(); + }); + const sa = t * 2.2; + ctx.save(); ctx.beginPath(); ctx.moveTo(cx, cy); ctx.arc(cx, cy, R * 2.9, sa, sa + 0.5); ctx.closePath(); + ctx.fillStyle = rgba(AM, 0.1 * think); ctx.fill(); ctx.restore(); + ctx.shadowBlur = 0; + } + + const cr = R * (0.62 + level * 0.3); + const cg = ctx.createRadialGradient(cx, cy, 0, cx, cy, cr); + cg.addColorStop(0, rgba([255, 255, 255], 0.95)); + cg.addColorStop(0.25, rgba(accent, 0.95)); + cg.addColorStop(0.7, rgba(accent, 0.28)); + cg.addColorStop(1, rgba(accent, 0)); + ctx.fillStyle = cg; ctx.beginPath(); ctx.arc(cx, cy, cr, 0, 6.28); ctx.fill(); + ctx.globalCompositeOperation = "source-over"; + for (let k = 0; k < 3; k++) { + ctx.lineWidth = 1; ctx.strokeStyle = rgba([255, 255, 255], 0.5 - k * 0.13); + ctx.beginPath(); ctx.arc(cx, cy, R * (0.3 + k * 0.14), 0, 6.28); ctx.stroke(); + } + ctx.globalCompositeOperation = "lighter"; + ctx.strokeStyle = rgba([255, 255, 255], 0.85); ctx.lineWidth = 1.5; ctx.beginPath(); + for (let k = 0; k < 3; k++) { + const a = spin * 2 + k * 2.094, rr = R * 0.22; + const x = cx + Math.cos(a) * rr, y = cy + Math.sin(a) * rr; + k ? ctx.lineTo(x, y) : ctx.moveTo(x, y); + } + ctx.closePath(); ctx.stroke(); + ctx.globalCompositeOperation = "source-over"; + + raf = requestAnimationFrame(draw); + }; + raf = requestAnimationFrame(draw); + return () => { cancelAnimationFrame(raf); ro.disconnect(); }; + }, []); + + const c = CONF[state] ?? CONF.idle; + const amber = state === "thinking"; + + return ( +
+ + + + + + +
+
{(name || "JARVIS").toUpperCase()}
+
+ + {c.label} +
+
+ + + + {/* 点中心说话 */} +
+ ); +} + +const HUD_CSS = ` +.jhud{position:fixed;inset:0;z-index:60;overflow:hidden;color:#d6f2fc;user-select:none; + background:radial-gradient(120% 80% at 50% 44%,#08131d 0%,#04070d 62%,#02040a 100%); + font-family:ui-monospace,"SF Mono","JetBrains Mono",Menlo,Consolas,monospace; + animation:jhudIn .5s ease} +@keyframes jhudIn{from{opacity:0}to{opacity:1}} +.jhud::before{content:"";position:absolute;inset:0;pointer-events:none;opacity:.5; + background-image:linear-gradient(transparent 0 3px,rgba(56,225,255,.035) 3px 4px),linear-gradient(90deg,transparent 0 3px,rgba(56,225,255,.028) 3px 4px); + background-size:100% 4px,4px 100%;mask-image:radial-gradient(100% 100% at 50% 45%,#000 55%,transparent 100%)} +.jhud::after{content:"";position:absolute;inset:0;pointer-events:none;z-index:3; + background:linear-gradient(rgba(56,225,255,0),rgba(56,225,255,.05) 50%,rgba(56,225,255,0));height:26%;animation:jhudSweep 6.5s linear infinite;opacity:.55} +@keyframes jhudSweep{0%{transform:translateY(-30%)}100%{transform:translateY(430%)}} +.jhud-canvas{position:absolute;inset:0;width:100%;height:100%;z-index:1} +.jhud-corner{position:absolute;width:44px;height:44px;border:1.5px solid #0e5a70;z-index:4;opacity:.8} +.jhud-corner.tl{top:20px;left:20px;border-right:0;border-bottom:0} +.jhud-corner.tr{top:20px;right:20px;border-left:0;border-bottom:0} +.jhud-corner.bl{bottom:20px;left:20px;border-right:0;border-top:0} +.jhud-corner.br{bottom:20px;right:20px;border-left:0;border-top:0} +.jhud-status{position:absolute;top:30px;left:38px;z-index:5;letter-spacing:.14em;font-size:12px;line-height:1.9} +.jhud-brand{color:#38e1ff;font-size:15px;letter-spacing:.28em;text-shadow:0 0 14px rgba(56,225,255,.55)} +.jhud-row{display:flex;gap:8px;align-items:center;color:#5f8496} +.jhud-row b{color:#d6f2fc;font-weight:500} +.jhud-dot{width:7px;height:7px;border-radius:50%;background:#38e1ff;box-shadow:0 0 10px #38e1ff} +.jhud-dot.live{background:#ff4d5e;box-shadow:0 0 10px #ff4d5e;animation:jhudBlink 1s steps(2) infinite} +.jhud-dot.amber{background:#ffb638;box-shadow:0 0 10px #ffb638} +@keyframes jhudBlink{50%{opacity:.25}} +.jhud-close{position:absolute;top:26px;right:34px;z-index:6;display:flex;align-items:center;gap:6px; + background:rgba(8,22,32,.6);border:1px solid #0e2a38;color:#7fa6b6;border-radius:999px;padding:7px 13px; + font-family:inherit;font-size:11px;letter-spacing:.14em;cursor:pointer;backdrop-filter:blur(6px);transition:.18s} +.jhud-close:hover{border-color:#38e1ff;color:#d6f2fc} +.jhud-core{position:absolute;left:50%;top:50%;transform:translate(-50%,-50%);width:26vmin;height:26vmin; + border-radius:50%;background:transparent;border:0;cursor:pointer;z-index:5} +.jhud-word{position:absolute;left:50%;top:calc(50% + 20vmin);transform:translateX(-50%);z-index:5; + font-size:13px;letter-spacing:.5em;color:#38e1ff;text-shadow:0 0 16px rgba(56,225,255,.5);text-transform:uppercase;white-space:nowrap} +.jhud-word.amber{color:#ffb638;text-shadow:0 0 16px rgba(255,182,56,.5)} +.jhud-word.live{color:#ff4d5e;text-shadow:0 0 16px rgba(255,77,94,.5)} +.jhud-decode{position:absolute;left:50%;bottom:56px;transform:translateX(-50%);z-index:5; + width:min(760px,88vw);text-align:center;line-height:1.55} +.jhud-me{color:#5f8496;font-size:13px;margin:0 0 10px} +.jhud-me span,.jhud-reply span{font-size:10px;letter-spacing:.24em;margin-right:8px;opacity:.75} +.jhud-reply{color:#eaf8ff;font-size:16px;margin:0;text-shadow:0 0 10px rgba(56,225,255,.25)} +.jhud-reply span{color:#38e1ff} +.jhud-cursor{display:inline-block;width:8px;height:2px;background:#38e1ff;margin-left:3px;vertical-align:middle;box-shadow:0 0 8px #38e1ff;animation:jhudBlink .8s steps(2) infinite} +@media (prefers-reduced-motion: reduce){.jhud::after{display:none}.jhud-dot.live,.jhud-cursor{animation:none}} +`; diff --git a/sundynix-desktop/frontend/src/shell/VoiceDock.tsx b/sundynix-desktop/frontend/src/shell/VoiceDock.tsx index d1303d9..8b89baf 100644 --- a/sundynix-desktop/frontend/src/shell/VoiceDock.tsx +++ b/sundynix-desktop/frontend/src/shell/VoiceDock.tsx @@ -1,7 +1,9 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { Mic, Square, Loader2, Volume2, X, Settings2 } from "lucide-react"; +import { Mic, Square, Loader2, Volume2, X, Settings2, Maximize2 } from "lucide-react"; import { VoiceClient, type VoiceState } from "../lib/voice"; import { JarvisSettings } from "./JarvisSettings"; +import { JarvisHud } from "./JarvisHud"; +import { getMyJarvis } from "../lib/api"; import { useToast } from "../ui/Toast"; import { cn } from "../ui/cn"; @@ -29,6 +31,8 @@ export function VoiceDock({ onTask }: Props) { const [reply, setReply] = useState(""); // JARVIS 回答（打字机，逐 token 累加） const [open, setOpen] = useState(false); // 是否展开对话气泡 const [settingsOpen, setSettingsOpen] = useState(false); // JARVIS 设置弹窗 + const [fullscreen, setFullscreen] = useState(false); // 全屏 JARVIS 模式 + const [name, setName] = useState("JARVIS"); // 助手名（HUD 品牌位） // 懒建客户端（首次点按时，带上用户手势→AudioContext 才能启动）。 const ensureClient = useCallback((): VoiceClient => { @@ -55,6 +59,14 @@ export function VoiceDock({ onTask }: Props) { useEffect(() => () => clientRef.current?.close(), []); + // Esc 退出全屏 JARVIS 模式。 + useEffect(() => { + if (!fullscreen) return; + const onKey = (e: KeyboardEvent) => e.key === "Escape" && setFullscreen(false); + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [fullscreen]); + const onMic = useCallback(async () => { const c = ensureClient(); try { @@ -71,21 +83,40 @@ export function VoiceDock({ onTask }: Props) { } }, [ensureClient, state, toast]); + // 进全屏 JARVIS 模式：建好客户端（这样 HUD 能读实时电平），拉一次助手名做品牌位。 + const openFullscreen = useCallback(() => { + ensureClient(); + getMyJarvis() + .then((j) => setName(j.name || "JARVIS")) + .catch(() => {}); + setFullscreen(true); + }, [ensureClient]); + const active = state === "listening"; const busy = state === "connecting" || state === "thinking"; return ( <>
- {/* 设置齿轮：名字 / 人设 / 我的豆包配置 */} - + {/* 小工具：全屏 JARVIS 模式 + 设置 */} +
+ + +
{/* 对话气泡：我说的（转写）+ JARVIS 回答（打字机） */} {open && (transcript || reply) && ( @@ -146,6 +177,18 @@ export function VoiceDock({ onTask }: Props) {
setSettingsOpen(false)} /> + {fullscreen && ( + clientRef.current?.level() ?? 0} + transcript={transcript} + reply={reply} + hint={HINT[state]} + onMic={onMic} + onClose={() => setFullscreen(false)} + /> + )} ); }