feat(voice): 提升桌面端语音交互与本地任务执行支持
This commit is contained in:
Blizzard
@@ -15,9 +15,11 @@ import (
|
||||
)
|
||||
|
||||
// App 经 Wails v3 的 Service 绑定暴露给前端，承载只有桌面端能做的原生能力：
|
||||
// 文件读写、系统"另存为"框、用系统默认应用打开、原生通知。
|
||||
// 文件读写、系统"另存为"框、用系统默认应用打开、原生通知、本地执行 runner（JARVIS 的手）。
|
||||
// v3 中不再需要注入 ctx——对话框经 application.Get().Dialog 获取。
|
||||
type App struct{}
|
||||
type App struct {
|
||||
runner LocalRunner // 本地执行器（localrunner.go）：用户显式开启才连接
|
||||
}
|
||||
|
||||
// Ping 供前端探活 Go 桥是否就绪。
|
||||
func (a *App) Ping() string { return "sundynix-desktop ok" }
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
/**
|
||||
* App 经 Wails v3 的 Service 绑定暴露给前端，承载只有桌面端能做的原生能力：
|
||||
* 文件读写、系统"另存为"框、用系统默认应用打开、原生通知。
|
||||
* 文件读写、系统"另存为"框、用系统默认应用打开、原生通知、本地执行 runner（JARVIS 的手）。
|
||||
* v3 中不再需要注入 ctx——对话框经 application.Get().Dialog 获取。
|
||||
* @module
|
||||
*/
|
||||
@@ -12,6 +12,13 @@
|
||||
// @ts-ignore: Unused imports
|
||||
import { Call as $Call, CancellablePromise as $CancellablePromise, Create as $Create } from "@wailsio/runtime";
|
||||
|
||||
/**
|
||||
* LocalRunnerStatus 返回 "offline" / "connecting" / "online:<workdir>"（前端状态显示）。
|
||||
*/
|
||||
export function LocalRunnerStatus(): $CancellablePromise<string> {
|
||||
return $Call.ByID(1389428436);
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify 弹一条系统通知（best-effort：macOS 用 osascript，其它平台暂静默）。
|
||||
*/
|
||||
@@ -42,3 +49,18 @@ export function PrintReportPage(filename: string, html: string): $CancellablePro
|
||||
export function SaveReportAs(url: string, filename: string): $CancellablePromise<string> {
|
||||
return $Call.ByID(1437856486, url, filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* StartLocalRunner 开启本地文件访问：以 workdir 为沙箱根连接 gateway 注册执行器。
|
||||
* 幂等：重复调用先停旧连接。断线自动重连（5s 退避）直到 StopLocalRunner。
|
||||
*/
|
||||
export function StartLocalRunner(gatewayURL: string, token: string, workdir: string): $CancellablePromise<void> {
|
||||
return $Call.ByID(2808780368, gatewayURL, token, workdir);
|
||||
}
|
||||
|
||||
/**
|
||||
* StopLocalRunner 关闭本地文件访问（幂等）。
|
||||
*/
|
||||
export function StopLocalRunner(): $CancellablePromise<void> {
|
||||
return $Call.ByID(476635130);
|
||||
}
|
||||
|
||||
@@ -377,7 +377,14 @@ export default function App() {
|
||||
selfUserId={user?.id ?? ""}
|
||||
/>
|
||||
<InviteMembers open={inviteOpen} onClose={() => setInviteOpen(false)} tenantName={tenant?.tenant?.name ?? ""} />
|
||||
<VoiceDock onTask={onVoiceTask} />
|
||||
{/* JARVIS 界面动作（P2）：服务端白名单过后，这里再按已知 ViewKey 核一次才执行（双保险）。 */}
|
||||
<VoiceDock
|
||||
onTask={onVoiceTask}
|
||||
onNavigate={(view, taskId) => {
|
||||
const allowed: ViewKey[] = ["home", "studio", "kb", "report", "runs", "memory", "usage"];
|
||||
if (allowed.includes(view as ViewKey)) goto(view as ViewKey, taskId);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</ToastProvider>
|
||||
);
|
||||
|
||||
@@ -27,6 +27,28 @@ export function notify(title: string, body: string): void {
|
||||
if (inWails()) void App.Notify(title, body);
|
||||
}
|
||||
|
||||
// ---- 本地执行 runner（JARVIS「本地的手」，仅桌面壳可用；浏览器一律不可用）----
|
||||
|
||||
export function localRunnerAvailable(): boolean {
|
||||
return inWails();
|
||||
}
|
||||
|
||||
// startLocalRunner：以 workdir 为沙箱根开启本地文件访问（Go host 连 gateway 注册执行器）。
|
||||
export async function startLocalRunner(gatewayURL: string, token: string, workdir: string): Promise<void> {
|
||||
if (!inWails()) throw new Error("本地文件访问仅桌面端可用");
|
||||
await App.StartLocalRunner(gatewayURL, token, workdir);
|
||||
}
|
||||
|
||||
export function stopLocalRunner(): void {
|
||||
if (inWails()) void App.StopLocalRunner();
|
||||
}
|
||||
|
||||
// localRunnerStatus："offline" / "connecting" / "online:<workdir>"；浏览器恒 offline。
|
||||
export async function localRunnerStatus(): Promise<string> {
|
||||
if (!inWails()) return "offline";
|
||||
return App.LocalRunnerStatus();
|
||||
}
|
||||
|
||||
// printReportHtml：把已渲染的报告 HTML 在打印视图里出 PDF（"打印→存为 PDF"）。
|
||||
// 走前端打印是为了让中文(CJK)零字体依赖即可正确排版——后端 PDF 需内嵌 CJK 字体，较重。
|
||||
// 桌面壳内 WKWebView 会把 window.open 拦成 null（实机验过），改走原生桥：
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { micTapAction, nextAfterDrain, shouldSendMic, type VoiceState } from "./voice";
|
||||
|
||||
// 连续对话的纯决策函数：状态门控 / 播完去向 / 点按语义。
|
||||
// VoiceClient 本体依赖 WebSocket/AudioContext（jsdom 难直测），决策逻辑抽纯函数在这测。
|
||||
|
||||
describe("shouldSendMic 上行音频门控", () => {
|
||||
it("只在聆听态上行", () => {
|
||||
expect(shouldSendMic("listening")).toBe(true);
|
||||
});
|
||||
|
||||
it.each<VoiceState>(["idle", "connecting", "ready", "thinking", "speaking"])(
|
||||
"%s 态不上行（防回声喂 ASR + 省计费）",
|
||||
(s) => {
|
||||
expect(shouldSendMic(s)).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("nextAfterDrain 播完去向", () => {
|
||||
it("对话模式回聆听（自动重听）", () => {
|
||||
expect(nextAfterDrain(true)).toBe("listening");
|
||||
});
|
||||
|
||||
it("非对话模式回 ready 待命", () => {
|
||||
expect(nextAfterDrain(false)).toBe("ready");
|
||||
});
|
||||
});
|
||||
|
||||
describe("micTapAction 点按语义（对话开关）", () => {
|
||||
it("聆听中点 → 退出对话", () => {
|
||||
expect(micTapAction("listening", true)).toBe("stop");
|
||||
expect(micTapAction("listening", false)).toBe("stop");
|
||||
});
|
||||
|
||||
it("朗读中点（对话里）→ 打断但留在对话", () => {
|
||||
expect(micTapAction("speaking", true)).toBe("interrupt");
|
||||
});
|
||||
|
||||
it("朗读中点（非对话）→ 开始对话（startConversation 内部先打断）", () => {
|
||||
expect(micTapAction("speaking", false)).toBe("start");
|
||||
});
|
||||
|
||||
it.each<VoiceState>(["idle", "ready", "connecting", "thinking"])("%s 态点 → 开始对话", (s) => {
|
||||
expect(micTapAction(s, false)).toBe("start");
|
||||
expect(micTapAction(s, true)).toBe("start");
|
||||
});
|
||||
});
|
||||
@@ -15,9 +15,38 @@ export interface VoiceCallbacks {
|
||||
onTranscript?: (text: string, final: boolean) => void;
|
||||
onTask?: (taskId: string) => void;
|
||||
onReply?: (deltaText: string) => void; // Agent 回答增量文本（打字机，逐 token）
|
||||
onTurnStart?: () => void; // 新一轮开始（含自动重听）——UI 清上一轮转写/回答
|
||||
onAction?: (action: string, view: string, taskId: string) => void; // JARVIS 界面动作（navigate 等，白名单执行）
|
||||
onAnnounce?: (text: string) => void; // 主动播报文本（随后照常来 speaking/音频/tts_end）
|
||||
onError?: (msg: string) => void;
|
||||
}
|
||||
|
||||
// ---- 连续对话的纯决策函数（抽出便于单测；VoiceClient/VoiceDock 共用）----
|
||||
|
||||
// shouldSendMic：当前状态是否应上行麦克风音频。对话模式麦克风常开，
|
||||
// 但只在聆听时发——thinking/speaking 不发（防扬声器回声喂给 ASR + 省 ASR 时长计费）。
|
||||
export function shouldSendMic(state: VoiceState): boolean {
|
||||
return state === "listening";
|
||||
}
|
||||
|
||||
// nextAfterDrain：TTS 排队音频播完后的去向。对话模式回聆听继续多轮，否则回 ready 待命。
|
||||
export function nextAfterDrain(conversation: boolean): VoiceState {
|
||||
return conversation ? "listening" : "ready";
|
||||
}
|
||||
|
||||
// micTapAction：点按麦克风的语义（对话开关）。
|
||||
// 聆听中点 → 退出对话；朗读中点（在对话里）→ 打断但留在对话；其余 → 开始对话。
|
||||
export type MicTapAction = "start" | "stop" | "interrupt";
|
||||
export function micTapAction(state: VoiceState, inConversation: boolean): MicTapAction {
|
||||
if (state === "listening") return "stop";
|
||||
if (state === "speaking" && inConversation) return "interrupt";
|
||||
return "start";
|
||||
}
|
||||
|
||||
// 健壮性定时器时长（仅对话模式生效）
|
||||
const IDLE_EXIT_MS = 30_000; // 聆听空转：30s 无任何转写 → 自动退出对话（防 ASR 长连接白烧计费）
|
||||
const THINK_WATCHDOG_MS = 90_000; // 思考看门狗：final 后 90s 没等到朗读 → 回聆听继续对话
|
||||
|
||||
const UP_SAMPLE_RATE = 16000; // 上行 ASR 采样率（与网关 voice.AudioSampleRate 一致）
|
||||
const DOWN_SAMPLE_RATE = 24000; // 下行 TTS 采样率（与网关 voice.TTSSampleRate 一致）
|
||||
|
||||
@@ -67,6 +96,12 @@ export class VoiceClient {
|
||||
private lvlBuf = new Uint8Array(512); // 复用的时域采样缓冲（level() 每帧读，别每帧新建）
|
||||
private drainTimer: number | null = null; // tts_end 后等排队音频放完的定时器
|
||||
|
||||
// 对话模式（免按键连续多轮）：麦克风常开、VAD 自动断句提交、答完自动重听
|
||||
private conversation = false;
|
||||
private idleTimer: number | null = null; // 聆听空转定时器
|
||||
private idleDeadline = 0; // 空转退出时刻（epoch ms），UI 读它画最后 10s 倒计时环
|
||||
private thinkTimer: number | null = null; // 思考看门狗定时器
|
||||
|
||||
constructor(cb: VoiceCallbacks) {
|
||||
this.cb = cb;
|
||||
}
|
||||
@@ -94,6 +129,9 @@ export class VoiceClient {
|
||||
reject(new Error("ws error"));
|
||||
};
|
||||
ws.onclose = () => {
|
||||
this.conversation = false; // 连接断了对话就断，别留定时器空转
|
||||
this.clearIdleTimer();
|
||||
this.clearThinkTimer();
|
||||
this.stopMic();
|
||||
if (this.state !== "idle") this.setState("idle");
|
||||
};
|
||||
@@ -103,7 +141,7 @@ export class VoiceClient {
|
||||
|
||||
private onMessage(e: MessageEvent) {
|
||||
if (typeof e.data === "string") {
|
||||
let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string };
|
||||
let m: { type: string; text?: string; final?: boolean; task_id?: string; msg?: string; action?: string; view?: string };
|
||||
try {
|
||||
m = JSON.parse(e.data);
|
||||
} catch {
|
||||
@@ -115,7 +153,13 @@ export class VoiceClient {
|
||||
break;
|
||||
case "transcript":
|
||||
this.cb.onTranscript?.(m.text ?? "", !!m.final);
|
||||
if (m.final) this.setState("thinking");
|
||||
if (m.final) {
|
||||
this.clearIdleTimer();
|
||||
this.setState("thinking");
|
||||
if (this.conversation) this.armThinkTimer(); // 看门狗：任务失败/无 TTS 也能回到聆听
|
||||
} else if (this.conversation && this.state === "listening") {
|
||||
this.armIdleTimer(); // 有声音活动 → 空转计时重来
|
||||
}
|
||||
break;
|
||||
case "task":
|
||||
if (m.task_id) this.cb.onTask?.(m.task_id);
|
||||
@@ -124,13 +168,41 @@ export class VoiceClient {
|
||||
if (m.text) this.cb.onReply?.(m.text); // 打字机：回答增量文本
|
||||
break;
|
||||
case "speaking":
|
||||
this.clearThinkTimer();
|
||||
this.setState("speaking");
|
||||
break;
|
||||
case "tts_end":
|
||||
// 别 resetPlayback！服务端"音频发完" ≠ 客户端"播完"：音频按 nextStart 预约到未来时刻播，
|
||||
// 而火山合成远快于真实语速，收到 tts_end 时大半音频还排在队列里没播。stop 掉就只剩前几个字。
|
||||
// 让排队音频自然放完，最后一段结束再回 ready。
|
||||
this.drainThenReady();
|
||||
// 让排队音频自然放完，最后一段结束再回 ready/聆听。
|
||||
this.clearThinkTimer();
|
||||
if (this.state === "speaking") {
|
||||
// 对话模式：此刻就补发 start——服务端重建 ASR 的握手（几百毫秒）与排队音频播放重叠，
|
||||
// 音频一播完立刻能听；上行音频门控在 shouldSendMic，回聆听态前不会发。
|
||||
if (this.conversation) this.send({ type: "start" });
|
||||
this.drainThenReady();
|
||||
} else if (this.state === "thinking") {
|
||||
// 没进朗读就结束（TTS 启动失败/无音频）：别干等看门狗，立刻进下一轮/回待命。
|
||||
if (this.conversation) {
|
||||
this.send({ type: "start" });
|
||||
this.cb.onTurnStart?.();
|
||||
this.setState("listening");
|
||||
this.armIdleTimer();
|
||||
} else {
|
||||
this.setState("ready");
|
||||
}
|
||||
}
|
||||
// 其余态（已打断回聆听等）：忽略——这是上一轮 speak 收尾的迟到 tts_end，
|
||||
// 再发 start 会把用户已开口的新一轮重置掉。
|
||||
break;
|
||||
case "action":
|
||||
// JARVIS 界面动作（P2 动作通道）：交给 UI 层白名单执行，客户端不盲信。
|
||||
this.cb.onAction?.(m.action ?? "", m.view ?? "", m.task_id ?? "");
|
||||
break;
|
||||
case "announce":
|
||||
// 主动播报（P3）：文本进对话流；音频随后照常走 speaking/二进制帧/tts_end,
|
||||
// 现有状态机零新分支（speaking 会关上行门控，播完 drain 回聆听）。
|
||||
if (m.text) this.cb.onAnnounce?.(m.text);
|
||||
break;
|
||||
case "error":
|
||||
this.cb.onError?.(m.msg ?? "语音出错");
|
||||
@@ -197,6 +269,96 @@ export class VoiceClient {
|
||||
this.resetPlayback();
|
||||
}
|
||||
|
||||
// ---- 对话模式：免按键连续多轮 ----
|
||||
|
||||
inConversation(): boolean {
|
||||
return this.conversation;
|
||||
}
|
||||
|
||||
// startConversation 进入连续对话：麦克风常开，之后 VAD 自动断句提交、答完自动重听，
|
||||
// 直到 stopConversation（点按退出 / 空转超时）。
|
||||
async startConversation(graph?: string): Promise<void> {
|
||||
await this.connect();
|
||||
this.ensurePlayCtx(); // 用户手势内建好播放上下文（自动播放策略）
|
||||
if (this.state === "speaking") this.bargeIn();
|
||||
this.send({ type: "start", graph });
|
||||
await this.startMic(); // 失败（无麦/无权限）直接抛给 UI，不进对话模式
|
||||
this.conversation = true;
|
||||
this.cb.onTurnStart?.();
|
||||
this.setState("listening");
|
||||
this.armIdleTimer();
|
||||
}
|
||||
|
||||
// stopConversation 退出连续对话：关麦、清定时器、回待命。幂等。
|
||||
stopConversation(): void {
|
||||
this.conversation = false;
|
||||
this.clearIdleTimer();
|
||||
this.clearThinkTimer();
|
||||
this.stopMic();
|
||||
if (this.state !== "idle") this.setState("ready");
|
||||
}
|
||||
|
||||
// interruptAndListen 对话模式里打断朗读并立即回聆听（点按打断，留在对话）。
|
||||
interruptAndListen(): void {
|
||||
this.bargeIn(); // 服务端掐 TTS + 本地清排队音频/drain 定时器
|
||||
this.clearThinkTimer();
|
||||
this.send({ type: "start" }); // 服务端重置轮 + 重建 ASR
|
||||
this.cb.onTurnStart?.();
|
||||
this.setState("listening");
|
||||
this.armIdleTimer();
|
||||
}
|
||||
|
||||
// ---- 健壮性定时器（仅对话模式）----
|
||||
|
||||
private armIdleTimer(): void {
|
||||
this.clearIdleTimer();
|
||||
this.idleDeadline = Date.now() + IDLE_EXIT_MS;
|
||||
this.idleTimer = window.setTimeout(() => {
|
||||
this.idleTimer = null;
|
||||
this.idleDeadline = 0;
|
||||
if (this.conversation && this.state === "listening") {
|
||||
this.stopConversation();
|
||||
this.cb.onError?.("长时间没听到声音，已退出连续对话");
|
||||
}
|
||||
}, IDLE_EXIT_MS);
|
||||
}
|
||||
|
||||
private clearIdleTimer(): void {
|
||||
this.idleDeadline = 0;
|
||||
if (this.idleTimer !== null) {
|
||||
window.clearTimeout(this.idleTimer);
|
||||
this.idleTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
// idleRemainingMs 距空转自动退出还剩多少毫秒；不在「对话中聆听」则返回 null。
|
||||
// UI 用它画最后 10s 的倒计时环（开口即 armIdleTimer 重置，环自然消失）。
|
||||
idleRemainingMs(): number | null {
|
||||
if (!this.conversation || this.state !== "listening" || this.idleDeadline === 0) return null;
|
||||
return Math.max(0, this.idleDeadline - Date.now());
|
||||
}
|
||||
|
||||
private armThinkTimer(): void {
|
||||
this.clearThinkTimer();
|
||||
this.thinkTimer = window.setTimeout(() => {
|
||||
this.thinkTimer = null;
|
||||
if (this.conversation && this.state === "thinking") {
|
||||
this.cb.onError?.("等回答超时，继续聆听");
|
||||
this.send({ type: "start" });
|
||||
this.cb.onTurnStart?.();
|
||||
this.setState("listening");
|
||||
this.armIdleTimer();
|
||||
}
|
||||
}, THINK_WATCHDOG_MS);
|
||||
}
|
||||
|
||||
private clearThinkTimer(): void {
|
||||
if (this.thinkTimer !== null) {
|
||||
window.clearTimeout(this.thinkTimer);
|
||||
this.thinkTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async startMic(): Promise<void> {
|
||||
if (this.micNode) return;
|
||||
// WKWebView（Wails 原生壳）等非安全上下文里 navigator.mediaDevices 可能未暴露——
|
||||
@@ -253,6 +415,8 @@ export class VoiceClient {
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
node.onaudioprocess = (ev) => {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||||
// 对话模式麦克风常开，但只在聆听态上行——thinking/speaking 不发（防回声 + 省 ASR 计费）。
|
||||
if (!shouldSendMic(this.state)) return;
|
||||
const f32 = ev.inputBuffer.getChannelData(0);
|
||||
const ds = downsampleTo16k(f32, ctx.sampleRate);
|
||||
this.ws.send(floatToPCM16(ds));
|
||||
@@ -313,7 +477,8 @@ export class VoiceClient {
|
||||
this.nextStart = 0;
|
||||
}
|
||||
|
||||
// drainThenReady 处理 tts_end：不打断，等排队音频按调度自然放完，最后回 ready。
|
||||
// drainThenReady 处理 tts_end：不打断，等排队音频按调度自然放完，再去下一站——
|
||||
// 对话模式回聆听（自动重听，新一轮），否则回 ready 待命。
|
||||
// speaking 态要维持到真正播完（HUD 频谱靠它读 playAnalyser）；播完把队列清干净。
|
||||
private drainThenReady(): void {
|
||||
if (this.drainTimer !== null) window.clearTimeout(this.drainTimer);
|
||||
@@ -323,7 +488,15 @@ export class VoiceClient {
|
||||
this.drainTimer = null;
|
||||
this.sources = [];
|
||||
this.nextStart = 0;
|
||||
if (this.state === "speaking") this.setState("ready"); // 期间被打断/新一轮改了态就不覆盖
|
||||
if (this.state !== "speaking") return; // 期间被打断/新一轮改了态就不覆盖
|
||||
const next = nextAfterDrain(this.conversation);
|
||||
if (next === "listening") {
|
||||
this.cb.onTurnStart?.(); // 新一轮：UI 清上一轮转写/回答
|
||||
this.setState("listening"); // start 已在 tts_end 时提前发，ASR 握手与播放重叠
|
||||
this.armIdleTimer();
|
||||
} else {
|
||||
this.setState("ready");
|
||||
}
|
||||
}, remainMs + 120); // +120ms 余量，等末尾 source 真正 onended
|
||||
}
|
||||
|
||||
@@ -333,6 +506,9 @@ export class VoiceClient {
|
||||
|
||||
// close 彻底关闭会话（发 bye、停采集/播放、断 WS）。
|
||||
close(): void {
|
||||
this.conversation = false;
|
||||
this.clearIdleTimer();
|
||||
this.clearThinkTimer();
|
||||
this.send({ type: "bye" });
|
||||
this.stopMic();
|
||||
this.resetPlayback();
|
||||
|
||||
@@ -3,11 +3,14 @@ import ReactDOM from "react-dom/client";
|
||||
import App from "./App";
|
||||
import "./index.css";
|
||||
import { applyInitialTheme } from "./lib/theme";
|
||||
import { ErrorBoundary } from "./ui/ErrorBoundary";
|
||||
|
||||
applyInitialTheme(); // 先于渲染设好主题类，避免首屏闪烁
|
||||
|
||||
ReactDOM.createRoot(document.getElementById("root")!).render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
<ErrorBoundary>
|
||||
<App />
|
||||
</ErrorBoundary>
|
||||
</React.StrictMode>,
|
||||
);
|
||||
|
||||
@@ -2,7 +2,8 @@ import { useEffect, useState } from "react";
|
||||
import { Dialog } from "../ui/Dialog";
|
||||
import { Button } from "../ui/Button";
|
||||
import { useToast } from "../ui/Toast";
|
||||
import { getMyJarvis, saveMyJarvis, type JarvisConfig } from "../lib/api";
|
||||
import { getMyJarvis, saveMyJarvis, type JarvisConfig, GATEWAY, getToken } from "../lib/api";
|
||||
import { localRunnerAvailable, localRunnerStatus, startLocalRunner, stopLocalRunner } from "../lib/desktop";
|
||||
|
||||
// 每用户 JARVIS 设置：名字 / 人设 / （高级）自带豆包配置。
|
||||
// 名字与人设归用户自己；豆包配置齐全则语音走用户的账号，否则走系统兜底。
|
||||
@@ -105,12 +106,85 @@ export function JarvisSettings({ open, onClose }: { open: boolean; onClose: () =
|
||||
<Field label="音色 voice_type" value={cfg.tts_voice_type} onChange={(v) => set("tts_voice_type", v)} placeholder="zh_male_m191_uranus_bigtts" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<LocalAccessSection />
|
||||
</div>
|
||||
)}
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
// LocalAccessSection 本地文件访问（JARVIS「本地的手」，只读起步）：用户显式选目录 + 开启，
|
||||
// JARVIS 才能看/读该目录内的文件；随时可关。仅桌面壳可用（浏览器隐藏整个区块）。
|
||||
function LocalAccessSection() {
|
||||
const toast = useToast();
|
||||
const [dir, setDir] = useState("");
|
||||
const [status, setStatus] = useState("offline");
|
||||
const [busy, setBusy] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!localRunnerAvailable()) return;
|
||||
let alive = true;
|
||||
const poll = () => localRunnerStatus().then((s) => alive && setStatus(s)).catch(() => {});
|
||||
poll();
|
||||
const iv = window.setInterval(poll, 2000);
|
||||
return () => {
|
||||
alive = false;
|
||||
window.clearInterval(iv);
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!localRunnerAvailable()) return null;
|
||||
|
||||
const online = status.startsWith("online");
|
||||
const onToggle = async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
if (online || status === "connecting") {
|
||||
stopLocalRunner();
|
||||
setStatus("offline");
|
||||
} else {
|
||||
if (!dir.trim()) throw new Error("先填一个允许 JARVIS 访问的本地目录（绝对路径）");
|
||||
await startLocalRunner(GATEWAY, getToken(), dir.trim());
|
||||
setStatus("connecting");
|
||||
toast.push("success", "本地文件访问已开启（只读，锁定在该目录内）");
|
||||
}
|
||||
} catch (e) {
|
||||
toast.push("error", (e as Error).message);
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-2 rounded-md border border-line bg-ink-900/50 p-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-slate-400">本地文件访问（试验 · 只读）</span>
|
||||
<span className={online ? "text-[11px] text-emerald-400" : "text-[11px] text-slate-500"}>
|
||||
{online ? "已开启" : status === "connecting" ? "连接中…" : "未开启"}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-[11px] leading-relaxed text-slate-500">
|
||||
开启后 JARVIS 能<b>查看/读取</b>下面这个目录里的文件（仅此目录，不能写、不能执行命令）。
|
||||
对话里可以说“看看我工作目录里有什么”。
|
||||
</p>
|
||||
{online ? (
|
||||
<p className="break-all font-mono text-[11px] text-slate-400">{status.slice("online:".length)}</p>
|
||||
) : (
|
||||
<input
|
||||
className="w-full rounded-md border border-line bg-ink-850 px-3 py-1.5 font-mono text-sm text-slate-100 focus:border-brand focus:outline-none"
|
||||
value={dir}
|
||||
onChange={(e) => setDir(e.target.value)}
|
||||
placeholder="/Users/你/Documents/某个目录"
|
||||
/>
|
||||
)}
|
||||
<Button variant={online ? "ghost" : "secondary"} size="sm" onClick={onToggle} disabled={busy}>
|
||||
{online || status === "connecting" ? "关闭本地访问" : "开启本地访问"}
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Field({
|
||||
label,
|
||||
value,
|
||||
|
||||
@@ -1,64 +1,178 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Mic, Square, Loader2, Volume2, X, Settings2, Maximize2 } from "lucide-react";
|
||||
import { VoiceClient, type VoiceState } from "../lib/voice";
|
||||
import { ExternalLink, Loader2, Mic, MicOff, Settings2, Maximize2, Volume2, X } from "lucide-react";
|
||||
import { VoiceClient, micTapAction, type VoiceState } from "../lib/voice";
|
||||
import { JarvisSettings } from "./JarvisSettings";
|
||||
import { JarvisHud } from "./JarvisHud";
|
||||
import { getMyJarvis } from "../lib/api";
|
||||
import { useToast } from "../ui/Toast";
|
||||
import { cn } from "../ui/cn";
|
||||
|
||||
// JARVIS 语音坞：右下角悬浮麦克风。点按说话 → 转写 → 提交任务 → 跳运行页 → 朗读回答。
|
||||
// onTask 把语音触发的 task_id 交回 App，接入既有运行·观测流（与键盘提交同一条路）。
|
||||
// JARVIS 语音坞：右下角悬浮麦克风，连续对话模式（点一次进对话，VAD 自动断句、答完自动重听）。
|
||||
// 气泡是「迷你对话流」——保留最近几轮，旧轮淡化，新轮追加（不清屏丢上下文）。
|
||||
// onTask 不再每轮强制跳运行页（连续对话会被拽走）：任务收进对话流里的芯片，点击才跳。
|
||||
|
||||
interface Props {
|
||||
onTask: (taskId: string) => void;
|
||||
// JARVIS 界面动作（P2 动作通道）：view 已过服务端白名单，App 层再按已知视图执行一次（双保险）。
|
||||
onNavigate?: (view: string, taskId?: string) => void;
|
||||
}
|
||||
|
||||
const HINT: Record<VoiceState, string> = {
|
||||
idle: "点击说话",
|
||||
connecting: "连接中…",
|
||||
ready: "点击说话",
|
||||
listening: "聆听中，再点结束",
|
||||
thinking: "思考中…",
|
||||
speaking: "朗读中，点击打断",
|
||||
// 一轮对话：我说的 + JARVIS 答的 + 本轮触发的任务（芯片入口）。
|
||||
interface Turn {
|
||||
me: string;
|
||||
ai: string;
|
||||
taskId?: string;
|
||||
}
|
||||
|
||||
const KEEP_TURNS = 3; // 对话流保留最近几轮（含当前轮）
|
||||
|
||||
// hintText 状态提示（带人格位：名字来自每用户 JARVIS 设置）。
|
||||
function hintText(state: VoiceState, name: string, idleLeft: number | null): string {
|
||||
if (idleLeft !== null) return `${idleLeft}s 后自动退出 · 说话取消`;
|
||||
switch (state) {
|
||||
case "connecting":
|
||||
return "连接中…";
|
||||
case "listening":
|
||||
return "聆听中 · 说完自动发送 · 点击退出";
|
||||
case "thinking":
|
||||
return `${name} 正在思考`;
|
||||
case "speaking":
|
||||
return "朗读中 · 点击打断";
|
||||
default:
|
||||
return "点击开始对话";
|
||||
}
|
||||
}
|
||||
|
||||
// 呼吸环节奏：同一元素三种状态只变速度——聆听慢呼吸 / 思考更缓 / 朗读加快。
|
||||
const RING_DUR: Partial<Record<VoiceState, string>> = {
|
||||
listening: "2.2s",
|
||||
thinking: "3.2s",
|
||||
speaking: "1.4s",
|
||||
};
|
||||
|
||||
export function VoiceDock({ onTask }: Props) {
|
||||
const DOCK_CSS = `
|
||||
@keyframes jdock-breathe {
|
||||
0%, 100% { transform: scale(1); opacity: .85; }
|
||||
50% { transform: scale(1.12); opacity: .3; }
|
||||
}
|
||||
`;
|
||||
|
||||
// VuBars 实时电平条：聆听读麦克风、朗读读放音（client.level() 已按状态切）。
|
||||
// 「麦是活的/在出声」的信任信号——常开麦不给可见反馈，用户不敢用。
|
||||
function VuBars({ getLevel, active }: { getLevel: () => number; active: boolean }) {
|
||||
const ref = useRef<HTMLDivElement>(null);
|
||||
useEffect(() => {
|
||||
if (!active) return;
|
||||
let raf = 0;
|
||||
const tick = () => {
|
||||
const el = ref.current;
|
||||
if (el) {
|
||||
const lvl = getLevel();
|
||||
for (let i = 0; i < el.children.length; i++) {
|
||||
const jitter = 0.55 + 0.45 * Math.sin(Date.now() / 90 + i * 1.7);
|
||||
(el.children[i] as HTMLElement).style.height = `${3 + lvl * 14 * jitter}px`;
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(tick);
|
||||
};
|
||||
raf = requestAnimationFrame(tick);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [active, getLevel]);
|
||||
return (
|
||||
<div ref={ref} className="flex h-4 items-end gap-[2px]" aria-hidden>
|
||||
{Array.from({ length: 9 }, (_, i) => (
|
||||
<span key={i} className="w-[3px] rounded-[1px] bg-accent-400" style={{ height: 3 }} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function VoiceDock({ onTask, onNavigate }: Props) {
|
||||
const toast = useToast();
|
||||
const clientRef = useRef<VoiceClient | null>(null);
|
||||
const [state, setState] = useState<VoiceState>("idle");
|
||||
const [transcript, setTranscript] = useState(""); // 我说的（ASR 转写）
|
||||
const [reply, setReply] = useState(""); // JARVIS 回答（打字机，逐 token 累加）
|
||||
const [conv, setConv] = useState(false); // 对话模式进行中（呼吸环/静默标识依赖它）
|
||||
const [turns, setTurns] = useState<Turn[]>([]); // 迷你对话流（最近 KEEP_TURNS 轮）
|
||||
const [open, setOpen] = useState(false); // 是否展开对话气泡
|
||||
const [settingsOpen, setSettingsOpen] = useState(false); // JARVIS 设置弹窗
|
||||
const [fullscreen, setFullscreen] = useState(false); // 全屏 JARVIS 模式
|
||||
const [name, setName] = useState("JARVIS"); // 助手名（HUD 品牌位）
|
||||
const [name, setName] = useState("JARVIS"); // 助手名（提示/对话流/HUD 品牌位）
|
||||
const [idleLeft, setIdleLeft] = useState<number | null>(null); // 空转退出倒计时（最后 10s 才非空）
|
||||
|
||||
// onNavigate 经 ref 供长寿命 VoiceClient 回调使用（client 只建一次，闭包别锁死旧 props）。
|
||||
const onNavigateRef = useRef(onNavigate);
|
||||
onNavigateRef.current = onNavigate;
|
||||
|
||||
// 助手名：挂载即拉一次（未登录/失败保持默认）。
|
||||
useEffect(() => {
|
||||
getMyJarvis()
|
||||
.then((j) => setName(j.name || "JARVIS"))
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
// 懒建客户端（首次点按时，带上用户手势→AudioContext 才能启动）。
|
||||
const ensureClient = useCallback((): VoiceClient => {
|
||||
if (!clientRef.current) {
|
||||
clientRef.current = new VoiceClient({
|
||||
onState: setState,
|
||||
onTranscript: (text, final) => {
|
||||
setTranscript(text);
|
||||
if (final) setOpen(true);
|
||||
onState: (s) => {
|
||||
setState(s);
|
||||
setConv(clientRef.current?.inConversation() ?? false);
|
||||
},
|
||||
onTranscript: (text) => {
|
||||
setTurns((ts) => {
|
||||
const last = ts[ts.length - 1] ?? { me: "", ai: "" };
|
||||
return [...ts.slice(0, -1), { ...last, me: text }];
|
||||
});
|
||||
setOpen(true);
|
||||
},
|
||||
// 任务触发：收进当前轮的芯片，点击才跳运行页（连续对话不被拽走）。
|
||||
onTask: (taskId) => {
|
||||
onTask(taskId);
|
||||
setOpen(true); // 留着气泡显示打字机回答
|
||||
setTurns((ts) => {
|
||||
const last = ts[ts.length - 1] ?? { me: "", ai: "" };
|
||||
return [...ts.slice(0, -1), { ...last, taskId }];
|
||||
});
|
||||
},
|
||||
onReply: (delta) => {
|
||||
setReply((r) => r + delta); // 打字机：增量拼接，LLM 首 token 即刻可见
|
||||
setTurns((ts) => {
|
||||
const last = ts[ts.length - 1] ?? { me: "", ai: "" };
|
||||
return [...ts.slice(0, -1), { ...last, ai: last.ai + delta }];
|
||||
});
|
||||
setOpen(true);
|
||||
},
|
||||
// 新一轮开始（含答完自动重听）：对话流追加空轮，旧轮保留淡化——不清屏丢上下文。
|
||||
onTurnStart: () => {
|
||||
setTurns((ts) => [...ts.filter((t) => t.me || t.ai), { me: "", ai: "" }].slice(-KEEP_TURNS));
|
||||
},
|
||||
// JARVIS 界面动作：目前只认 navigate，交 App 层执行（那里再按已知视图核一次）。
|
||||
onAction: (action, view, taskId) => {
|
||||
if (action === "navigate") onNavigateRef.current?.(view, taskId || undefined);
|
||||
},
|
||||
// 主动播报：作为一条 JARVIS 独立轮插进对话流（音频随后照常来）。
|
||||
onAnnounce: (text) => {
|
||||
setTurns((ts) => [...ts.filter((t) => t.me || t.ai), { me: "", ai: text }].slice(-KEEP_TURNS));
|
||||
setOpen(true);
|
||||
},
|
||||
onError: (msg) => toast.push("error", msg),
|
||||
});
|
||||
}
|
||||
return clientRef.current;
|
||||
}, [onTask, toast]);
|
||||
}, [toast]);
|
||||
|
||||
useEffect(() => () => clientRef.current?.close(), []);
|
||||
|
||||
// 空转倒计时轮询：只在「对话中聆听」时跑；最后 10s 显示倒计时环，开口即消。
|
||||
useEffect(() => {
|
||||
if (!(conv && state === "listening")) {
|
||||
setIdleLeft(null);
|
||||
return;
|
||||
}
|
||||
const iv = window.setInterval(() => {
|
||||
const ms = clientRef.current?.idleRemainingMs() ?? null;
|
||||
setIdleLeft(ms !== null && ms <= 10_000 ? Math.ceil(ms / 1000) : null);
|
||||
}, 250);
|
||||
return () => window.clearInterval(iv);
|
||||
}, [conv, state]);
|
||||
|
||||
// Esc 退出全屏 JARVIS 模式。
|
||||
useEffect(() => {
|
||||
if (!fullscreen) return;
|
||||
@@ -67,23 +181,30 @@ export function VoiceDock({ onTask }: Props) {
|
||||
return () => window.removeEventListener("keydown", onKey);
|
||||
}, [fullscreen]);
|
||||
|
||||
// 点按语义 = 对话开关：待命点 → 进入连续对话；聆听中点 → 退出；朗读中点 → 打断但留在对话。
|
||||
const onMic = useCallback(async () => {
|
||||
const c = ensureClient();
|
||||
try {
|
||||
if (state === "listening") {
|
||||
c.stopListening();
|
||||
} else {
|
||||
setTranscript("");
|
||||
setReply(""); // 新一轮：清上一轮的回答
|
||||
setOpen(true);
|
||||
await c.startListening(); // speaking 中会先打断再开新一轮
|
||||
switch (micTapAction(state, c.inConversation())) {
|
||||
case "stop":
|
||||
c.stopConversation();
|
||||
break;
|
||||
case "interrupt":
|
||||
c.interruptAndListen();
|
||||
break;
|
||||
case "start":
|
||||
setOpen(true); // 清屏交给 onTurnStart（startConversation 里触发）
|
||||
await c.startConversation();
|
||||
break;
|
||||
}
|
||||
} catch (e) {
|
||||
toast.push("error", (e as Error).message || "麦克风启动失败（检查权限）");
|
||||
} finally {
|
||||
setConv(clientRef.current?.inConversation() ?? false);
|
||||
}
|
||||
}, [ensureClient, state, toast]);
|
||||
|
||||
// 进全屏 JARVIS 模式：建好客户端（这样 HUD 能读实时电平），拉一次助手名做品牌位。
|
||||
// 进全屏 JARVIS 模式：建好客户端（HUD 读实时电平），刷一次助手名。
|
||||
const openFullscreen = useCallback(() => {
|
||||
ensureClient();
|
||||
getMyJarvis()
|
||||
@@ -92,11 +213,16 @@ export function VoiceDock({ onTask }: Props) {
|
||||
setFullscreen(true);
|
||||
}, [ensureClient]);
|
||||
|
||||
const active = state === "listening";
|
||||
const busy = state === "connecting" || state === "thinking";
|
||||
const busy = state === "connecting";
|
||||
const hint = hintText(state, name, idleLeft);
|
||||
const visibleTurns = turns.filter((t) => t.me || t.ai || t.taskId);
|
||||
const ringDur = conv ? RING_DUR[state] : undefined;
|
||||
// 倒计时环：SVG 周长 182.2（r=29），剩余秒数映射到 dashoffset（收缩）。
|
||||
const cdOffset = idleLeft !== null ? (182.2 * (10 - idleLeft)) / 10 : 0;
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{DOCK_CSS}</style>
|
||||
<div className="pointer-events-none fixed bottom-6 right-6 z-40 flex flex-col items-end gap-2">
|
||||
{/* 小工具：全屏 JARVIS 模式 + 设置 */}
|
||||
<div className="pointer-events-auto flex items-center gap-2">
|
||||
@@ -104,7 +230,7 @@ export function VoiceDock({ onTask }: Props) {
|
||||
onClick={openFullscreen}
|
||||
title="全屏 JARVIS 模式"
|
||||
aria-label="全屏 JARVIS 模式"
|
||||
className="flex h-8 w-8 items-center justify-center rounded-full border border-line bg-ink-850/90 text-slate-400 shadow-md backdrop-blur transition hover:border-brand hover:text-brand-300"
|
||||
className="flex h-8 w-8 items-center justify-center rounded-full border border-line bg-ink-850/90 text-slate-400 shadow-md backdrop-blur transition hover:border-brand hover:text-accent-400"
|
||||
>
|
||||
<Maximize2 className="h-4 w-4" />
|
||||
</button>
|
||||
@@ -118,24 +244,44 @@ export function VoiceDock({ onTask }: Props) {
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* 对话气泡：我说的（转写）+ JARVIS 回答（打字机） */}
|
||||
{open && (transcript || reply) && (
|
||||
<div className="pointer-events-auto max-w-xs rounded-2xl border border-line bg-ink-850/95 px-4 py-3 text-sm shadow-xl backdrop-blur">
|
||||
{/* 迷你对话流：最近几轮，旧轮淡化，新轮在底部 */}
|
||||
{open && visibleTurns.length > 0 && (
|
||||
<div className="pointer-events-auto w-64 max-w-xs rounded-2xl border border-line bg-ink-850/95 px-4 py-3 text-sm shadow-xl backdrop-blur">
|
||||
<div className="flex items-start gap-2">
|
||||
<div className="flex-1 space-y-1.5">
|
||||
{transcript && (
|
||||
<p className="leading-relaxed text-slate-400">
|
||||
<span className="mr-1 text-[11px] text-slate-500">我</span>
|
||||
{transcript}
|
||||
</p>
|
||||
)}
|
||||
{reply && (
|
||||
<p className="leading-relaxed text-slate-100">
|
||||
<span className="mr-1 text-[11px] text-brand-300">JARVIS</span>
|
||||
{reply}
|
||||
{state === "thinking" && <span className="ml-0.5 inline-block h-3.5 w-[2px] translate-y-[2px] animate-pulse bg-brand-300 align-middle" />}
|
||||
</p>
|
||||
)}
|
||||
<div className="flex-1 space-y-2">
|
||||
{visibleTurns.map((t, i) => {
|
||||
const current = i === visibleTurns.length - 1;
|
||||
return (
|
||||
<div key={i} className={cn("space-y-1", !current && "opacity-40")}>
|
||||
{t.me && (
|
||||
<p className="leading-relaxed text-slate-400">
|
||||
<span className="mr-1 text-[11px] text-slate-500">我</span>
|
||||
{t.me}
|
||||
</p>
|
||||
)}
|
||||
{(t.ai || (current && state === "thinking")) && (
|
||||
<p className="leading-relaxed text-slate-100">
|
||||
<span className="mr-1 text-[11px] text-accent-400">{name}</span>
|
||||
{t.ai || <span className="text-slate-400">正在思考…</span>}
|
||||
{current && (state === "thinking" || state === "speaking") && t.ai && (
|
||||
<span className="ml-0.5 inline-block h-3.5 w-[2px] translate-y-[2px] animate-pulse bg-accent-400 align-middle" />
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
{t.taskId && (
|
||||
<button
|
||||
onClick={() => onTask(t.taskId!)}
|
||||
title="查看运行轨迹"
|
||||
className="flex w-full items-center gap-1.5 rounded-lg border border-ink-600/60 bg-ink-800/80 px-2 py-1 text-[11px] text-slate-300 transition hover:border-brand hover:text-accent-400"
|
||||
>
|
||||
<span className="h-1.5 w-1.5 rounded-full bg-brand" />
|
||||
任务 · {t.taskId.slice(-8)}
|
||||
<ExternalLink className="ml-auto h-3 w-3" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<button className="mt-0.5 text-slate-500 hover:text-slate-300" onClick={() => setOpen(false)} aria-label="收起">
|
||||
<X className="h-3.5 w-3.5" />
|
||||
@@ -144,36 +290,66 @@ export function VoiceDock({ onTask }: Props) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 麦克风按钮 */}
|
||||
<button
|
||||
onClick={onMic}
|
||||
title={HINT[state]}
|
||||
aria-label={HINT[state]}
|
||||
className={cn(
|
||||
"pointer-events-auto relative flex h-14 w-14 items-center justify-center rounded-full shadow-xl transition",
|
||||
"focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
|
||||
active
|
||||
? "bg-danger text-white"
|
||||
: state === "speaking"
|
||||
? "bg-brand text-white"
|
||||
{/* 电平/静默指示 + 麦克风按钮 */}
|
||||
<div className="pointer-events-auto flex items-center gap-2.5">
|
||||
{conv && (state === "listening" || state === "speaking") && (
|
||||
<VuBars getLevel={() => clientRef.current?.level() ?? 0} active />
|
||||
)}
|
||||
{conv && state === "thinking" && (
|
||||
<span className="flex items-center gap-1 rounded-full bg-ink-900/80 px-2.5 py-1 text-[10px] text-slate-400">
|
||||
<MicOff className="h-3 w-3" />
|
||||
麦克风已静默
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
onClick={onMic}
|
||||
title={hint}
|
||||
aria-label={hint}
|
||||
className={cn(
|
||||
"relative flex h-14 w-14 items-center justify-center rounded-full shadow-xl transition",
|
||||
"focus:outline-none focus-visible:ring-2 focus-visible:ring-brand/60",
|
||||
conv && state === "thinking"
|
||||
? "bg-ink-800 text-accent-400"
|
||||
: "bg-brand text-white hover:bg-brand-500 active:scale-95",
|
||||
)}
|
||||
>
|
||||
{active && <span className="absolute inset-0 animate-ping rounded-full bg-danger/40" />}
|
||||
{busy ? (
|
||||
<Loader2 className="h-6 w-6 animate-spin" />
|
||||
) : state === "speaking" ? (
|
||||
<Volume2 className="h-6 w-6" />
|
||||
) : active ? (
|
||||
<Square className="h-5 w-5" />
|
||||
) : (
|
||||
<Mic className="h-6 w-6" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
>
|
||||
{/* 呼吸环：对话进行中常亮（三种状态只变节奏）——环在=点击是退出/打断，环灭=点击是开始 */}
|
||||
{ringDur && (
|
||||
<span
|
||||
className="pointer-events-none absolute -inset-1.5 rounded-full border-2 border-accent-400"
|
||||
style={{ animation: `jdock-breathe ${ringDur} ease-in-out infinite` }}
|
||||
/>
|
||||
)}
|
||||
{/* 空转退出倒计时环（最后 10s，琥珀收缩） */}
|
||||
{idleLeft !== null && (
|
||||
<svg viewBox="0 0 64 64" className="pointer-events-none absolute -inset-1.5 -rotate-90" aria-hidden>
|
||||
<circle cx="32" cy="32" r="29" fill="none" stroke="rgba(148,163,184,.25)" strokeWidth="2.5" />
|
||||
<circle
|
||||
cx="32" cy="32" r="29" fill="none" stroke="#e0a94f" strokeWidth="2.5" strokeLinecap="round"
|
||||
strokeDasharray="182.2" strokeDashoffset={cdOffset}
|
||||
/>
|
||||
</svg>
|
||||
)}
|
||||
{busy ? (
|
||||
<Loader2 className="h-6 w-6 animate-spin" />
|
||||
) : state === "speaking" ? (
|
||||
<Volume2 className="h-6 w-6" />
|
||||
) : state === "thinking" && conv ? (
|
||||
<Loader2 className="h-6 w-6 animate-spin" />
|
||||
) : (
|
||||
<Mic className="h-6 w-6" />
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* 状态提示 */}
|
||||
<span className="pointer-events-none rounded-full bg-ink-900/80 px-2.5 py-0.5 text-[11px] text-slate-400">
|
||||
{HINT[state]}
|
||||
<span
|
||||
className={cn(
|
||||
"pointer-events-none rounded-full bg-ink-900/80 px-2.5 py-0.5 text-[11px]",
|
||||
idleLeft !== null ? "text-amber-400" : "text-slate-400",
|
||||
)}
|
||||
>
|
||||
{hint}
|
||||
</span>
|
||||
</div>
|
||||
<JarvisSettings open={settingsOpen} onClose={() => setSettingsOpen(false)} />
|
||||
@@ -182,9 +358,9 @@ export function VoiceDock({ onTask }: Props) {
|
||||
name={name}
|
||||
state={state}
|
||||
getLevel={() => clientRef.current?.level() ?? 0}
|
||||
transcript={transcript}
|
||||
reply={reply}
|
||||
hint={HINT[state]}
|
||||
transcript={visibleTurns[visibleTurns.length - 1]?.me ?? ""}
|
||||
reply={visibleTurns[visibleTurns.length - 1]?.ai ?? ""}
|
||||
hint={hint}
|
||||
onMic={onMic}
|
||||
onClose={() => setFullscreen(false)}
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { Component, type ReactNode } from "react";
|
||||
|
||||
// 全局错误边界：任何组件渲染崩溃时显示错误卡片，而不是 React 整树卸载变白屏。
|
||||
// （曾经：stats 空数据 → Home 读 null.length 崩 → 用户只看到"闪一下变白"，毫无线索。）
|
||||
|
||||
interface Props {
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
interface State {
|
||||
error: Error | null;
|
||||
}
|
||||
|
||||
export class ErrorBoundary extends Component<Props, State> {
|
||||
state: State = { error: null };
|
||||
|
||||
static getDerivedStateFromError(error: Error): State {
|
||||
return { error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error): void {
|
||||
console.error("[ErrorBoundary] 界面渲染崩溃:", error);
|
||||
}
|
||||
|
||||
render() {
|
||||
if (!this.state.error) return this.props.children;
|
||||
return (
|
||||
<div className="flex h-screen items-center justify-center bg-ink-950 p-8">
|
||||
<div className="max-w-md rounded-xl border border-line bg-ink-900 p-6 text-center">
|
||||
<div className="text-base font-medium text-slate-100">界面出错了</div>
|
||||
<p className="mt-2 break-all text-xs leading-relaxed text-slate-500">
|
||||
{this.state.error.message}
|
||||
</p>
|
||||
<button
|
||||
onClick={() => window.location.reload()}
|
||||
className="mt-4 rounded-lg bg-brand px-4 py-1.5 text-sm text-white transition hover:bg-brand-500"
|
||||
>
|
||||
重新加载
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -93,11 +93,13 @@ export function Home({ onSelect, userName, spaceName }: { onSelect: (v: ViewKey,
|
||||
return () => { alive = false; clearInterval(id); };
|
||||
}, []);
|
||||
|
||||
const taskTrend = ov?.task_trend.map((d) => d.count) ?? [];
|
||||
const tokenTrend = ov?.token_trend.map((d) => d.count) ?? [];
|
||||
// 注意：后端空数据时 nil slice 序列化成 null（不是 []）——可选链只保护 ov 不保护字段，
|
||||
// `ov?.task_trend.map` 在 task_trend=null 时照样崩（曾导致新账号进来整页白屏）。一律 ?? [] 兜底。
|
||||
const taskTrend = (ov?.task_trend ?? []).map((d) => d.count);
|
||||
const tokenTrend = (ov?.token_trend ?? []).map((d) => d.count);
|
||||
const maxTask = Math.max(...taskTrend, 1);
|
||||
const fmtTokens = (n: number) => (n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${(n / 1e3).toFixed(1)}K` : `${n}`);
|
||||
const hasRuns = !!ov && ov.recent_runs.length > 0;
|
||||
const hasRuns = (ov?.recent_runs ?? []).length > 0;
|
||||
const quality = ov?.eval_count ? `${Math.round((ov.eval_avg ?? 0) * 100)}%` : "—";
|
||||
|
||||
return (
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
// 本地执行 runner（JARVIS「本地的手」，LOCAL_AGENT_DESIGN 档 A / JARVIS_BRAIN_DESIGN P4）：
|
||||
// 桌面端 Go host 连 gateway 的 /api/v1/local/runner WS，把自己注册成本用户的本地执行器；
|
||||
// 服务端把 local_* 工具调用转发过来，这里在**用户自选工作目录的沙箱内**执行并回结果。
|
||||
//
|
||||
// 安全铁律（P1 只读起步）：
|
||||
// - 只实现 list_dir / read_file，无写无 exec；
|
||||
// - 一切路径锁死在用户显式选择的 workdir 根下（清洗 + 软链解析后前缀校验，越界即拒）；
|
||||
// - 用户不点"开启"，runner 永不连接——本地访问是显式授权，不是默认能力。
|
||||
|
||||
type runnerReq struct {
|
||||
ID string `json:"id"`
|
||||
Tool string `json:"tool"`
|
||||
Args map[string]any `json:"args,omitempty"`
|
||||
}
|
||||
|
||||
type runnerResp struct {
|
||||
ID string `json:"id"`
|
||||
OK bool `json:"ok"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Workdir string `json:"workdir,omitempty"`
|
||||
}
|
||||
|
||||
// LocalRunner 管一条 runner 连接的生命周期（App 持有单例）。
|
||||
type LocalRunner struct {
|
||||
mu sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
workdir string
|
||||
status string // offline / connecting / online
|
||||
}
|
||||
|
||||
// StartLocalRunner 开启本地文件访问：以 workdir 为沙箱根连接 gateway 注册执行器。
|
||||
// 幂等：重复调用先停旧连接。断线自动重连（5s 退避）直到 StopLocalRunner。
|
||||
func (a *App) StartLocalRunner(gatewayURL, token, workdir string) error {
|
||||
abs, err := filepath.Abs(strings.TrimSpace(workdir))
|
||||
if err != nil {
|
||||
return fmt.Errorf("工作目录无效: %w", err)
|
||||
}
|
||||
if fi, err := os.Stat(abs); err != nil || !fi.IsDir() {
|
||||
return fmt.Errorf("工作目录不存在或不是目录: %s", abs)
|
||||
}
|
||||
a.runner.stop()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
a.runner.mu.Lock()
|
||||
a.runner.cancel = cancel
|
||||
a.runner.workdir = abs
|
||||
a.runner.status = "connecting"
|
||||
a.runner.mu.Unlock()
|
||||
|
||||
wsURL := strings.Replace(strings.TrimRight(gatewayURL, "/"), "http", "ws", 1) +
|
||||
"/api/v1/local/runner?token=" + token
|
||||
go a.runner.loop(ctx, wsURL, abs)
|
||||
return nil
|
||||
}
|
||||
|
||||
// StopLocalRunner 关闭本地文件访问（幂等）。
|
||||
func (a *App) StopLocalRunner() { a.runner.stop() }
|
||||
|
||||
// LocalRunnerStatus 返回 "offline" / "connecting" / "online:<workdir>"（前端状态显示）。
|
||||
func (a *App) LocalRunnerStatus() string {
|
||||
a.runner.mu.Lock()
|
||||
defer a.runner.mu.Unlock()
|
||||
if a.runner.status == "online" {
|
||||
return "online:" + a.runner.workdir
|
||||
}
|
||||
return a.runner.status
|
||||
}
|
||||
|
||||
func (r *LocalRunner) stop() {
|
||||
r.mu.Lock()
|
||||
if r.cancel != nil {
|
||||
r.cancel()
|
||||
r.cancel = nil
|
||||
}
|
||||
r.status = "offline"
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *LocalRunner) setStatus(s string) {
|
||||
r.mu.Lock()
|
||||
r.status = s
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// loop 连接→服务→断线重连（5s 退避），直到 ctx 取消。
|
||||
func (r *LocalRunner) loop(ctx context.Context, wsURL, root string) {
|
||||
for {
|
||||
if err := r.serve(ctx, wsURL, root); err != nil && ctx.Err() == nil {
|
||||
r.setStatus("connecting")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.setStatus("offline")
|
||||
return
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serve 一条连接的会话：发 hello → 循环收请求、沙箱内执行、回结果。
|
||||
func (r *LocalRunner) serve(ctx context.Context, wsURL, root string) error {
|
||||
dctx, dcancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
conn, _, err := websocket.Dial(dctx, wsURL, nil)
|
||||
dcancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close(websocket.StatusNormalClosure, "bye")
|
||||
conn.SetReadLimit(1 << 20)
|
||||
r.setStatus("online")
|
||||
|
||||
hello, _ := json.Marshal(runnerResp{ID: "hello", OK: true, Workdir: root})
|
||||
if err := conn.Write(ctx, websocket.MessageText, hello); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
_, data, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var req runnerReq
|
||||
if json.Unmarshal(data, &req) != nil {
|
||||
continue
|
||||
}
|
||||
resp := execLocal(root, &req)
|
||||
out, _ := json.Marshal(resp)
|
||||
if err := conn.Write(ctx, websocket.MessageText, out); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 沙箱执行（只读）----
|
||||
|
||||
const (
|
||||
maxReadBytes = 64 * 1024 // read_file 上限：64KB，超出截断（语音/对话场景足够）
|
||||
maxDirEntries = 200 // list_dir 上限条数
|
||||
)
|
||||
|
||||
// resolveInRoot 把相对路径解析进沙箱根：清洗 + 软链解析后必须仍在 root 下，越界即错。
|
||||
func resolveInRoot(root, rel string) (string, error) {
|
||||
p := filepath.Join(root, filepath.Clean("/"+rel)) // 前置 "/" 再 Clean：吃掉 ../ 逃逸
|
||||
// 软链解析（目标可能不存在：解析其父目录）
|
||||
resolved, err := filepath.EvalSymlinks(p)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
resolved = p
|
||||
}
|
||||
rootR, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
rootR = root
|
||||
}
|
||||
if resolved != rootR && !strings.HasPrefix(resolved, rootR+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("路径越出工作目录沙箱")
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func execLocal(root string, req *runnerReq) *runnerResp {
|
||||
fail := func(msg string) *runnerResp { return &runnerResp{ID: req.ID, OK: false, Error: msg} }
|
||||
rel, _ := req.Args["path"].(string)
|
||||
|
||||
switch req.Tool {
|
||||
case "local_list_dir":
|
||||
p, err := resolveInRoot(root, rel)
|
||||
if err != nil {
|
||||
return fail(err.Error())
|
||||
}
|
||||
ents, err := os.ReadDir(p)
|
||||
if err != nil {
|
||||
return fail("读目录失败: " + err.Error())
|
||||
}
|
||||
sort.Slice(ents, func(i, j int) bool { return ents[i].Name() < ents[j].Name() })
|
||||
type item struct {
|
||||
Name string `json:"name"`
|
||||
Dir bool `json:"dir"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
}
|
||||
out := make([]item, 0, len(ents))
|
||||
for i, e := range ents {
|
||||
if i >= maxDirEntries {
|
||||
break
|
||||
}
|
||||
it := item{Name: e.Name(), Dir: e.IsDir()}
|
||||
if fi, err := e.Info(); err == nil && !e.IsDir() {
|
||||
it.Size = fi.Size()
|
||||
}
|
||||
out = append(out, it)
|
||||
}
|
||||
data, _ := json.Marshal(map[string]any{"dir": rel, "entries": out, "truncated": len(ents) > maxDirEntries})
|
||||
return &runnerResp{ID: req.ID, OK: true, Content: string(data)}
|
||||
|
||||
case "local_read_file":
|
||||
if strings.TrimSpace(rel) == "" {
|
||||
return fail("缺少文件路径")
|
||||
}
|
||||
p, err := resolveInRoot(root, rel)
|
||||
if err != nil {
|
||||
return fail(err.Error())
|
||||
}
|
||||
fi, err := os.Stat(p)
|
||||
if err != nil {
|
||||
return fail("文件不存在: " + rel)
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return fail("这是目录不是文件: " + rel)
|
||||
}
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
return fail("打开失败: " + err.Error())
|
||||
}
|
||||
defer f.Close()
|
||||
buf := make([]byte, maxReadBytes+1)
|
||||
n, _ := f.Read(buf)
|
||||
content := string(buf[:min(n, maxReadBytes)])
|
||||
if n > maxReadBytes {
|
||||
content += "\n…（文件过大，已截断到 64KB）"
|
||||
}
|
||||
return &runnerResp{ID: req.ID, OK: true, Content: content}
|
||||
|
||||
default:
|
||||
return fail("本地执行器不支持该操作: " + req.Tool + "（只读版仅 list_dir/read_file）")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 沙箱是本地执行的安全命门：路径清洗/软链逃逸/越界读全都要拒——这里逐项钉死。
|
||||
|
||||
func newSandbox(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("hello"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "sub"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "sub", "b.txt"), []byte("world"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestResolveInRootBlocksEscape(t *testing.T) {
|
||||
root := newSandbox(t)
|
||||
// 断言基准要用软链解析后的 root：macOS 的 TempDir 在 /var（→/private/var 软链）下，
|
||||
// resolveInRoot 返回的是解析后的绝对路径，拿未解析 root 做前缀比较会误报。
|
||||
rootR, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
rootR = root
|
||||
}
|
||||
for _, rel := range []string{"..", "../..", "../../etc/passwd", "sub/../../outside", "/etc/passwd"} {
|
||||
p, err := resolveInRoot(root, rel)
|
||||
// 前置 "/"+Clean 把绝对路径/.. 都钉回 root 下（如 root/etc/passwd，不算逃逸）；
|
||||
// 无论哪种形式，成功解析的结果都必须仍在 root（解析后）之内。
|
||||
if err == nil && p != rootR && !strings.HasPrefix(p, rootR+string(filepath.Separator)) {
|
||||
t.Fatalf("路径 %q 逃出了沙箱: %s", rel, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveInRootBlocksSymlinkEscape(t *testing.T) {
|
||||
root := newSandbox(t)
|
||||
outside := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("secret"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(root, "evil")
|
||||
if err := os.Symlink(outside, link); err != nil {
|
||||
t.Skip("无法创建软链，跳过")
|
||||
}
|
||||
if _, err := resolveInRoot(root, "evil/secret.txt"); err == nil {
|
||||
t.Fatal("软链逃逸未被拦截")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecLocalListAndRead(t *testing.T) {
|
||||
root := newSandbox(t)
|
||||
|
||||
// list_dir 根目录
|
||||
resp := execLocal(root, &runnerReq{ID: "1", Tool: "local_list_dir", Args: map[string]any{"path": ""}})
|
||||
if !resp.OK {
|
||||
t.Fatalf("list_dir 失败: %s", resp.Error)
|
||||
}
|
||||
var listing struct {
|
||||
Entries []struct {
|
||||
Name string `json:"name"`
|
||||
Dir bool `json:"dir"`
|
||||
} `json:"entries"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(resp.Content), &listing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(listing.Entries) != 2 {
|
||||
t.Fatalf("期望 2 项，得 %d", len(listing.Entries))
|
||||
}
|
||||
|
||||
// read_file 子目录文件
|
||||
resp = execLocal(root, &runnerReq{ID: "2", Tool: "local_read_file", Args: map[string]any{"path": "sub/b.txt"}})
|
||||
if !resp.OK || resp.Content != "world" {
|
||||
t.Fatalf("read_file 失败: ok=%v content=%q err=%s", resp.OK, resp.Content, resp.Error)
|
||||
}
|
||||
|
||||
// read_file 越界必须拒
|
||||
resp = execLocal(root, &runnerReq{ID: "3", Tool: "local_read_file", Args: map[string]any{"path": "../outside.txt"}})
|
||||
if resp.OK {
|
||||
t.Fatal("越界读未被拦截")
|
||||
}
|
||||
|
||||
// 未知工具（写/exec 都不在只读版里）必须拒
|
||||
resp = execLocal(root, &runnerReq{ID: "4", Tool: "local_write_file", Args: map[string]any{"path": "a.txt"}})
|
||||
if resp.OK {
|
||||
t.Fatal("未注册操作未被拦截")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user