feat(observability): 执行可视化 — 节点级实时轨迹(运行·观测)
把任务执行做成可观测:Dispatcher 在每个节点/阶段发结构化 ExecEvent, 经独立 NATS 通道回流,前端逐节点点亮(状态/耗时/工具入参产出)。 - shared: contract.ExecEvent + ExecSubject(sundynix.exec.<id>,与 Token 流分流); bus.PublishExec/CompleteExec/SubscribeExec(core NATS,复用结束头) - dispatcher: execTracer(自增 Seq 保序 + span 自动计耗时); Orchestrator 加 ExecSink;通用图(init 召回 / 各 tool 入参→产出 / prompt / model 首token+token数)与报告编排(规划大纲 / 各章并行 start-end / 渲染)全程埋点 - gateway: SubscribeExec + GET /tasks/:id/exec SSE(与 token 流并行) - desktop: streamExec + deriveNodes(按 node 归并 start/end/error/info); 复用组件 ExecTrace(竖向轨道,按 kind 着色,运行中脉冲灯); 新 RunsView(运行·观测:轨迹+输出双栏);BottomDrawer 轨迹/工具调用 tab 接真实数据; ReportView 加执行轨迹栏;左导航「运行」置就绪 实测: - 报告任务 /exec:规划(2680ms,4章) → 4 章并行(seq 交错,各~7-8s 重叠=真并行, 每章带 docs 知识库检索预览+成稿字数) → 渲染(docx 落盘) - 通用图 /exec:tool:kb_search(678ms,入参→Milvus 产出) → prompt(2消息) → model(首token 860ms / 4 tokens) - 浏览器(Preview):报告页执行轨迹逐节点点亮、章节带耗时/字数/检索片段,完成后下载 Word Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ package eino
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -30,7 +31,7 @@ type RunCtx struct {
|
||||
//
|
||||
// 工具/检索节点按拓扑序真实调用 MCP(sundynix.tools.go.*),结果注入模型上下文。
|
||||
// 分支/并行节点暂未编译(TODO:compose.Branch / fan-out)。
|
||||
func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task) (compose.Runnable[*contract.Task, *schema.Message], error) {
|
||||
func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task, tr *execTracer) (compose.Runnable[*contract.Task, *schema.Message], error) {
|
||||
plan := dsl.Compile(t.Graph) // 系统提示词 / 用户输入 / 默认兜底
|
||||
flow, _ := dsl.Parse(t.Graph)
|
||||
|
||||
@@ -41,13 +42,17 @@ func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task) (compo
|
||||
func(ctx context.Context, task *contract.Task) (*RunCtx, error) {
|
||||
uid, _ := task.Meta[contract.MetaUserID].(string)
|
||||
sid, _ := task.Meta[contract.MetaSessionID].(string)
|
||||
end := tr.span("init", "memory", "召回画像与历史")
|
||||
profile := o.fetchMemory(ctx, uid, plan.Query)
|
||||
history := o.fetchHistory(ctx, sid)
|
||||
end(fmt.Sprintf("画像 %d 字 · 历史 %d 条", len([]rune(profile)), len(history)), nil)
|
||||
return &RunCtx{
|
||||
UserID: uid,
|
||||
SessionID: sid,
|
||||
System: plan.System,
|
||||
Query: plan.Query,
|
||||
Profile: o.fetchMemory(ctx, uid, plan.Query),
|
||||
History: o.fetchHistory(ctx, sid),
|
||||
Profile: profile,
|
||||
History: history,
|
||||
}, nil
|
||||
})); err != nil {
|
||||
return nil, err
|
||||
@@ -64,7 +69,7 @@ func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task) (compo
|
||||
}
|
||||
key := fmt.Sprintf("tool_%d", idx)
|
||||
idx++
|
||||
if err := g.AddLambdaNode(key, compose.InvokableLambda(o.makeToolNode(t.ID, tool, args))); err != nil {
|
||||
if err := g.AddLambdaNode(key, compose.InvokableLambda(o.makeToolNode(t.ID, tool, args, tr))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := g.AddEdge(prev, key); err != nil {
|
||||
@@ -75,7 +80,12 @@ func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task) (compo
|
||||
}
|
||||
|
||||
// prompt:黑板 → []*schema.Message(系统提示词 + 画像 + 工具产出 + 历史 + 用户输入)。
|
||||
if err := g.AddLambdaNode("prompt", compose.InvokableLambda(buildMessages)); err != nil {
|
||||
if err := g.AddLambdaNode("prompt", compose.InvokableLambda(
|
||||
func(ctx context.Context, rc *RunCtx) ([]*schema.Message, error) {
|
||||
msgs, err := buildMessages(ctx, rc)
|
||||
tr.info("prompt", "prompt", "组装提示词", fmt.Sprintf("%d 条消息 · 工具产出 %d 段", len(msgs), len(rc.ToolOut)))
|
||||
return msgs, err
|
||||
})); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := g.AddEdge(prev, "prompt"); err != nil {
|
||||
@@ -101,9 +111,11 @@ func (o *Orchestrator) compileFlow(ctx context.Context, t *contract.Task) (compo
|
||||
}
|
||||
|
||||
// makeToolNode 返回一个真实调用 MCP 工具的图节点:把结果增补进黑板,失败降级不阻断。
|
||||
func (o *Orchestrator) makeToolNode(taskID, tool string, args map[string]any) func(context.Context, *RunCtx) (*RunCtx, error) {
|
||||
func (o *Orchestrator) makeToolNode(taskID, tool string, args map[string]any, tr *execTracer) func(context.Context, *RunCtx) (*RunCtx, error) {
|
||||
node := "tool:" + tool
|
||||
return func(ctx context.Context, rc *RunCtx) (*RunCtx, error) {
|
||||
if o.tools == nil {
|
||||
tr.info(node, "tool", "工具 "+tool, "工具总线未接入,跳过")
|
||||
return rc, nil
|
||||
}
|
||||
// 未显式带查询词则注入当前用户输入,便于检索类工具。
|
||||
@@ -114,19 +126,34 @@ func (o *Orchestrator) makeToolNode(taskID, tool string, args map[string]any) fu
|
||||
if call["q"] == nil && call["query"] == nil {
|
||||
call["q"] = rc.Query
|
||||
}
|
||||
end := tr.span(node, "tool", "调用工具 "+tool)
|
||||
cctx, cancel := context.WithTimeout(ctx, toolCallTimeout)
|
||||
defer cancel()
|
||||
res, err := o.tools.CallTool(cctx, contract.ToolSubjectGo(tool), &contract.ToolCall{
|
||||
Tool: tool, TaskID: taskID, Args: call,
|
||||
})
|
||||
if err != nil || res == nil || !res.OK || res.Content == "" {
|
||||
if err != nil {
|
||||
end("调用失败,降级跳过", err)
|
||||
return rc, nil
|
||||
}
|
||||
if res == nil || !res.OK || res.Content == "" {
|
||||
end("无结果,降级跳过", nil)
|
||||
return rc, nil // 工具不可用/无结果 → 降级跳过
|
||||
}
|
||||
end("入参 "+previewArgs(call)+" → 产出 "+truncate(res.Content, 160), nil)
|
||||
rc.ToolOut = append(rc.ToolOut, "["+tool+"] "+res.Content)
|
||||
return rc, nil
|
||||
}
|
||||
}
|
||||
|
||||
// previewArgs 把工具入参压成一行短预览。
|
||||
func previewArgs(args map[string]any) string {
|
||||
if data, err := json.Marshal(args); err == nil {
|
||||
return truncate(string(data), 120)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildMessages 把黑板组装为发给模型的消息序列。
|
||||
func buildMessages(_ context.Context, rc *RunCtx) ([]*schema.Message, error) {
|
||||
var sys strings.Builder
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package eino
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/sundynix/sundynix-shared/contract"
|
||||
)
|
||||
|
||||
// ExecSink 是执行可视化事件的回流出口(由 NATS bus 实现):
|
||||
// 把节点/阶段生命周期事件发到 sundynix.exec.<id>,供"运行·观测"实时点亮轨迹。
|
||||
type ExecSink interface {
|
||||
PublishExec(taskID string, data []byte) error
|
||||
CompleteExec(taskID string) error
|
||||
}
|
||||
|
||||
// execTracer 为一个任务发结构化执行事件,自增 Seq 保序、span 自动计耗时。
|
||||
type execTracer struct {
|
||||
sink ExecSink
|
||||
task string
|
||||
seq int32
|
||||
}
|
||||
|
||||
// tracer 为某任务建一个事件发射器(sink 为空时所有方法变空操作)。
|
||||
func (o *Orchestrator) tracer(taskID string) *execTracer {
|
||||
return &execTracer{sink: o.exec, task: taskID}
|
||||
}
|
||||
|
||||
func (e *execTracer) emit(node, kind, phase, label, detail string, ms int64) {
|
||||
if e == nil || e.sink == nil {
|
||||
return
|
||||
}
|
||||
ev := contract.ExecEvent{
|
||||
Seq: int(atomic.AddInt32(&e.seq, 1)), TS: time.Now().UnixMilli(),
|
||||
Node: node, Kind: kind, Phase: phase, Label: label, Detail: detail, MS: ms,
|
||||
}
|
||||
if data, err := json.Marshal(&ev); err == nil {
|
||||
_ = e.sink.PublishExec(e.task, data)
|
||||
}
|
||||
}
|
||||
|
||||
// info 发一条瞬时事件(无耗时)。
|
||||
func (e *execTracer) info(node, kind, label, detail string) {
|
||||
e.emit(node, kind, "info", label, detail, 0)
|
||||
}
|
||||
|
||||
// span 发 start,并返回一个结束函数:调用时按 err 发 end / error,附带耗时。
|
||||
func (e *execTracer) span(node, kind, label string) func(detail string, err error) {
|
||||
e.emit(node, kind, "start", label, "", 0)
|
||||
t0 := time.Now()
|
||||
return func(detail string, err error) {
|
||||
ms := time.Since(t0).Milliseconds()
|
||||
if err != nil {
|
||||
e.emit(node, kind, "error", label, err.Error(), ms)
|
||||
return
|
||||
}
|
||||
e.emit(node, kind, "end", label, detail, ms)
|
||||
}
|
||||
}
|
||||
|
||||
// done 关闭该任务的执行事件流(让 SSE 客户端收尾)。
|
||||
func (e *execTracer) done() {
|
||||
if e == nil || e.sink == nil {
|
||||
return
|
||||
}
|
||||
_ = e.sink.CompleteExec(e.task)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
@@ -38,11 +39,13 @@ type Orchestrator struct {
|
||||
breaker *harness.CircuitBreaker
|
||||
sink TokenSink
|
||||
tools ToolCaller
|
||||
exec ExecSink
|
||||
}
|
||||
|
||||
// NewOrchestrator 持有依赖;图按任务的 DSL 在 Handle 内动态编译。
|
||||
func NewOrchestrator(pool *llm.Pool, breaker *harness.CircuitBreaker, sink TokenSink, tools ToolCaller) (*Orchestrator, error) {
|
||||
return &Orchestrator{pool: pool, breaker: breaker, sink: sink, tools: tools}, nil
|
||||
// exec 为执行可视化事件出口(可为 nil,则不发轨迹事件)。
|
||||
func NewOrchestrator(pool *llm.Pool, breaker *harness.CircuitBreaker, sink TokenSink, tools ToolCaller, exec ExecSink) (*Orchestrator, error) {
|
||||
return &Orchestrator{pool: pool, breaker: breaker, sink: sink, tools: tools, exec: exec}, nil
|
||||
}
|
||||
|
||||
// Handle 消费一个任务:按 DSL 编译 Eino 图并执行,把 Token 流回流到 sundynix.streams.<id>。
|
||||
@@ -51,22 +54,30 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
log.Printf("[eino] circuit open, drop task %s", t.ID)
|
||||
return nil
|
||||
}
|
||||
tr := o.tracer(t.ID)
|
||||
defer tr.done()
|
||||
|
||||
// 报告生成走专用多步编排(规划→分章并行检索撰写→汇聚→渲染 Word),而非通用对话图。
|
||||
if intent, _ := t.Meta[contract.MetaIntent].(string); intent == contract.IntentReport {
|
||||
return o.handleReport(ctx, t)
|
||||
return o.handleReport(ctx, t, tr)
|
||||
}
|
||||
log.Printf("[eino] task %s received (graph=%d bytes), compiling DSL → Eino graph...", t.ID, len(t.Graph))
|
||||
tr.info("task", "system", "任务受理", fmt.Sprintf("DSL %d 字节,编译 Eino 图", len(t.Graph)))
|
||||
|
||||
run, err := o.compileFlow(ctx, t)
|
||||
endCompile := tr.span("compile", "system", "编译 Eino 图")
|
||||
run, err := o.compileFlow(ctx, t, tr)
|
||||
if err != nil {
|
||||
endCompile("", err)
|
||||
log.Printf("[eino] task %s compile error: %v", t.ID, err)
|
||||
_ = o.sink.CompleteStream(t.ID)
|
||||
o.breaker.Report(false)
|
||||
return err
|
||||
}
|
||||
endCompile("图编译完成", nil)
|
||||
|
||||
stream, err := run.Stream(ctx, t)
|
||||
if err != nil {
|
||||
tr.emit("model", "model", "error", "模型推理", err.Error(), 0)
|
||||
log.Printf("[eino] task %s graph error: %v", t.ID, err)
|
||||
_ = o.sink.CompleteStream(t.ID)
|
||||
o.breaker.Report(false)
|
||||
@@ -76,6 +87,7 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
|
||||
n := 0
|
||||
var answer strings.Builder
|
||||
t0 := time.Now()
|
||||
for {
|
||||
chunk, rerr := stream.Recv()
|
||||
if errors.Is(rerr, io.EOF) {
|
||||
@@ -88,6 +100,9 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
if chunk == nil || chunk.Content == "" {
|
||||
continue
|
||||
}
|
||||
if n == 0 {
|
||||
tr.emit("model", "model", "start", "模型流式推理", fmt.Sprintf("首 token %dms", time.Since(t0).Milliseconds()), 0)
|
||||
}
|
||||
if perr := o.sink.PublishToken(t.ID, []byte(chunk.Content)); perr != nil {
|
||||
log.Printf("[eino] publish token failed: %v", perr)
|
||||
break
|
||||
@@ -95,6 +110,7 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
answer.WriteString(chunk.Content)
|
||||
n++
|
||||
}
|
||||
tr.emit("model", "model", "end", "模型流式推理", fmt.Sprintf("%d tokens / %d 字", n, len([]rune(answer.String()))), time.Since(t0).Milliseconds())
|
||||
|
||||
if cerr := o.sink.CompleteStream(t.ID); cerr != nil {
|
||||
log.Printf("[eino] complete stream failed: %v", cerr)
|
||||
|
||||
@@ -38,7 +38,7 @@ type reportSection struct {
|
||||
//
|
||||
// 全程把人可读的 Markdown 进度与正文经 sundynix.streams.<id> 流回客户端;
|
||||
// 最终调 mcp-go 的 report_render 落盘 docx,客户端凭 task_id 下载。
|
||||
func (o *Orchestrator) handleReport(ctx context.Context, t *contract.Task) error {
|
||||
func (o *Orchestrator) handleReport(ctx context.Context, t *contract.Task, tr *execTracer) error {
|
||||
defer func() { _ = o.sink.CompleteStream(t.ID) }()
|
||||
|
||||
topic, _ := t.Meta[contract.MetaTopic].(string)
|
||||
@@ -50,9 +50,12 @@ func (o *Orchestrator) handleReport(ctx context.Context, t *contract.Task) error
|
||||
topic = "未命名报告"
|
||||
}
|
||||
log.Printf("[report] task %s 生成报告: topic=%q kb=%q", t.ID, topic, kb)
|
||||
tr.info("task", "system", "报告任务受理", fmt.Sprintf("主题:%s%s", topic, kbSuffix(kb)))
|
||||
|
||||
o.emit(t.ID, "> 正在规划大纲…\n\n")
|
||||
endPlan := tr.span("plan", "plan", "规划大纲")
|
||||
outline := o.planOutline(ctx, topic)
|
||||
endPlan(fmt.Sprintf("%d 章:%s", len(outline.Sections), strings.Join(outline.Sections, " / ")), nil)
|
||||
|
||||
o.emit(t.ID, fmt.Sprintf("**报告大纲**(%d 章)\n", len(outline.Sections)))
|
||||
for i, s := range outline.Sections {
|
||||
@@ -64,7 +67,7 @@ func (o *Orchestrator) handleReport(ctx context.Context, t *contract.Task) error
|
||||
o.emit(t.ID, "\n> 正在并行撰写各章…\n\n")
|
||||
}
|
||||
|
||||
sections := o.writeSections(ctx, topic, kb, outline.Sections)
|
||||
sections := o.writeSections(ctx, topic, kb, outline.Sections, tr)
|
||||
|
||||
// 把完整报告正文流式呈现给客户端。
|
||||
o.emit(t.ID, "\n---\n\n# "+firstNonEmpty(outline.Title, topic)+"\n\n")
|
||||
@@ -74,16 +77,26 @@ func (o *Orchestrator) handleReport(ctx context.Context, t *contract.Task) error
|
||||
|
||||
// 渲染真实 Word 文档。
|
||||
o.emit(t.ID, "> 正在渲染 Word 文档…\n\n")
|
||||
endRender := tr.span("render", "render", "渲染 Word 文档")
|
||||
if path := o.renderReport(ctx, t.ID, firstNonEmpty(outline.Title, topic), sections); path != "" {
|
||||
endRender("docx 已落盘:"+path, nil)
|
||||
o.emit(t.ID, "---\n✅ 报告已生成 Word 文档,可点击上方「下载 Word」保存。\n")
|
||||
log.Printf("[report] task %s 完成,docx=%s", t.ID, path)
|
||||
} else {
|
||||
endRender("渲染服务不可用", fmt.Errorf("render unavailable"))
|
||||
o.emit(t.ID, "---\n⚠️ Word 渲染未完成(渲染服务不可用),以上为报告正文。\n")
|
||||
}
|
||||
o.breaker.Report(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func kbSuffix(kb string) string {
|
||||
if kb == "" {
|
||||
return "(不挂知识库)"
|
||||
}
|
||||
return ",知识库 " + kb
|
||||
}
|
||||
|
||||
// planOutline 让模型规划 3–5 章大纲;模型不可用/解析失败则用通用兜底大纲。
|
||||
func (o *Orchestrator) planOutline(ctx context.Context, topic string) reportOutline {
|
||||
fallback := reportOutline{Title: topic, Sections: []string{"背景与现状", "核心分析", "结论与建议"}}
|
||||
@@ -110,7 +123,7 @@ func (o *Orchestrator) planOutline(ctx context.Context, topic string) reportOutl
|
||||
}
|
||||
|
||||
// writeSections 各章节并行撰写(有界并发),结果按原顺序返回。
|
||||
func (o *Orchestrator) writeSections(ctx context.Context, topic, kb string, headings []string) []reportSection {
|
||||
func (o *Orchestrator) writeSections(ctx context.Context, topic, kb string, headings []string, tr *execTracer) []reportSection {
|
||||
out := make([]reportSection, len(headings))
|
||||
sem := make(chan struct{}, reportFanout)
|
||||
var wg sync.WaitGroup
|
||||
@@ -120,7 +133,11 @@ func (o *Orchestrator) writeSections(ctx context.Context, topic, kb string, head
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
out[i] = reportSection{Heading: h, Body: o.writeSection(ctx, topic, kb, h)}
|
||||
node := fmt.Sprintf("section:%d", i)
|
||||
end := tr.span(node, "section", fmt.Sprintf("第%d章 %s", i+1, h))
|
||||
body := o.writeSection(ctx, topic, kb, h, tr, node)
|
||||
end(fmt.Sprintf("成稿 %d 字", len([]rune(body))), nil)
|
||||
out[i] = reportSection{Heading: h, Body: body}
|
||||
}(i, h)
|
||||
}
|
||||
wg.Wait()
|
||||
@@ -128,8 +145,11 @@ func (o *Orchestrator) writeSections(ctx context.Context, topic, kb string, head
|
||||
}
|
||||
|
||||
// writeSection 撰写一章:先 RAG 检索参考资料(若挂了知识库),再让模型成稿。
|
||||
func (o *Orchestrator) writeSection(ctx context.Context, topic, kb, heading string) string {
|
||||
func (o *Orchestrator) writeSection(ctx context.Context, topic, kb, heading string, tr *execTracer, node string) string {
|
||||
refs := o.retrieve(ctx, kb, topic+" "+heading)
|
||||
if refs != "" {
|
||||
tr.info(node, "section", "检索参考资料", truncate(strings.ReplaceAll(refs, "\n", " "), 120))
|
||||
}
|
||||
if !o.pool.Ready() {
|
||||
if refs != "" {
|
||||
return "(模型未配置,以下为检索到的参考资料)\n" + refs
|
||||
|
||||
@@ -53,6 +53,16 @@ func (s *Subscriber) CompleteStream(taskID string) error {
|
||||
return s.inner.CompleteStream(taskID)
|
||||
}
|
||||
|
||||
// PublishExec / CompleteExec 让 Subscriber 满足 eino.ExecSink,
|
||||
// 把执行轨迹事件回流到 sundynix.exec.<taskID>(与 Token 流分开)。
|
||||
func (s *Subscriber) PublishExec(taskID string, data []byte) error {
|
||||
return s.inner.PublishExec(taskID, data)
|
||||
}
|
||||
|
||||
func (s *Subscriber) CompleteExec(taskID string) error {
|
||||
return s.inner.CompleteExec(taskID)
|
||||
}
|
||||
|
||||
// CallTool 让 Subscriber 满足 eino.ToolCaller,经 NATS request-reply 调起第 5 层 MCP 工具。
|
||||
func (s *Subscriber) CallTool(ctx context.Context, subject string, call *contract.ToolCall) (*contract.ToolResult, error) {
|
||||
return s.inner.CallTool(ctx, subject, call)
|
||||
|
||||
Reference in New Issue
Block a user