feat(observability): OpenTelemetry 全链路追踪(Jaeger + NATS 跨总线传播)

新增 sundynix-shared/otelx:otelx.Init(ctx,服务名) 注册 W3C 传播器 + OTLP/HTTP
批量导出(默认 localhost:4318,docker 里的 Jaeger);OTEL_SDK_DISABLED=true 关导出。
Jaeger 不在线 / 导出器构建失败都不阻断启动(可观测性是增益而非依赖)。

- NATS 跨进程传播(无现成中间件):bus/trace.go 的 natsHeaderCarrier + inject/extract,
  PublishTask/CallTool 注入 traceparent,ConsumeTasks/ServeTool 抽出续上 → 链路跨总线连成一棵树。
- 埋点:gateway 挂 otelgin(HTTP server span,链路根);dispatcher task.execute →
  node.<kind>(每节点,nctx 下传使工具/LLM 挂到节点下)→ llm.stream/llm.generate;
  bus 自动出 tool.call(client)↔tool.serve(server) 成对跨服务 span。
- docker-compose 加 jaeger all-in-one(UI :16686,OTLP :4318)。
- 依赖修复:otlptracehttp 触发 genproto 单体(旧)vs 拆分模块 ambiguous import(milvus 拉旧版),
  pin genproto 至后拆分版(go.work 工作区全局生效)。
- production_readiness.md 1.1 更新为「已实现」。

验证:真实 input→retriever→agent 任务在 Jaeger 出 14 span / 3 服务的完整树,
跨 NATS(publish→consume)、跨服务(tool.call→tool.serve)均连通,
瓶颈 kb_search 692ms、llm 1597ms 一眼可见;四模块 build+vet+test 全绿。

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Blizzard
2026-06-24 11:35:08 +08:00
parent 46ef3df221
commit a2d184b7ec
22 changed files with 675 additions and 67 deletions
+19 -8
View File
@@ -8,10 +8,13 @@ import (
"time"
"github.com/cloudwego/eino/schema"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/sundynix/sundynix-dispatcher/internal/dsl"
"github.com/sundynix/sundynix-dispatcher/internal/harness"
"github.com/sundynix/sundynix-shared/contract"
"github.com/sundynix/sundynix-shared/otelx"
)
// defaultAgentSystem 是 agent 节点未填系统提示词时的兜底。
@@ -99,6 +102,13 @@ func (o *Orchestrator) runGraph(ctx context.Context, t *contract.Task, tr *execT
if !active[n.ID] {
continue // 被 branch 剪掉的下游,不执行
}
// 每个节点一个 span:下游的工具调用 / LLM span 会自动挂到本节点之下,
// Jaeger 里即呈现「节点 → 工具 → embedding/milvus」的层级。
nctx, nspan := otelx.Tracer().Start(ctx, "node."+n.Kind,
trace.WithAttributes(
attribute.String("sundynix.node.id", n.ID),
attribute.String("sundynix.node.kind", n.Kind),
))
propagate := targetsOf(outE[n.ID]) // 默认激活全部出边;branch 会改写
switch n.Kind {
case "input":
@@ -108,39 +118,40 @@ func (o *Orchestrator) runGraph(ctx context.Context, t *contract.Task, tr *execT
tr.info("input:"+n.ID, "system", labelOf(n, "输入"), truncate(b.query, 80))
case "memory":
if cbool(n.Config, "profile") {
b.profile = o.fetchMemory(ctx, b.uid, b.query)
b.profile = o.fetchMemory(nctx, b.uid, b.query)
}
if cbool(n.Config, "history") {
b.history = o.fetchHistory(ctx, b.sid)
b.history = o.fetchHistory(nctx, b.sid)
}
tr.info("memory:"+n.ID, "memory", labelOf(n, "记忆"),
fmt.Sprintf("画像 %d 字 · 历史 %d 条", len([]rune(b.profile)), len(b.history)))
case "retriever":
o.retrieverNode(ctx, n, b, tr)
o.retrieverNode(nctx, n, b, tr)
case "tool":
o.execToolNode(ctx, t.ID, n, b, tr)
o.execToolNode(nctx, t.ID, n, b, tr)
case "agent":
sys := firstNonEmpty(cstr(n.Config, "system"), plan.System)
if cbool(n.Config, "autonomous") { // 开启自主工具 → ReAct(模型自己选工具)
o.runReactAgent(ctx, t.ID, b, sys, n, tr, "agent:"+n.ID)
o.runReactAgent(nctx, t.ID, b, sys, n, tr, "agent:"+n.ID)
} else {
o.runConversation(ctx, t.ID, b, sys, tr, "agent:"+n.ID)
o.runConversation(nctx, t.ID, b, sys, tr, "agent:"+n.ID)
}
case "aggregate":
merged := aggregate(cstr(n.Config, "strategy"), append(append([]string{}, b.refs...), b.toolOut...))
b.refs, b.toolOut = merged, nil
tr.info("aggregate:"+n.ID, "system", labelOf(n, "汇聚"), "策略:"+firstNonEmpty(cstr(n.Config, "strategy"), "拼接"))
case "render":
o.renderNode(ctx, t.ID, n, b, tr)
o.renderNode(nctx, t.ID, n, b, tr)
case "branch":
propagate = o.branchNode(n, b, outE[n.ID], nodeByID, tr)
case "map":
o.mapNode(ctx, t.ID, n, b, tr)
o.mapNode(nctx, t.ID, n, b, tr)
case "output":
tr.info("output:"+n.ID, "system", labelOf(n, "输出"), "目标:"+firstNonEmpty(cstr(n.Config, "target"), "屏幕"))
default:
tr.info(n.Kind+":"+n.ID, "system", labelOf(n, n.Kind), "未识别节点,跳过")
}
nspan.End()
for _, tgt := range propagate {
active[tgt] = true
}
@@ -12,11 +12,15 @@ import (
"github.com/cloudwego/eino/components/model"
"github.com/cloudwego/eino/schema"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/sundynix/sundynix-dispatcher/internal/dsl"
"github.com/sundynix/sundynix-dispatcher/internal/harness"
"github.com/sundynix/sundynix-dispatcher/internal/llm"
"github.com/sundynix/sundynix-shared/contract"
"github.com/sundynix/sundynix-shared/otelx"
)
// TokenSink 是 Token 流回流出口(由 NATS bus 实现)。
@@ -103,6 +107,12 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
log.Printf("[eino] 跳过空任务(无 id")
return nil
}
// 链路根(dispatcher 侧):续上 gateway 经 NATS 传来的 trace,覆盖整个图执行。
ctx, span := otelx.Tracer().Start(ctx, "task.execute",
trace.WithAttributes(attribute.String("sundynix.task_id", t.ID)))
defer span.End()
tr := o.tracer(t.ID)
defer tr.done()
@@ -133,6 +143,8 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
// 按 DSL 图执行:compose.GraphEINO_COMPOSE=1)或自研 graph.go(默认);agent 节点流式回流 token。
answer, err := o.executeGraph(tctx, t, tr)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
log.Printf("[eino] task %s graph error: %v", t.ID, err)
_ = o.sink.CompleteStream(t.ID)
o.breaker.Report(false)