feat(observability): OpenTelemetry 全链路追踪(Jaeger + NATS 跨总线传播)
新增 sundynix-shared/otelx:otelx.Init(ctx,服务名) 注册 W3C 传播器 + OTLP/HTTP 批量导出(默认 localhost:4318,docker 里的 Jaeger);OTEL_SDK_DISABLED=true 关导出。 Jaeger 不在线 / 导出器构建失败都不阻断启动(可观测性是增益而非依赖)。 - NATS 跨进程传播(无现成中间件):bus/trace.go 的 natsHeaderCarrier + inject/extract, PublishTask/CallTool 注入 traceparent,ConsumeTasks/ServeTool 抽出续上 → 链路跨总线连成一棵树。 - 埋点:gateway 挂 otelgin(HTTP server span,链路根);dispatcher task.execute → node.<kind>(每节点,nctx 下传使工具/LLM 挂到节点下)→ llm.stream/llm.generate; bus 自动出 tool.call(client)↔tool.serve(server) 成对跨服务 span。 - docker-compose 加 jaeger all-in-one(UI :16686,OTLP :4318)。 - 依赖修复:otlptracehttp 触发 genproto 单体(旧)vs 拆分模块 ambiguous import(milvus 拉旧版), pin genproto 至后拆分版(go.work 工作区全局生效)。 - production_readiness.md 1.1 更新为「已实现」。 验证:真实 input→retriever→agent 任务在 Jaeger 出 14 span / 3 服务的完整树, 跨 NATS(publish→consume)、跨服务(tool.call→tool.serve)均连通, 瓶颈 kb_search 692ms、llm 1597ms 一眼可见;四模块 build+vet+test 全绿。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/sundynix/sundynix-shared/contract"
|
||||
"github.com/sundynix/sundynix-shared/secrets"
|
||||
@@ -100,13 +102,27 @@ func (b *Bus) EnsureTaskStream(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// PublishTask 把任务发布到 sundynix.tasks.<id>,返回序列号。
|
||||
// 发布前把链路上下文注入消息头,使 dispatcher 消费时能续上同一条 trace。
|
||||
func (b *Bus) PublishTask(ctx context.Context, t *contract.Task) (uint64, error) {
|
||||
data, err := t.Marshal()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ack, err := b.js.Publish(ctx, contract.TaskSubject(t.ID), data)
|
||||
ctx, span := tracer().Start(ctx, "nats.publish task",
|
||||
trace.WithSpanKind(trace.SpanKindProducer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("messaging.system", "nats"),
|
||||
attribute.String("messaging.destination.name", contract.TaskSubject(t.ID)),
|
||||
attribute.String("sundynix.task_id", t.ID),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
msg := nats.NewMsg(contract.TaskSubject(t.ID))
|
||||
msg.Data = data
|
||||
injectTrace(ctx, msg.Header) // 把 traceparent 写进消息头 → 跨总线传播
|
||||
ack, err := b.js.PublishMsg(ctx, msg)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return 0, fmt.Errorf("publish task: %w", err)
|
||||
}
|
||||
return ack.Sequence, nil
|
||||
@@ -186,14 +202,28 @@ func (b *Bus) CallTool(ctx context.Context, subject string, call *contract.ToolC
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal tool call: %w", err)
|
||||
}
|
||||
msg, err := b.nc.RequestWithContext(ctx, subject, data)
|
||||
ctx, span := tracer().Start(ctx, "tool.call "+call.Tool,
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(
|
||||
attribute.String("sundynix.tool", call.Tool),
|
||||
attribute.String("messaging.destination.name", subject),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
req := nats.NewMsg(subject)
|
||||
req.Data = data
|
||||
injectTrace(ctx, req.Header) // 把链路上下文带给第 5 层工具服务
|
||||
msg, err := b.nc.RequestMsgWithContext(ctx, req)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return nil, fmt.Errorf("call tool %s: %w", subject, err)
|
||||
}
|
||||
var res contract.ToolResult
|
||||
if err := json.Unmarshal(msg.Data, &res); err != nil {
|
||||
span.RecordError(err)
|
||||
return nil, fmt.Errorf("unmarshal tool result: %w", err)
|
||||
}
|
||||
span.SetAttributes(attribute.Bool("sundynix.tool.ok", res.OK))
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
@@ -210,7 +240,17 @@ func (b *Bus) ServeTool(subject, queue string, h ToolHandler) (unsub func() erro
|
||||
respond(m, &contract.ToolResult{OK: false, Error: "bad tool call: " + err.Error()})
|
||||
return
|
||||
}
|
||||
respond(m, h(context.Background(), &call))
|
||||
// 还原上游链路并开服务端 span(成为 dispatcher tool.call span 的子节点)。
|
||||
mctx := extractTrace(context.Background(), m.Header)
|
||||
mctx, span := tracer().Start(mctx, "tool.serve "+call.Tool,
|
||||
trace.WithSpanKind(trace.SpanKindServer),
|
||||
trace.WithAttributes(attribute.String("sundynix.tool", call.Tool)))
|
||||
res := h(mctx, &call)
|
||||
if res != nil {
|
||||
span.SetAttributes(attribute.Bool("sundynix.tool.ok", res.OK))
|
||||
}
|
||||
span.End()
|
||||
respond(m, res)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serve tool %s: %w", subject, err)
|
||||
@@ -376,7 +416,17 @@ func (b *Bus) ConsumeTasks(ctx context.Context, h TaskHandler) (stop func(), err
|
||||
_ = msg.Term() // 脏数据,丢弃不重投
|
||||
return
|
||||
}
|
||||
if err := h(ctx, t); err != nil {
|
||||
// 从消息头还原上游链路,开消费 span(成为 gateway 发布 span 的子节点)。
|
||||
mctx := extractTrace(ctx, nats.Header(msg.Headers()))
|
||||
mctx, span := tracer().Start(mctx, "nats.consume task",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(attribute.String("sundynix.task_id", t.ID)))
|
||||
herr := h(mctx, t)
|
||||
if herr != nil {
|
||||
span.RecordError(herr)
|
||||
}
|
||||
span.End()
|
||||
if herr != nil {
|
||||
_ = msg.NakWithDelay(time.Second) // 处理失败,延迟重投
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user