feat(observability): OpenTelemetry 全链路追踪（Jaeger + NATS 跨总线传播）
新增 sundynix-shared/otelx：otelx.Init(ctx,服务名) 注册 W3C 传播器 + OTLP/HTTP 批量导出（默认 localhost:4318，docker 里的 Jaeger）；OTEL_SDK_DISABLED=true 关导出。 Jaeger 不在线 / 导出器构建失败都不阻断启动（可观测性是增益而非依赖）。 - NATS 跨进程传播（无现成中间件）：bus/trace.go 的 natsHeaderCarrier + inject/extract， PublishTask/CallTool 注入 traceparent，ConsumeTasks/ServeTool 抽出续上 → 链路跨总线连成一棵树。 - 埋点：gateway 挂 otelgin（HTTP server span，链路根）；dispatcher task.execute → node.<kind>（每节点，nctx 下传使工具/LLM 挂到节点下）→ llm.stream/llm.generate； bus 自动出 tool.call(client)↔tool.serve(server) 成对跨服务 span。 - docker-compose 加 jaeger all-in-one（UI :16686，OTLP :4318）。 - 依赖修复：otlptracehttp 触发 genproto 单体(旧)vs 拆分模块 ambiguous import（milvus 拉旧版）， pin genproto 至后拆分版（go.work 工作区全局生效）。 - production_readiness.md 1.1 更新为「已实现」。 验证：真实 input→retriever→agent 任务在 Jaeger 出 14 span / 3 服务的完整树， 跨 NATS（publish→consume）、跨服务（tool.call→tool.serve）均连通， 瓶颈 kb_search 692ms、llm 1597ms 一眼可见；四模块 build+vet+test 全绿。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Blizzard
@@ -14,11 +14,17 @@ import (
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/harness"
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/llm"
|
||||
dnats "github.com/sundynix/sundynix-dispatcher/internal/nats"
|
||||
"github.com/sundynix/sundynix-shared/otelx"
|
||||
"github.com/sundynix/sundynix-shared/secrets"
|
||||
)
|
||||
|
||||
func main() {
|
||||
secrets.MustHaveKeyInProd() // 生产须配 SUNDYNIX_SECRET_KEY 以解密下发的 api_key 密文
|
||||
|
||||
// 链路追踪：消费 span 续上 gateway 的 trace，再向下展开节点/工具/LLM span。
|
||||
shutdownTrace, _ := otelx.Init(context.Background(), "sundynix-dispatcher")
|
||||
defer func() { _ = shutdownTrace(context.Background()) }()
|
||||
|
||||
natsURL := envOr("NATS_URL", "nats://localhost:4222")
|
||||
|
||||
pool := llm.NewPool() // LLM Pool: vLLM / Ollama 集群
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/sundynix/sundynix-dispatcher
|
||||
|
||||
go 1.25.0
|
||||
go 1.25.8
|
||||
|
||||
require (
|
||||
github.com/cloudwego/eino v0.9.9
|
||||
@@ -14,11 +14,14 @@ require (
|
||||
github.com/bytedance/gopkg v0.1.3 // indirect
|
||||
github.com/bytedance/sonic v1.15.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.5.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/eino-contrib/jsonschema v1.0.3 // indirect
|
||||
github.com/evanphx/json-patch v0.5.2 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/goph/emperror v0.17.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -42,6 +45,10 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/yargevad/filepathx v1.0.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
golang.org/x/arch v0.11.0 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
|
||||
|
||||
@@ -16,6 +16,8 @@ github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9V
|
||||
github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE=
|
||||
github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
|
||||
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cloudwego/eino v0.9.9 h1:x63hvRif6ANPh9YEPoTIrp1potEeoLQFAjOclKaX/Kg=
|
||||
@@ -38,6 +40,11 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -142,6 +149,14 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
|
||||
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
|
||||
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
|
||||
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
|
||||
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
|
||||
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4=
|
||||
|
||||
@@ -8,10 +8,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/dsl"
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/harness"
|
||||
"github.com/sundynix/sundynix-shared/contract"
|
||||
"github.com/sundynix/sundynix-shared/otelx"
|
||||
)
|
||||
|
||||
// defaultAgentSystem 是 agent 节点未填系统提示词时的兜底。
|
||||
@@ -99,6 +102,13 @@ func (o *Orchestrator) runGraph(ctx context.Context, t *contract.Task, tr *execT
|
||||
if !active[n.ID] {
|
||||
continue // 被 branch 剪掉的下游，不执行
|
||||
}
|
||||
// 每个节点一个 span：下游的工具调用 / LLM span 会自动挂到本节点之下，
|
||||
// Jaeger 里即呈现「节点 → 工具 → embedding/milvus」的层级。
|
||||
nctx, nspan := otelx.Tracer().Start(ctx, "node."+n.Kind,
|
||||
trace.WithAttributes(
|
||||
attribute.String("sundynix.node.id", n.ID),
|
||||
attribute.String("sundynix.node.kind", n.Kind),
|
||||
))
|
||||
propagate := targetsOf(outE[n.ID]) // 默认激活全部出边；branch 会改写
|
||||
switch n.Kind {
|
||||
case "input":
|
||||
@@ -108,39 +118,40 @@ func (o *Orchestrator) runGraph(ctx context.Context, t *contract.Task, tr *execT
|
||||
tr.info("input:"+n.ID, "system", labelOf(n, "输入"), truncate(b.query, 80))
|
||||
case "memory":
|
||||
if cbool(n.Config, "profile") {
|
||||
b.profile = o.fetchMemory(ctx, b.uid, b.query)
|
||||
b.profile = o.fetchMemory(nctx, b.uid, b.query)
|
||||
}
|
||||
if cbool(n.Config, "history") {
|
||||
b.history = o.fetchHistory(ctx, b.sid)
|
||||
b.history = o.fetchHistory(nctx, b.sid)
|
||||
}
|
||||
tr.info("memory:"+n.ID, "memory", labelOf(n, "记忆"),
|
||||
fmt.Sprintf("画像 %d 字 · 历史 %d 条", len([]rune(b.profile)), len(b.history)))
|
||||
case "retriever":
|
||||
o.retrieverNode(ctx, n, b, tr)
|
||||
o.retrieverNode(nctx, n, b, tr)
|
||||
case "tool":
|
||||
o.execToolNode(ctx, t.ID, n, b, tr)
|
||||
o.execToolNode(nctx, t.ID, n, b, tr)
|
||||
case "agent":
|
||||
sys := firstNonEmpty(cstr(n.Config, "system"), plan.System)
|
||||
if cbool(n.Config, "autonomous") { // 开启自主工具 → ReAct（模型自己选工具）
|
||||
o.runReactAgent(ctx, t.ID, b, sys, n, tr, "agent:"+n.ID)
|
||||
o.runReactAgent(nctx, t.ID, b, sys, n, tr, "agent:"+n.ID)
|
||||
} else {
|
||||
o.runConversation(ctx, t.ID, b, sys, tr, "agent:"+n.ID)
|
||||
o.runConversation(nctx, t.ID, b, sys, tr, "agent:"+n.ID)
|
||||
}
|
||||
case "aggregate":
|
||||
merged := aggregate(cstr(n.Config, "strategy"), append(append([]string{}, b.refs...), b.toolOut...))
|
||||
b.refs, b.toolOut = merged, nil
|
||||
tr.info("aggregate:"+n.ID, "system", labelOf(n, "汇聚"), "策略："+firstNonEmpty(cstr(n.Config, "strategy"), "拼接"))
|
||||
case "render":
|
||||
o.renderNode(ctx, t.ID, n, b, tr)
|
||||
o.renderNode(nctx, t.ID, n, b, tr)
|
||||
case "branch":
|
||||
propagate = o.branchNode(n, b, outE[n.ID], nodeByID, tr)
|
||||
case "map":
|
||||
o.mapNode(ctx, t.ID, n, b, tr)
|
||||
o.mapNode(nctx, t.ID, n, b, tr)
|
||||
case "output":
|
||||
tr.info("output:"+n.ID, "system", labelOf(n, "输出"), "目标："+firstNonEmpty(cstr(n.Config, "target"), "屏幕"))
|
||||
default:
|
||||
tr.info(n.Kind+":"+n.ID, "system", labelOf(n, n.Kind), "未识别节点，跳过")
|
||||
}
|
||||
nspan.End()
|
||||
for _, tgt := range propagate {
|
||||
active[tgt] = true
|
||||
}
|
||||
|
||||
@@ -12,11 +12,15 @@ import (
|
||||
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/dsl"
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/harness"
|
||||
"github.com/sundynix/sundynix-dispatcher/internal/llm"
|
||||
"github.com/sundynix/sundynix-shared/contract"
|
||||
"github.com/sundynix/sundynix-shared/otelx"
|
||||
)
|
||||
|
||||
// TokenSink 是 Token 流回流出口（由 NATS bus 实现）。
|
||||
@@ -103,6 +107,12 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
log.Printf("[eino] 跳过空任务（无 id）")
|
||||
return nil
|
||||
}
|
||||
|
||||
// 链路根（dispatcher 侧）：续上 gateway 经 NATS 传来的 trace，覆盖整个图执行。
|
||||
ctx, span := otelx.Tracer().Start(ctx, "task.execute",
|
||||
trace.WithAttributes(attribute.String("sundynix.task_id", t.ID)))
|
||||
defer span.End()
|
||||
|
||||
tr := o.tracer(t.ID)
|
||||
defer tr.done()
|
||||
|
||||
@@ -133,6 +143,8 @@ func (o *Orchestrator) Handle(ctx context.Context, t *contract.Task) error {
|
||||
// 按 DSL 图执行：compose.Graph（EINO_COMPOSE=1）或自研 graph.go（默认）；agent 节点流式回流 token。
|
||||
answer, err := o.executeGraph(tctx, t, tr)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
log.Printf("[eino] task %s graph error: %v", t.ID, err)
|
||||
_ = o.sink.CompleteStream(t.ID)
|
||||
o.breaker.Report(false)
|
||||
|
||||
@@ -13,8 +13,11 @@ import (
|
||||
"github.com/cloudwego/eino-ext/components/model/openai"
|
||||
"github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/sundynix/sundynix-shared/contract"
|
||||
"github.com/sundynix/sundynix-shared/otelx"
|
||||
)
|
||||
|
||||
// requestTimeout 是单次推理请求的上限。
|
||||
@@ -102,8 +105,15 @@ func (p *Pool) ChatStream(ctx context.Context, msgs []ChatMessage, onToken func(
|
||||
if cm == nil {
|
||||
return fmt.Errorf("no model configured")
|
||||
}
|
||||
ctx, span := otelx.Tracer().Start(ctx, "llm.stream",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llm.model", p.ModelName()),
|
||||
attribute.Int("llm.messages", len(msgs)),
|
||||
))
|
||||
defer span.End()
|
||||
sr, err := cm.Stream(ctx, toSchema(msgs))
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return fmt.Errorf("llm stream: %w", err)
|
||||
}
|
||||
defer sr.Close()
|
||||
@@ -128,8 +138,15 @@ func (p *Pool) Chat(ctx context.Context, msgs []ChatMessage) (string, error) {
|
||||
if cm == nil {
|
||||
return "", fmt.Errorf("no model configured")
|
||||
}
|
||||
ctx, span := otelx.Tracer().Start(ctx, "llm.generate",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llm.model", p.ModelName()),
|
||||
attribute.Int("llm.messages", len(msgs)),
|
||||
))
|
||||
defer span.End()
|
||||
out, err := cm.Generate(ctx, toSchema(msgs))
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
return "", fmt.Errorf("llm generate: %w", err)
|
||||
}
|
||||
return out.Content, nil
|
||||
|
||||
Reference in New Issue
Block a user