Blizzard
e3b0a4f81c
bus.ConsumeTasks 由单条串行改为限并发分发：每个任务进独立 worker goroutine，
并发上限 = 信号量 + 消费者 MaxAckPending（DISPATCHER_CONCURRENCY，默认 8）。
- 背压：并发满则在 select{sem, ctx.Done} 处等空位；关停时留消息不 ack 待重投。
- 健壮：worker 内 recover panic → Term（避免崩溃循环）；ack/nak/span 收口在 worker。
- 并发安全已核：CircuitBreaker 有锁、Orchestrator.turns 有 turnMu、pool RWMutex、
evaluate 本就异步。
效果：一个 HITL 待审任务（Handle 阻塞至多 5min）或长 LLM 生成不再冻结后续任务。
验证：单测 TestConcurrentConsume（A 阻塞时 B 完成）；live 实测 HITL 停 waiting 期间
普通任务 3s 跑完且 HITL 不受影响。全模块 build+vet+test 全绿。
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
257 lines
7.0 KiB
Go
257 lines
7.0 KiB
Go
package bus_test
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"testing"
|
||
"time"
|
||
|
||
natsserver "github.com/nats-io/nats-server/v2/server"
|
||
natstest "github.com/nats-io/nats-server/v2/test"
|
||
|
||
"github.com/sundynix/sundynix-shared/bus"
|
||
"github.com/sundynix/sundynix-shared/contract"
|
||
)
|
||
|
||
// startEmbeddedNATS 启动一个内嵌、开启 JetStream 的 NATS 服务器，免 Docker。
|
||
func startEmbeddedNATS(t *testing.T) string {
|
||
t.Helper()
|
||
opts := natstest.DefaultTestOptions
|
||
opts.Port = -1 // 随机端口
|
||
opts.JetStream = true
|
||
opts.StoreDir = t.TempDir()
|
||
srv := natstest.RunServer(&opts)
|
||
if !srv.ReadyForConnections(5 * time.Second) {
|
||
t.Fatal("embedded NATS not ready")
|
||
}
|
||
t.Cleanup(srv.Shutdown)
|
||
_ = natsserver.Server{} // 触发包引用
|
||
return srv.ClientURL()
|
||
}
|
||
|
||
// TestTaskRoundTrip 模拟 Gateway 发布 → NATS → Dispatcher 消费 的完整任务流。
|
||
func TestTaskRoundTrip(t *testing.T) {
|
||
url := startEmbeddedNATS(t)
|
||
|
||
// --- Gateway 侧：连接并声明任务流 ---
|
||
gw, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("gateway connect: %v", err)
|
||
}
|
||
defer gw.Close()
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer cancel()
|
||
|
||
if err := gw.EnsureTaskStream(ctx); err != nil {
|
||
t.Fatalf("ensure stream: %v", err)
|
||
}
|
||
|
||
// --- Dispatcher 侧：连接并开始消费 ---
|
||
dp, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("dispatcher connect: %v", err)
|
||
}
|
||
defer dp.Close()
|
||
|
||
got := make(chan *contract.Task, 1)
|
||
stop, err := dp.ConsumeTasks(ctx, func(_ context.Context, task *contract.Task) error {
|
||
got <- task
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("consume: %v", err)
|
||
}
|
||
defer stop()
|
||
|
||
// --- Gateway 发布一个任务 ---
|
||
want := &contract.Task{
|
||
ID: "task_demo_001",
|
||
Graph: json.RawMessage(`{"nodes":[{"id":"n1","type":"agent"}],"edges":[]}`),
|
||
Meta: map[string]any{"user": "wt"},
|
||
}
|
||
seq, err := gw.PublishTask(ctx, want)
|
||
if err != nil {
|
||
t.Fatalf("publish: %v", err)
|
||
}
|
||
if seq == 0 {
|
||
t.Fatal("expected non-zero stream sequence")
|
||
}
|
||
|
||
// --- 断言 Dispatcher 收到同一个任务 ---
|
||
select {
|
||
case task := <-got:
|
||
if task.ID != want.ID {
|
||
t.Fatalf("task id = %q, want %q", task.ID, want.ID)
|
||
}
|
||
if task.Meta["user"] != "wt" {
|
||
t.Fatalf("task meta lost: %+v", task.Meta)
|
||
}
|
||
t.Logf("✓ 任务流打通：Gateway publish (seq=%d) → NATS → Dispatcher consume，task_id=%s", seq, task.ID)
|
||
case <-time.After(5 * time.Second):
|
||
t.Fatal("timeout: dispatcher 未收到任务")
|
||
}
|
||
}
|
||
|
||
// TestToolCallRoundTrip 模拟 Dispatcher 经 NATS 调用 → mcp-go 响应 的工具调用闭环。
|
||
func TestToolCallRoundTrip(t *testing.T) {
|
||
url := startEmbeddedNATS(t)
|
||
|
||
// --- mcp-go 侧：以队列组订阅工具主题并响应 ---
|
||
srv, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("mcp connect: %v", err)
|
||
}
|
||
defer srv.Close()
|
||
|
||
unsub, err := srv.ServeTool(contract.SubjectToolsGoAll, contract.QueueToolsGo,
|
||
func(_ context.Context, call *contract.ToolCall) *contract.ToolResult {
|
||
if call.Tool != "wiki_search" {
|
||
return &contract.ToolResult{OK: false, Error: "unknown tool"}
|
||
}
|
||
return &contract.ToolResult{OK: true, Content: "命中:" + call.Args["q"].(string)}
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("serve tool: %v", err)
|
||
}
|
||
defer func() { _ = unsub() }()
|
||
|
||
// --- Dispatcher 侧：同步调用工具 ---
|
||
dp, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("dispatcher connect: %v", err)
|
||
}
|
||
defer dp.Close()
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||
defer cancel()
|
||
|
||
res, err := dp.CallTool(ctx, contract.ToolSubjectGo("wiki_search"), &contract.ToolCall{
|
||
Tool: "wiki_search",
|
||
TaskID: "task_tool_001",
|
||
Args: map[string]any{"q": "向量检索"},
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("call tool: %v", err)
|
||
}
|
||
if !res.OK || res.Content != "命中:向量检索" {
|
||
t.Fatalf("tool result = %+v, want ok content=命中:向量检索", res)
|
||
}
|
||
t.Logf("✓ 工具调用闭环：Dispatcher → sundynix.tools.go.wiki_search → mcp-go → %q", res.Content)
|
||
}
|
||
|
||
// TestTokenStreamRoundTrip 模拟 Dispatcher 回流 Token → Gateway 订阅 的流式闭环。
|
||
func TestTokenStreamRoundTrip(t *testing.T) {
|
||
url := startEmbeddedNATS(t)
|
||
|
||
// Gateway 侧：先订阅（core NATS 无持久化，须先连）。
|
||
gw, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("gateway connect: %v", err)
|
||
}
|
||
defer gw.Close()
|
||
|
||
const taskID = "task_stream_001"
|
||
var got []string
|
||
done := make(chan struct{})
|
||
unsub, err := gw.SubscribeTokens(taskID,
|
||
func(tok []byte) { got = append(got, string(tok)) },
|
||
func() { close(done) },
|
||
)
|
||
if err != nil {
|
||
t.Fatalf("subscribe tokens: %v", err)
|
||
}
|
||
defer func() { _ = unsub() }()
|
||
|
||
// Dispatcher 侧：逐 Token 回流后发结束信号。
|
||
dp, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("dispatcher connect: %v", err)
|
||
}
|
||
defer dp.Close()
|
||
|
||
want := []string{"Hello", " ", "Agent", "!"}
|
||
for _, tok := range want {
|
||
if err := dp.PublishToken(taskID, []byte(tok)); err != nil {
|
||
t.Fatalf("publish token: %v", err)
|
||
}
|
||
}
|
||
if err := dp.CompleteStream(taskID); err != nil {
|
||
t.Fatalf("complete stream: %v", err)
|
||
}
|
||
|
||
select {
|
||
case <-done:
|
||
joined := ""
|
||
for _, s := range got {
|
||
joined += s
|
||
}
|
||
if joined != "Hello Agent!" {
|
||
t.Fatalf("token stream = %q, want %q", joined, "Hello Agent!")
|
||
}
|
||
t.Logf("✓ Token 流闭环：Dispatcher 回流 %d 个 token → Gateway 拼回 %q", len(got), joined)
|
||
case <-time.After(5 * time.Second):
|
||
t.Fatal("timeout: 未收到流结束信号")
|
||
}
|
||
}
|
||
|
||
// TestConcurrentConsume 验证并发消费：一个长时间阻塞的任务（模拟 HITL 待审）
|
||
// 不应阻塞后续任务——旧的串行消费下本测试会超时失败。
|
||
func TestConcurrentConsume(t *testing.T) {
|
||
url := startEmbeddedNATS(t)
|
||
|
||
gw, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("gateway connect: %v", err)
|
||
}
|
||
defer gw.Close()
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cancel()
|
||
if err := gw.EnsureTaskStream(ctx); err != nil {
|
||
t.Fatalf("ensure stream: %v", err)
|
||
}
|
||
|
||
dp, err := bus.Connect(url)
|
||
if err != nil {
|
||
t.Fatalf("dispatcher connect: %v", err)
|
||
}
|
||
defer dp.Close()
|
||
|
||
blockA := make(chan struct{})
|
||
doneB := make(chan string, 1)
|
||
stop, err := dp.ConsumeTasks(ctx, func(_ context.Context, task *contract.Task) error {
|
||
switch task.ID {
|
||
case "A":
|
||
<-blockA // 模拟审批长阻塞
|
||
case "B":
|
||
doneB <- task.ID
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
t.Fatalf("consume: %v", err)
|
||
}
|
||
defer stop()
|
||
defer close(blockA)
|
||
|
||
// 先发 A 并等它被消费、卡在 handler 里；再发 B。
|
||
if _, err := gw.PublishTask(ctx, &contract.Task{ID: "A", Graph: json.RawMessage(`{}`)}); err != nil {
|
||
t.Fatalf("publish A: %v", err)
|
||
}
|
||
time.Sleep(400 * time.Millisecond)
|
||
if _, err := gw.PublishTask(ctx, &contract.Task{ID: "B", Graph: json.RawMessage(`{}`)}); err != nil {
|
||
t.Fatalf("publish B: %v", err)
|
||
}
|
||
|
||
select {
|
||
case id := <-doneB:
|
||
if id != "B" {
|
||
t.Fatalf("unexpected task done: %q", id)
|
||
}
|
||
t.Log("✓ 并发消费生效：A 仍阻塞时 B 已完成")
|
||
case <-time.After(4 * time.Second):
|
||
t.Fatal("B 被阻塞的 A 卡住——并发消费未生效（仍是串行）")
|
||
}
|
||
}
|