55d50417a9
failover/熔断的运行时态原来只在 dispatcher 日志、admin 看不到 —— 本次接到概览可见:
- harness: CircuitBreaker.Snapshot() 只读观测访问器(state + fails,不动状态机)
- llm: Pool.ModelHealth() 上报主备链每模型 {provider,model,role,state,fails};
buildWithFallbacks 把模型名↔breaker 配对(同包直接读 failoverModel.breakers);
newFailoverModel 改返回具体类型以便读 breakers
- dispatcher 心跳 payload 加 models[]
- gateway /admin/overview 独立超时 Ping dispatcher,合并进 models.health
- admin 概览「模型路由」新增「运行时链路态(实时)」:逐模型状态点
(🟢在线/🔴熔断中+失败数/🟡半开探测/单点)
- 单测:Snapshot、Pool.ModelHealth(名字↔态配对/单点/空)
- live:配坏主→提交任务打熔断→概览显示 broken-demo「熔断中·失败3」,备用在线
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
131 lines
3.4 KiB
Go
131 lines
3.4 KiB
Go
package harness
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// newTestCB 造一个可控时钟、低阈值的熔断器,便于确定性测试。
|
|
func newTestCB(threshold int, cooldown time.Duration, clock *time.Time) *CircuitBreaker {
|
|
c := NewCircuitBreaker()
|
|
c.threshold = threshold
|
|
c.cooldown = cooldown
|
|
c.now = func() time.Time { return *clock }
|
|
return c
|
|
}
|
|
|
|
// Snapshot 应如实反映当前状态与连续失败计数(管理端展示用,不改状态机)。
|
|
func TestCircuitBreaker_Snapshot(t *testing.T) {
|
|
now := time.Unix(0, 0)
|
|
c := newTestCB(3, 10*time.Second, &now)
|
|
if s := c.Snapshot(); s.State != Closed || s.Fails != 0 {
|
|
t.Fatalf("初始应 closed/0, got %+v", s)
|
|
}
|
|
c.Report(false)
|
|
c.Report(false)
|
|
if s := c.Snapshot(); s.State != Closed || s.Fails != 2 {
|
|
t.Fatalf("2 次失败未到阈值应 closed/2, got %+v", s)
|
|
}
|
|
c.Report(false) // 第 3 次 → 熔断
|
|
if s := c.Snapshot(); s.State != Open || s.Fails != 3 {
|
|
t.Fatalf("阈值后应 open/3, got %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_OpensAfterThreshold(t *testing.T) {
|
|
now := time.Unix(0, 0)
|
|
c := newTestCB(3, 10*time.Second, &now)
|
|
|
|
if !c.Allow() || c.State() != Closed {
|
|
t.Fatal("初始应闭合放行")
|
|
}
|
|
c.Report(false)
|
|
c.Report(false)
|
|
if c.State() != Closed {
|
|
t.Fatal("未达阈值不应断开")
|
|
}
|
|
c.Report(false) // 第 3 次连续失败 → 断开
|
|
if c.State() != Open {
|
|
t.Fatalf("达阈值应断开, got %v", c.State())
|
|
}
|
|
if c.Allow() {
|
|
t.Error("断开态应拒绝放行")
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_SuccessResetsFails(t *testing.T) {
|
|
now := time.Unix(0, 0)
|
|
c := newTestCB(3, 10*time.Second, &now)
|
|
c.Report(false)
|
|
c.Report(false)
|
|
c.Report(true) // 成功清零连续失败
|
|
c.Report(false)
|
|
c.Report(false)
|
|
if c.State() != Closed {
|
|
t.Errorf("成功应清零计数,不应断开, got %v", c.State())
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_HalfOpenRecovers(t *testing.T) {
|
|
now := time.Unix(0, 0)
|
|
c := newTestCB(2, 10*time.Second, &now)
|
|
c.Report(false)
|
|
c.Report(false) // 断开
|
|
if c.State() != Open || c.Allow() {
|
|
t.Fatal("应断开并拒绝")
|
|
}
|
|
// 冷却未到 → 仍拒绝。
|
|
now = now.Add(5 * time.Second)
|
|
if c.Allow() {
|
|
t.Fatal("冷却未到不应放行")
|
|
}
|
|
// 冷却到点 → 半开放行一个探测,第二个被拒。
|
|
now = now.Add(6 * time.Second)
|
|
if !c.Allow() || c.State() != HalfOpen {
|
|
t.Fatalf("冷却到点应转半开并放行探测, state=%v", c.State())
|
|
}
|
|
if c.Allow() {
|
|
t.Error("半开态超过探测名额应拒绝")
|
|
}
|
|
// 探测成功 → 恢复闭合。
|
|
c.Report(true)
|
|
if c.State() != Closed || !c.Allow() {
|
|
t.Errorf("探测成功应恢复闭合, state=%v", c.State())
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_HalfOpenProbeFailReopens(t *testing.T) {
|
|
now := time.Unix(0, 0)
|
|
c := newTestCB(1, 10*time.Second, &now)
|
|
c.Report(false) // 阈值 1 → 立即断开
|
|
now = now.Add(11 * time.Second)
|
|
if !c.Allow() || c.State() != HalfOpen {
|
|
t.Fatal("应转半开")
|
|
}
|
|
c.Report(false) // 探测失败 → 重新断开,冷却从此刻重算
|
|
if c.State() != Open {
|
|
t.Fatalf("探测失败应重新断开, got %v", c.State())
|
|
}
|
|
if c.Allow() {
|
|
t.Error("重新断开后冷却内应拒绝")
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_ConcurrentSafe(t *testing.T) {
|
|
now := time.Now()
|
|
c := newTestCB(1000000, time.Second, &now)
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 50; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := 0; j < 200; j++ {
|
|
c.Allow()
|
|
c.Report(j%2 == 0)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait() // 仅验证无数据竞争 / 死锁(配合 -race)
|
|
}
|