Blizzard
93e9d3b195
评测闭环延伸出自愈：当自动评测判定输出为 poor(综合<0.5)，dispatcher 在热路径外 自动用「原问题+初版回答+评审短板(flags/评语)」(有来源则连来源一并喂回、要求严格基于来源) 让模型重写，重评后仅当新分严格更高才采纳(绝不退步)；采纳的修订版落会话历史， 保证多轮上下文用的是好答案而非被判低分的初版。评测终值带 corrected 标记经 NATS→网关落库。 - maxRefineRounds=1：poor 稀少，1 轮重写+重评够用，防成本失控 - canRefine 门控：模型就绪且熔断未开才纠偏，避免后端抖时雪上加霜 - 单 goroutine 串 评测→纠偏→落历史，杜绝原两 goroutine 对答案版本的竞态 - 契约 EvalEvent / Eval 表 / upsert / GET /tasks/:id/eval 均加 corrected 字段 - refine_test.go：采纳更优 / 不退步 / 非低分不触发 三测；live 验证好答案不误触发 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
108 lines
3.7 KiB
Go
108 lines
3.7 KiB
Go
package eino
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"testing"
|
||
|
||
"github.com/sundynix/sundynix-dispatcher/internal/harness"
|
||
"github.com/sundynix/sundynix-dispatcher/internal/llm"
|
||
"github.com/sundynix/sundynix-shared/contract"
|
||
)
|
||
|
||
// fakeEvalSink 捕获回流的评测事件，供断言纠偏终值。
|
||
type fakeEvalSink struct{ ev *contract.EvalEvent }
|
||
|
||
func (f *fakeEvalSink) PublishEval(ev *contract.EvalEvent) error { f.ev = ev; return nil }
|
||
|
||
// orchForEval 组一个仅评测/纠偏所需依赖的编排器（生成器 gen + 评审 judge + 评测出口 sink）。
|
||
func orchForEval(gen *fakeLLM, judge func(ctx context.Context, sys, user string) (string, error), sink *fakeEvalSink) *Orchestrator {
|
||
return &Orchestrator{
|
||
pool: gen,
|
||
breaker: harness.NewCircuitBreaker(),
|
||
eval: harness.NewEvaluator(func() bool { return true }, judge),
|
||
exec: &fakeExec{},
|
||
evalSink: sink,
|
||
}
|
||
}
|
||
|
||
// 低分 → 自动纠偏重生成 → 新答更优 → 采纳修订并标记 Corrected，落库终值升为 ok。
|
||
func TestEvaluate_AutoRefine_AdoptsBetter(t *testing.T) {
|
||
// 评审：含 refusal 标记的初版给 1 分，重写后的正常回答给 5 分。
|
||
judge := func(_ context.Context, _, user string) (string, error) {
|
||
if strings.Contains(user, "error:") {
|
||
return `{"score":1,"reason":"差"}`, nil
|
||
}
|
||
return `{"score":5,"reason":"好"}`, nil
|
||
}
|
||
gen := &fakeLLM{ready: true, chat: func(_ []llm.ChatMessage) (string, error) {
|
||
return "光合作用是植物利用光能把二氧化碳和水合成有机物并释放氧气的过程。", nil
|
||
}}
|
||
sink := &fakeEvalSink{}
|
||
o := orchForEval(gen, judge, sink)
|
||
|
||
final := o.evaluate(&contract.Task{ID: "t1"}, "介绍光合作用", "error: 我无法回答", nil)
|
||
|
||
if !strings.Contains(final, "光合作用") {
|
||
t.Fatalf("应采纳纠偏后的好答案，got %q", final)
|
||
}
|
||
if sink.ev == nil || !sink.ev.Corrected {
|
||
t.Fatalf("应标记 Corrected=true, got %+v", sink.ev)
|
||
}
|
||
if sink.ev.Level != contract.EvalOK {
|
||
t.Errorf("纠偏后应升为 ok，got %s (overall %.2f)", sink.ev.Level, sink.ev.Overall)
|
||
}
|
||
}
|
||
|
||
// 低分 → 纠偏后未提升 → 保留原版（不退步），不标记 Corrected，终值仍 poor。
|
||
func TestEvaluate_AutoRefine_KeepsOriginalWhenNoGain(t *testing.T) {
|
||
judge := func(_ context.Context, _, _ string) (string, error) {
|
||
return `{"score":1,"reason":"差"}`, nil // 怎么写都低分
|
||
}
|
||
gen := &fakeLLM{ready: true, chat: func(_ []llm.ChatMessage) (string, error) {
|
||
return "另一个同样糟糕的 error: 回答", nil
|
||
}}
|
||
sink := &fakeEvalSink{}
|
||
o := orchForEval(gen, judge, sink)
|
||
|
||
orig := "error: 初版很差的回答"
|
||
final := o.evaluate(&contract.Task{ID: "t2"}, "问题", orig, nil)
|
||
|
||
if final != orig {
|
||
t.Fatalf("未提升应保留原版，got %q", final)
|
||
}
|
||
if sink.ev.Corrected {
|
||
t.Error("未提升不应标记 Corrected")
|
||
}
|
||
if sink.ev.Level != contract.EvalPoor {
|
||
t.Errorf("应仍为 poor，got %s", sink.ev.Level)
|
||
}
|
||
}
|
||
|
||
// 非低分（ok/warn）不触发纠偏：生成器不应被调用。
|
||
func TestEvaluate_NoRefineWhenNotPoor(t *testing.T) {
|
||
called := false
|
||
gen := &fakeLLM{ready: true, chat: func(_ []llm.ChatMessage) (string, error) {
|
||
called = true
|
||
return "不该被调用", nil
|
||
}}
|
||
judge := func(_ context.Context, _, _ string) (string, error) {
|
||
return `{"score":5,"reason":"好"}`, nil
|
||
}
|
||
sink := &fakeEvalSink{}
|
||
o := orchForEval(gen, judge, sink)
|
||
|
||
orig := "一段质量不错的正常回答内容。"
|
||
final := o.evaluate(&contract.Task{ID: "t3"}, "问题", orig, nil)
|
||
|
||
if called {
|
||
t.Error("非低分不应触发纠偏（生成器被调用了）")
|
||
}
|
||
if final != orig {
|
||
t.Errorf("无纠偏应原样返回，got %q", final)
|
||
}
|
||
if sink.ev.Corrected || sink.ev.Level != contract.EvalOK {
|
||
t.Errorf("应为未纠偏的 ok，got level=%s corrected=%v", sink.ev.Level, sink.ev.Corrected)
|
||
}
|
||
}
|