fix(harness): T0.1 校准评测裁判 —— 让恒温器真触发(此前全 1.00 空转)
诊断:旧评测 LLM judge 恒给 0.94/1.00、从不判 poor → 低分自动纠偏闭环几乎从没 启动。根因两层: 1. 提示词软:只说"严格"但无评分基准、不强制挑毛病 → 模型恒锚定 4–5。 2. 数学更致命:归一 score/5 下限 0.2,叠加无来源 Overall=0.4*rule+0.6*s 的 0.4*rule≈0.4 底 → Overall 恒 ≥0.52、poor(<0.5)对"流畅但跑题/错误"永不可达。 修复: - judge 提示词改对抗性+rubric:默认怀疑、先点缺陷再打分、给死 1–5 评分基准、 要求用满区间;grounded judge 忠实度按编造说法递减(一处编造≤2)。 - 归一 score/5 → normJudge=(v-1)/4(1→0),让低质能压到 poor 触发纠偏。 - 测试:normJudge 区间 + 流畅跑题(judge=1)可达 poor + 好坏区分度;更新两处旧 断言(4→0.75, 2→0.25)。 live 验证(deepseek):跑题→0.40 poor→自动纠偏(0.40→0.55);截断→0.55 warn; 好答案→1.00 ok。校准前三者全 1.00。评测+纠偏的投入由"摆设"变"在用"。 DEPTH_ROADMAP T0.1 ✅。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ package harness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -58,8 +59,8 @@ func TestScore_WithLLMJudge(t *testing.T) {
|
||||
if r.LLM <= 0 {
|
||||
t.Fatalf("应有 LLM 分, got %.2f", r.LLM)
|
||||
}
|
||||
if r.LLM < 0.79 || r.LLM > 0.81 { // 4/5 = 0.8
|
||||
t.Errorf("LLM 分应归一化为 0.8, got %.2f", r.LLM)
|
||||
if r.LLM < 0.74 || r.LLM > 0.76 { // 校准版归一 normJudge(4)=(4-1)/4=0.75
|
||||
t.Errorf("LLM 分应归一化为 0.75, got %.2f", r.LLM)
|
||||
}
|
||||
if r.Reason != "相关且较完整" {
|
||||
t.Errorf("应解析出 reason, got %q", r.Reason)
|
||||
@@ -70,6 +71,40 @@ func TestScore_WithLLMJudge(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormJudge 钉死校准归一:1→0(最低,让低质能压到 poor)、3→0.5、5→1.0,用满区间。
|
||||
func TestNormJudge(t *testing.T) {
|
||||
cases := []struct {
|
||||
v, want float64
|
||||
}{{1, 0}, {2, 0.25}, {3, 0.5}, {4, 0.75}, {5, 1.0}, {0, 0}, {6, 1.0}}
|
||||
for _, c := range cases {
|
||||
if got := normJudge(c.v); got < c.want-1e-9 || got > c.want+1e-9 {
|
||||
t.Errorf("normJudge(%.0f)=%.3f, want %.3f", c.v, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestScore_PoorReachable 钉死校准核心:流畅但跑题的回答(judge 给 1)也能压到 poor 区间(<0.5)。
|
||||
// 旧归一(score/5 下限 0.2 + 0.4*rule 底)会让 Overall 恒 ≥0.52、poor 永不可达 → 纠偏闭环空转。
|
||||
func TestScore_PoorReachable(t *testing.T) {
|
||||
judge := func(score int) *Evaluator {
|
||||
return NewEvaluator(func() bool { return true },
|
||||
func(_ context.Context, _, _ string) (string, error) {
|
||||
return fmt.Sprintf(`{"score":%d,"reason":"x"}`, score), nil
|
||||
})
|
||||
}
|
||||
fluentOffTopic := "这是一段流畅、完整、看起来很认真的回答,但其实完全没有回答用户的问题。"
|
||||
// judge 判 1(跑题)→ Overall 应落入 poor 区间(<0.5),纠偏闭环才会触发。
|
||||
if r := judge(1).Score(context.Background(), "问题", fluentOffTopic, nil); r.Overall >= 0.5 {
|
||||
t.Fatalf("流畅但跑题(judge=1)应可达 poor(<0.5),got Overall=%.3f(恒温器空转的根因)", r.Overall)
|
||||
}
|
||||
// 区分度:好答案(judge=5)应明显高于差答案(judge=1)。
|
||||
good := judge(5).Score(context.Background(), "问题", fluentOffTopic, nil).Overall
|
||||
bad := judge(1).Score(context.Background(), "问题", fluentOffTopic, nil).Overall
|
||||
if good <= bad+0.3 {
|
||||
t.Fatalf("好坏答案区分度不足: good=%.3f bad=%.3f", good, bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScore_LLMJudgeBadJSONFallsBack(t *testing.T) {
|
||||
e := NewEvaluator(
|
||||
func() bool { return true },
|
||||
@@ -97,11 +132,11 @@ func TestScore_GroundedFaithfulness(t *testing.T) {
|
||||
if !strings.Contains(gotPrompt, "检索资料") || !strings.Contains(gotPrompt, "产品支持在线协作") {
|
||||
t.Fatalf("judge 提示词应包含检索资料, got: %q", gotPrompt)
|
||||
}
|
||||
if r.LLM < 0.79 || r.LLM > 0.81 { // quality 4/5
|
||||
t.Errorf("quality 应为 0.8, got %.2f", r.LLM)
|
||||
if r.LLM < 0.74 || r.LLM > 0.76 { // 校准归一 quality 4 → normJudge(4)=0.75
|
||||
t.Errorf("quality 应为 0.75, got %.2f", r.LLM)
|
||||
}
|
||||
if r.Faithful < 0.39 || r.Faithful > 0.41 { // faithfulness 2/5
|
||||
t.Errorf("忠实度应为 0.4, got %.2f", r.Faithful)
|
||||
if r.Faithful < 0.24 || r.Faithful > 0.26 { // 校准归一 faithfulness 2 → normJudge(2)=0.25
|
||||
t.Errorf("忠实度应为 0.25, got %.2f", r.Faithful)
|
||||
}
|
||||
if !contains(r.Flags, "未被来源支持:该产品支持离线模式") {
|
||||
t.Errorf("未支持说法应进 Flags, got %v", r.Flags)
|
||||
|
||||
Reference in New Issue
Block a user