package harness import ( "context" "strings" "testing" ) func TestRuleScore(t *testing.T) { cases := []struct { name string output string wantMax float64 // 期望分 ≤ 此值 wantMin float64 // 期望分 ≥ 此值 wantFlag string // 期望命中的标签(空=不校验) }{ {"正常", "杭州是浙江省会,历史悠久,有西湖等名胜,是著名的旅游与电商之城。", 1.0, 1.0, ""}, {"空", " ", 0, 0, "空输出"}, {"过短", "好的", 0.7, 0, "输出过短"}, {"拒答", "抱歉,我无法回答这个问题,因为信息不足,请谅解理解支持。", 0.8, 0, "疑似拒答/错误"}, } for _, c := range cases { s, flags := ruleScore(c.output) if s > c.wantMax+1e-9 || s < c.wantMin-1e-9 { t.Errorf("%s: 分 %.2f 不在 [%.2f,%.2f]", c.name, s, c.wantMin, c.wantMax) } if c.wantFlag != "" && !contains(flags, c.wantFlag) { t.Errorf("%s: 期望命中标签 %q, got %v", c.name, c.wantFlag, flags) } } } func TestRuleScore_HeavyRepeat(t *testing.T) { out := strings.Repeat("这是一句会重复很多次的废话\n", 4) _, flags := ruleScore(out) if !contains(flags, "重复啰嗦") { t.Errorf("应命中重复啰嗦, got %v", flags) } } func TestScore_RuleOnly(t *testing.T) { e := NewEvaluator(nil, nil) // 无 LLM → 仅规则 r := e.Score(context.Background(), "问题", "一段质量不错的较完整回答内容,长度足够,没有任何问题。", nil) if r.LLM != 0 || r.Overall != r.Rule { t.Errorf("无 LLM 时 Overall 应等于规则分, got overall=%.2f rule=%.2f llm=%.2f", r.Overall, r.Rule, r.LLM) } } func TestScore_WithLLMJudge(t *testing.T) { // 注入假评审:返回带围栏的 JSON,验证解析 + 归一化 + 综合权重。 e := NewEvaluator( func() bool { return true }, func(ctx context.Context, sys, user string) (string, error) { return "```json\n{\"score\":4,\"reason\":\"相关且较完整\"}\n```", nil }, ) r := e.Score(context.Background(), "介绍杭州", "杭州是浙江省会,西湖闻名,历史与现代交融,电商发达。", nil) if r.LLM <= 0 { t.Fatalf("应有 LLM 分, got %.2f", r.LLM) } if r.LLM < 0.79 || r.LLM > 0.81 { // 4/5 = 0.8 t.Errorf("LLM 分应归一化为 0.8, got %.2f", r.LLM) } if r.Reason != "相关且较完整" { t.Errorf("应解析出 reason, got %q", r.Reason) } want := 0.4*r.Rule + 0.6*r.LLM if r.Overall < want-1e-9 || r.Overall > want+1e-9 { t.Errorf("综合分应为 0.4*规则+0.6*LLM=%.3f, got %.3f", want, r.Overall) } } func TestScore_LLMJudgeBadJSONFallsBack(t *testing.T) { e := NewEvaluator( func() bool { return true }, func(ctx context.Context, sys, user string) (string, error) { return "我觉得还行吧", nil }, ) r := e.Score(context.Background(), "q", "一段足够长且正常的回答内容用于评测。", nil) if r.LLM != 0 || r.Overall != r.Rule { t.Errorf("LLM 返回非 JSON 应回退到规则分, got overall=%.2f llm=%.2f", r.Overall, r.LLM) } } func TestScore_GroundedFaithfulness(t *testing.T) { // 有检索来源 → 走忠实度评测:judge 返回 quality/faithfulness/unsupported。 var gotPrompt string e := NewEvaluator( func() bool { return true }, func(ctx context.Context, sys, user string) (string, error) { gotPrompt = user return `{"quality":4,"faithfulness":2,"unsupported":["该产品支持离线模式"],"reason":"部分说法无资料支撑"}`, nil }, ) r := e.Score(context.Background(), "这产品支持什么", "它支持在线与离线模式。", []string{"产品支持在线协作。", "产品提供云端存储。"}) if !strings.Contains(gotPrompt, "检索资料") || !strings.Contains(gotPrompt, "产品支持在线协作") { t.Fatalf("judge 提示词应包含检索资料, got: %q", gotPrompt) } if r.LLM < 0.79 || r.LLM > 0.81 { // quality 4/5 t.Errorf("quality 应为 0.8, got %.2f", r.LLM) } if r.Faithful < 0.39 || r.Faithful > 0.41 { // faithfulness 2/5 t.Errorf("忠实度应为 0.4, got %.2f", r.Faithful) } if !contains(r.Flags, "未被来源支持:该产品支持离线模式") { t.Errorf("未支持说法应进 Flags, got %v", r.Flags) } want := 0.3*r.Rule + 0.35*r.LLM + 0.35*r.Faithful if r.Overall < want-1e-9 || r.Overall > want+1e-9 { t.Errorf("综合分应为 0.3规则+0.35质量+0.35忠实=%.3f, got %.3f", want, r.Overall) } } func TestScore_NoSourcesSkipsFaithfulness(t *testing.T) { e := NewEvaluator( func() bool { return true }, func(ctx context.Context, sys, user string) (string, error) { return `{"score":5,"reason":"好"}`, nil }, ) r := e.Score(context.Background(), "q", "一段足够长且正常的回答内容用于评测。", nil) if r.Faithful != 0 { t.Errorf("无来源不应评忠实度, got %.2f", r.Faithful) } } func contains(ss []string, want string) bool { for _, s := range ss { if s == want { return true } } return false }