Blizzard
446b784fc6
补 Harness 已知洞：此前 LLM-judge 只看 input+output，看不到检索来源，幻觉其实没评。 - Evaluator.Score 增 sources 参数；有来源时走 llmJudgeGrounded：一次调用同时评 quality 质量 + faithfulness 忠实度，并列出 unsupported（未被来源支持的说法）→ 进 Flags。 综合分(有来源)=0.3规则+0.35质量+0.35忠实；无来源时维持原 0.4规则+0.6质量。Result 增 Faithful 字段。 - 透传检索来源：runGraph 返回 (answer, refs, err)，executeGraph 同步；Handle→evaluate(input,output,refs)； refsOf(board)=检索资料+工具产出。compose 路径暂返回 nil refs（不评忠实度）。 - eval 日志增「忠实 X.XX，来源 N」。 测试：单测覆盖 grounded（quality/faithfulness/unsupported 解析 + 加权 + flags）与无来源跳过； 3 处测试 runGraph 三返回值更新。live 实测 RAG 任务忠实 1.00/来源 1，judge 正确判定无编造。 project_analysis Harness 清单勾掉该项。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
77 lines
2.5 KiB
Go
77 lines
2.5 KiB
Go
package eino
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/sundynix/sundynix-dispatcher/internal/harness"
|
|
"github.com/sundynix/sundynix-dispatcher/internal/llm"
|
|
"github.com/sundynix/sundynix-shared/contract"
|
|
)
|
|
|
|
// echoLLM 回显最后一条 user 消息内容（确定性），便于两条执行路径逐字对比。
|
|
func echoLLM() *fakeLLM {
|
|
return &fakeLLM{
|
|
ready: true,
|
|
stream: func(m []llm.ChatMessage) string {
|
|
for i := len(m) - 1; i >= 0; i-- {
|
|
if m[i].Role == "user" {
|
|
return "ANS:" + m[i].Content
|
|
}
|
|
}
|
|
return "ANS:"
|
|
},
|
|
}
|
|
}
|
|
|
|
func runBoth(t *testing.T, graph string) (interp, comp string) {
|
|
t.Helper()
|
|
task := &contract.Task{ID: "t_eq", Graph: []byte(graph)}
|
|
|
|
o1 := &Orchestrator{pool: echoLLM(), breaker: harness.NewCircuitBreaker(), sink: &fakeSink{}}
|
|
a1, _, err := o1.runGraph(context.Background(), task, &execTracer{})
|
|
if err != nil {
|
|
t.Fatalf("runGraph: %v", err)
|
|
}
|
|
o2 := &Orchestrator{pool: echoLLM(), breaker: harness.NewCircuitBreaker(), sink: &fakeSink{}}
|
|
a2, err := o2.runComposeGraph(context.Background(), task, &execTracer{})
|
|
if err != nil {
|
|
t.Fatalf("runComposeGraph: %v", err)
|
|
}
|
|
return a1, a2
|
|
}
|
|
|
|
// TestComposeEquivalentLinear 多节点线性图：input→memory→agent→output，两路径成稿应逐字一致。
|
|
func TestComposeEquivalentLinear(t *testing.T) {
|
|
graph := `{"version":"1","nodes":[
|
|
{"id":"in","kind":"input","config":{"text":"什么是图编排"}},
|
|
{"id":"m","kind":"memory","config":{}},
|
|
{"id":"a","kind":"agent","config":{"system":"你是助手"}},
|
|
{"id":"out","kind":"output","config":{}}
|
|
],"edges":[
|
|
{"source":"in","target":"m"},{"source":"m","target":"a"},{"source":"a","target":"out"}
|
|
]}`
|
|
interp, comp := runBoth(t, graph)
|
|
if interp == "" || interp != comp {
|
|
t.Fatalf("线性图不等价: interp=%q compose=%q", interp, comp)
|
|
}
|
|
}
|
|
|
|
// TestComposeEquivalentBranch 分支图：input→branch→(真)A/(假)B，条件恒真应都走 A，两路径一致。
|
|
func TestComposeEquivalentBranch(t *testing.T) {
|
|
graph := `{"version":"1","nodes":[
|
|
{"id":"in","kind":"input","config":{"text":"hi"}},
|
|
{"id":"br","kind":"branch","config":{"condition":""}},
|
|
{"id":"a","kind":"agent","config":{"system":"A"}},
|
|
{"id":"b","kind":"agent","config":{"system":"B"}}
|
|
],"edges":[
|
|
{"source":"in","target":"br"},
|
|
{"source":"br","target":"a","sourceHandle":"true"},
|
|
{"source":"br","target":"b","sourceHandle":"false"}
|
|
]}`
|
|
interp, comp := runBoth(t, graph)
|
|
if interp == "" || interp != comp {
|
|
t.Fatalf("分支图不等价: interp=%q compose=%q", interp, comp)
|
|
}
|
|
}
|