From a17e25b6ba66b19c9603ed828e397d00256273b7 Mon Sep 17 00:00:00 2001 From: Blizzard Date: Tue, 30 Jun 2026 13:56:39 +0800 Subject: [PATCH] =?UTF-8?q?feat(rag):=20=E6=A3=80=E7=B4=A2=E8=B4=A8?= =?UTF-8?q?=E9=87=8F=E8=AF=84=E6=B5=8B=20=E2=80=94=E2=80=94=20=E5=8D=95?= =?UTF-8?q?=E8=B7=AF=E6=A3=80=E7=B4=A2=E8=83=BD=E5=8A=9B=20+=20recall@k/MR?= =?UTF-8?q?R=20=E8=AF=84=E6=B5=8B=E5=8F=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 引擎抽 searchPaths(三路召回) + SearchByMode(vector/fulltext/graph/hybrid， 纯检索不 rerank，公平对比)；kb_search 加 mode 参数(空=生产含rerank)， gateway KbSearch 透传 mode - scripts/rageval.py：标注语料+查询 → 四模式 recall@k/MRR 对比表（可复用） - live 量化：纯语义改写让全文0.88/图谱0.75 漏召回，混合 1.00 兜回， 混合=各路上界的稳健组合 Co-Authored-By: Claude Opus 4.8 --- scripts/rageval.py | 106 ++++++++++++++++++++++++ sundynix-gateway/internal/handler/kb.go | 4 + sundynix-mcp-go/internal/mcp/gateway.go | 10 ++- sundynix-mcp-go/internal/rag/rag.go | 48 ++++++++--- 4 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 scripts/rageval.py diff --git a/scripts/rageval.py b/scripts/rageval.py new file mode 100644 index 0000000..d9883ae --- /dev/null +++ b/scripts/rageval.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# RAG 离线检索质量评测：四模式(向量/全文/图谱/混合) recall@k + MRR，量化"混合 vs 单路"。 +import json, sys, time, urllib.request + +BASE = "http://127.0.0.1:8080" +KB = "evalset" +TOKEN = open(sys.argv[1]).read().strip() + +# 语料：每条一句话事实，唯一标记=载具名（用于判定召回）。 +CORPUS = [ + ("启明星一号", "启明星一号是一颗对地观测卫星，由总设计师周文渊研制，搭载高光谱相机。"), + ("玄鸟二号", "玄鸟二号是一型运载火箭，由总工程师林书豪设计，使用液氧甲烷发动机。"), + ("蛟龙三号", "蛟龙三号是一台载人潜水器，由首席科学家陈海平主持，最大下潜深度7000米。"), + ("烛龙四号", "烛龙四号是一颗深空探测器，由项目主管吴宇航负责，用于探测木星磁场。"), + ("麒麟五号", "麒麟五号是一座空间站，由总指挥赵长青建造，配备机械臂。"), + ("白泽六号", "白泽六号是一颗通信卫星，由总设计师钱明轩研制，实现量子通信。"), + ("饕餮七号", "饕餮七号是一艘破冰船，由总工程师孙铁柱主持，采用核动力驱动。"), + ("穷奇八号", "穷奇八号是一艘行星际飞船，由首席科学家周伯通设计，依靠太阳帆推进。"), + ("梼杌九号", "梼杌九号是一架预警机，由总设计师黄药师研制，配红外制导系统。"), + ("应龙十号", "应龙十号是一款高超声速飞行器，由总工程师欧阳锋设计，采用折叠翼。"), + ("重明十一号", "重明十一号是一架电动垂直起降飞行器，由总设计师洪七公研制，用全固态电池。"), + ("毕方十二号", "毕方十二号是一座防御平台，由总指挥段誉建造，装备激光武器。"), + # —— 干扰项：与各目标同族近义，逼向量路在相似项中区分 —— + ("天罡卫星", "天罡卫星是一颗对地观测卫星，由总设计师周文博研制，搭载多光谱相机。"), + ("地煞火箭", "地煞火箭是一型运载火箭，由总工程师林书文设计，使用液氧煤油发动机。"), + ("潜龙号", "潜龙号是一台载人潜水器，由首席科学家陈海峰主持，最大下潜深度5000米。"), + ("探火号", "探火号是一颗深空探测器，由项目主管吴宇辉负责，用于探测火星大气。"), + ("天宫号", "天宫号是一座空间站，由总指挥赵长河建造，配备太阳能帆板。"), + ("鹊桥号", "鹊桥号是一颗通信卫星，由总设计师钱明远研制，做激光通信。"), + ("破冰者号", "破冰者号是一艘破冰船，由总工程师孙铁山主持，采用柴油动力。"), + ("远航号", "远航号是一艘行星际飞船，由首席科学家周伯文设计，依靠核电推进。"), + ("天眼号", "天眼号是一架预警机，由总设计师黄药剂研制，配相控阵雷达。"), + ("风刃号", "风刃号是一款高超声速飞行器，由总工程师欧阳峰设计，采用乘波体。"), + ("飞羽号", "飞羽号是一架电动垂直起降飞行器，由总设计师洪七叔研制，用液流电池。"), + ("盾卫号", "盾卫号是一座防御平台，由总指挥段正淳建造，装备动能拦截弹。"), +] +# 查询：q=问句, m=期望命中的标记, t=类型(便于看各路强弱) +QUERIES = [ + ("哪台设备可以下潜到七千米的深海", "蛟龙三号", "语义"), + ("用来在冰封海域开路的核动力船", "饕餮七号", "语义"), + ("能垂直起降的电动飞行器用了什么电池", "重明十一号", "语义"), + ("靠激光拦截目标的防御系统是哪个", "毕方十二号", "语义"), + ("液氧甲烷发动机用在哪", "玄鸟二号", "关键词"), + ("木星磁场探测", "烛龙四号", "关键词"), + ("量子通信卫星", "白泽六号", "关键词"), + ("太阳帆推进", "穷奇八号", "关键词"), + ("周文渊负责研制的是什么", "启明星一号", "实体关系"), + ("孙铁柱主持的项目", "饕餮七号", "实体关系"), + ("黄药师设计了哪款预警机", "梼杌九号", "实体关系"), + ("赵长青建造的空间站叫什么", "麒麟五号", "实体关系"), + # —— 纯语义改写：尽量避开与目标文档的字面重叠，逼全文路失手、看混合是否兜住 —— + ("可搭乘人员探入数千米幽暗海沟的勘探艇", "蛟龙三号", "纯语义"), + ("环绕巨行星测量其磁场强度的太空仪器", "烛龙四号", "纯语义"), + ("能从陆地竖直升空、不烧油的载客飞行器", "重明十一号", "纯语义"), + ("以高能光束摧毁来袭目标的拦截阵地", "毕方十二号", "纯语义"), +] +MODES = ["vector", "fulltext", "graph", "hybrid"] + +def post(path, body): + req = urllib.request.Request(BASE + path, data=json.dumps(body).encode(), + headers={"Authorization": "Bearer " + TOKEN, "Content-Type": "application/json"}, method="POST") + return json.load(urllib.request.urlopen(req, timeout=20)) + +def ingest(): + for name, text in CORPUS: + post("/api/v1/kb/note", {"kb": KB, "name": name, "content": text}) + print(f"已提交 {len(CORPUS)} 篇，等待入库(切块/向量/图谱)…") + +def search(q, mode, topk=5): + r = post("/api/v1/kb/search", {"kb": KB, "q": q, "topK": topk, "mode": mode}) + return [h.get("Text") or h.get("text") or "" for h in (r.get("hits") or [])] + +def rank_of(hits, marker): + for i, t in enumerate(hits): + if marker in t: + return i + 1 + return 0 + +def main(): + if "--noingest" not in sys.argv: + ingest() + time.sleep(40) # 等队列把 12 篇全部 embed+图谱抽取完 + print(f"\n{'模式':<10}{'recall@3':>10}{'recall@5':>10}{'MRR':>8}") + print("-" * 40) + per_type = {m: {} for m in MODES} + for mode in MODES: + r3 = r5 = mrr = 0.0 + bytype = {} + for q, marker, typ in QUERIES: + hits = search(q, mode) + rk = rank_of(hits, marker) + hit3, hit5 = (1 if 0 < rk <= 3 else 0), (1 if 0 < rk <= 5 else 0) + r3 += hit3; r5 += hit5; mrr += (1.0 / rk if rk else 0.0) + bytype.setdefault(typ, [0, 0]); bytype[typ][0] += hit5; bytype[typ][1] += 1 + n = len(QUERIES) + print(f"{mode:<10}{r3/n:>10.2f}{r5/n:>10.2f}{mrr/n:>8.2f}") + per_type[mode] = {t: f"{v[0]}/{v[1]}" for t, v in bytype.items()} + types = list(dict.fromkeys(t for _, _, t in QUERIES)) + print("\n各模式 recall@5 分查询类型（看各路强弱）:") + print("模式".ljust(10) + "".join(t.rjust(10) for t in types)) + for mode in MODES: + d = per_type[mode] + print(mode.ljust(10) + "".join(d.get(t, "").rjust(10) for t in types)) + +if __name__ == "__main__": + main() diff --git a/sundynix-gateway/internal/handler/kb.go b/sundynix-gateway/internal/handler/kb.go index 85fbdbc..1f4dbfd 100644 --- a/sundynix-gateway/internal/handler/kb.go +++ b/sundynix-gateway/internal/handler/kb.go @@ -517,6 +517,7 @@ func (h *Handler) KbSearch(c *gin.Context) { KB string `json:"kb"` Q string `json:"q"` TopK int `json:"topK"` + Mode string `json:"mode"` // 空=生产混合(含rerank)；vector/fulltext/graph/hybrid=评测单路/纯融合 } if err := c.ShouldBindJSON(&body); err != nil || body.Q == "" { c.JSON(http.StatusBadRequest, gin.H{"error": "q required"}) @@ -526,6 +527,9 @@ func (h *Handler) KbSearch(c *gin.Context) { if body.TopK > 0 { args["topK"] = body.TopK } + if body.Mode != "" { + args["mode"] = body.Mode + } res, err := h.bus.CallTool(c.Request.Context(), contract.ToolSubjectGo("kb_search"), &contract.ToolCall{Tool: "kb_search", Args: args}) if err != nil { diff --git a/sundynix-mcp-go/internal/mcp/gateway.go b/sundynix-mcp-go/internal/mcp/gateway.go index d622f73..346fe91 100644 --- a/sundynix-mcp-go/internal/mcp/gateway.go +++ b/sundynix-mcp-go/internal/mcp/gateway.go @@ -341,7 +341,15 @@ func (g *Gateway) kbSearch(ctx context.Context, call *contract.ToolCall) *contra if !g.rag.Ready() { return &contract.ToolResult{OK: true, Content: "[]"} } - hits, err := g.rag.Search(ctx, kb, q, topK) + // mode 空=生产混合检索(含 rerank)；显式 vector/fulltext/graph/hybrid=评测用单路/纯融合(不 rerank)。 + mode, _ := call.Args["mode"].(string) + var hits []rag.Hit + var err error + if mode == "" { + hits, err = g.rag.Search(ctx, kb, q, topK) + } else { + hits = g.rag.SearchByMode(ctx, kb, q, topK, mode) + } if err != nil { return &contract.ToolResult{OK: false, Error: "kb_search: " + err.Error()} } diff --git a/sundynix-mcp-go/internal/rag/rag.go b/sundynix-mcp-go/internal/rag/rag.go index e3ea870..23f09b5 100644 --- a/sundynix-mcp-go/internal/rag/rag.go +++ b/sundynix-mcp-go/internal/rag/rag.go @@ -233,16 +233,7 @@ func (e *Engine) Search(ctx context.Context, kb, query string, topK int) ([]Hit, } fanout := topK * 3 - // 向量路 - vecs, err := e.embed().Embed(ctx, []string{query}) - if err != nil || len(vecs) == 0 { - return nil, err - } - vecHits, _ := e.mv.search(ctx, kb, vecs[0], fanout) - // 全文路 - ftHits := e.bleve.search(kb, query, fanout) - // 图谱路（GraphRAG：查询提到的实体的相连三元组） - graphHits := e.graph.search(ctx, kb, query, fanout) + vecHits, ftHits, graphHits := e.searchPaths(ctx, kb, query, fanout) // RRF 融合（三路，按文本去重） cand := rrf([][]Hit{vecHits, ftHits, graphHits}, fanout) log.Printf("[rag] hybrid: 向量=%d 全文=%d 图谱=%d → 融合=%d", len(vecHits), len(ftHits), len(graphHits), len(cand)) @@ -277,6 +268,43 @@ func (e *Engine) DeleteDoc(ctx context.Context, kb, fileID string) error { return nil } +// searchPaths 跑三路召回，返回各路命中（供混合融合与离线评测按单路对比）。 +func (e *Engine) searchPaths(ctx context.Context, kb, query string, fanout int) (vec, ft, graph []Hit) { + if vecs, err := e.embed().Embed(ctx, []string{query}); err == nil && len(vecs) > 0 { + vec, _ = e.mv.search(ctx, kb, vecs[0], fanout) + } + ft = e.bleve.search(kb, query, fanout) + graph = e.graph.search(ctx, kb, query, fanout) + return +} + +// SearchByMode 按指定模式返回 topK（评测用，纯检索不 rerank，便于公平对比）。 +// mode: vector|fulltext|graph|hybrid(RRF)。 +func (e *Engine) SearchByMode(ctx context.Context, kb, query string, topK int, mode string) []Hit { + if !e.Ready() || topK <= 0 { + if topK <= 0 { + topK = 5 + } + } + fanout := topK * 3 + vec, ft, graph := e.searchPaths(ctx, kb, query, fanout) + var hits []Hit + switch mode { + case "vector": + hits = vec + case "fulltext": + hits = ft + case "graph": + hits = graph + default: // hybrid + hits = rrf([][]Hit{vec, ft, graph}, fanout) + } + if len(hits) > topK { + hits = hits[:topK] + } + return hits +} + func (e *Engine) Close() { if e.mv != nil { e.mv.close()