feat: RAG 核心链 — embedding(provider) + Milvus 真连 + 入库/检索

mcp-go 接通向量 RAG:embedding(OpenAI 兼容 provider 抽象) + Milvus 真实连接,
kb_ingest 入库、wiki_search 真检索。retriever 节点一行不改即从桩变真。

- mcp-go internal/rag: embed.go(OpenAI 兼容 /embeddings 客户端) + milvus.go(milvus-sdk-go
  真连,集合按首次 embedding 维度懒建+AUTOINDEX/COSINE索引+加载,insert/向量search) +
  rag.go(Engine: 切块→embed→insert / embed query→search;embedding 或 Milvus 缺则降级)
- mcp-go gateway: 新工具 kb_ingest,wiki_search 换真(RAG 向量检索,kb 过滤 topK)
- mcp-go main: rag.Open 读 MILVUS_ADDR/EMBED_BASE_URL/EMBED_API_KEY/EMBED_MODEL 环境变量
- gateway: POST /api/v1/kb/ingest → kb_ingest(供知识库页/脚本)
- scripts/mock_embeddings.py: 确定性词法向量(字+bigram 哈希),无真 key 验证检索
- 开发期 embedding 接在线 API(无真 key 用 mock),见 llm-provider-strategy
- 验证: 全模块 build✓ + e2e PASS; live——入库5条→Milvus;retriever 节点查'向量数据库'
  →召回 Milvus 那条→DeepSeek 答'Milvus';查'知识图谱'→Neo4j(向量检索区分正确)

注: 当前向量单路;Bleve/Neo4j 融合 + rerank + 真实语义 embedding 为后续。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Blizzard
2026-06-10 17:07:36 +08:00
parent 71db0e295f
commit 84d1a1dd3a
11 changed files with 860 additions and 60 deletions
+35 -14
View File
@@ -6,12 +6,14 @@ import (
"encoding/json"
"fmt"
"log"
"strings"
sharedbus "github.com/sundynix/sundynix-shared/bus"
"github.com/sundynix/sundynix-shared/contract"
"github.com/sundynix/sundynix-mcp-go/internal/history"
"github.com/sundynix/sundynix-mcp-go/internal/memory"
"github.com/sundynix/sundynix-mcp-go/internal/rag"
"github.com/sundynix/sundynix-mcp-go/internal/search"
)
@@ -21,10 +23,11 @@ type Gateway struct {
search *search.Hybrid
memory *memory.Store
history *history.Store
rag *rag.Engine
}
func NewGateway(b *sharedbus.Bus, s *search.Hybrid, m *memory.Store, h *history.Store) *Gateway {
return &Gateway{bus: b, search: s, memory: m, history: h}
func NewGateway(b *sharedbus.Bus, s *search.Hybrid, m *memory.Store, h *history.Store, r *rag.Engine) *Gateway {
return &Gateway{bus: b, search: s, memory: m, history: h, rag: r}
}
// Serve 以队列组通配订阅 sundynix.tools.go.>,按工具名分发并阻塞。
@@ -34,7 +37,7 @@ func (g *Gateway) Serve(ctx context.Context) error {
return err
}
defer func() { _ = unsub() }()
log.Printf("[mcp_go] tools ready on %s (queue=%s): wiki_search, memory_get, memory_upsert, history_get, history_append, echo",
log.Printf("[mcp_go] tools ready on %s (queue=%s): wiki_search, kb_ingest, memory_get, memory_upsert, history_get, history_append, echo",
contract.SubjectToolsGoAll, contract.QueueToolsGo)
<-ctx.Done()
return ctx.Err()
@@ -46,6 +49,8 @@ func (g *Gateway) dispatch(ctx context.Context, call *contract.ToolCall) *contra
switch call.Tool {
case "wiki_search":
return g.wikiSearch(ctx, call)
case "kb_ingest":
return g.kbIngest(ctx, call)
case "memory_get":
return g.memoryGet(ctx, call)
case "memory_upsert":
@@ -110,24 +115,40 @@ func (g *Gateway) memoryUpsert(ctx context.Context, call *contract.ToolCall) *co
return &contract.ToolResult{OK: true, Content: fmt.Sprintf("已记住 %s 的「%s」", uid, key)}
}
// wikiSearch 调 Hybrid 混合检索引擎。引擎目前为桩(返回空),
// 这里仍把调用链路做真:真实接入 Bleve/Milvus/Neo4j 后无需改动协议
// wikiSearch 经 RAG 引擎做向量检索(embedding + Milvus)。
// RAG 未就绪时降级返回空命中(不阻断图执行)
func (g *Gateway) wikiSearch(ctx context.Context, call *contract.ToolCall) *contract.ToolResult {
q, _ := call.Args["q"].(string)
results, err := g.search.Query(ctx, q, 5)
kb, _ := call.Args["kb"].(string)
topK := 5
if v, ok := call.Args["topK"].(float64); ok && v > 0 {
topK = int(v)
}
if !g.rag.Ready() {
return &contract.ToolResult{OK: true, Content: "[wiki_search] RAG 未配置(需 embedding + Milvus),无召回"}
}
hits, err := g.rag.Search(ctx, kb, q, topK)
if err != nil {
return &contract.ToolResult{OK: false, Error: "wiki_search: " + err.Error()}
}
return &contract.ToolResult{
OK: true,
Content: fmt.Sprintf("[wiki_search] 命中 %d 条(Bleve+Milvus+Neo4j 混合检索桩)查询=%q", len(results), preview(q)),
var b strings.Builder
fmt.Fprintf(&b, "[wiki_search] 命中 %d 条(Milvus 向量检索):\n", len(hits))
for i, h := range hits {
fmt.Fprintf(&b, "%d. (%.3f) %s\n", i+1, h.Score, h.Text)
}
return &contract.ToolResult{OK: true, Content: strings.TrimRight(b.String(), "\n")}
}
func preview(s string) string {
r := []rune(s)
if len(r) > 40 {
return string(r[:40]) + "…"
// kbIngest 把文本入库(切块→embedding→Milvus)。
func (g *Gateway) kbIngest(ctx context.Context, call *contract.ToolCall) *contract.ToolResult {
kb, _ := call.Args["kb"].(string)
text, _ := call.Args["text"].(string)
if text == "" {
return &contract.ToolResult{OK: false, Error: "kb_ingest: text 必填"}
}
return s
n, err := g.rag.Ingest(ctx, kb, text)
if err != nil {
return &contract.ToolResult{OK: false, Error: "kb_ingest: " + err.Error()}
}
return &contract.ToolResult{OK: true, Content: fmt.Sprintf("已入库 %d 块到知识库 %q", n, kb)}
}