feat: RAG 核心链 — embedding(provider) + Milvus 真连 + 入库/检索

mcp-go 接通向量 RAG:embedding(OpenAI 兼容 provider 抽象) + Milvus 真实连接,
kb_ingest 入库、wiki_search 真检索。retriever 节点一行不改即从桩变真。

- mcp-go internal/rag: embed.go(OpenAI 兼容 /embeddings 客户端) + milvus.go(milvus-sdk-go
  真连,集合按首次 embedding 维度懒建+AUTOINDEX/COSINE索引+加载,insert/向量search) +
  rag.go(Engine: 切块→embed→insert / embed query→search;embedding 或 Milvus 缺则降级)
- mcp-go gateway: 新工具 kb_ingest,wiki_search 换真(RAG 向量检索,kb 过滤 topK)
- mcp-go main: rag.Open 读 MILVUS_ADDR/EMBED_BASE_URL/EMBED_API_KEY/EMBED_MODEL 环境变量
- gateway: POST /api/v1/kb/ingest → kb_ingest(供知识库页/脚本)
- scripts/mock_embeddings.py: 确定性词法向量(字+bigram 哈希),无真 key 验证检索
- 开发期 embedding 接在线 API(无真 key 用 mock),见 llm-provider-strategy
- 验证: 全模块 build✓ + e2e PASS; live——入库5条→Milvus;retriever 节点查'向量数据库'
  →召回 Milvus 那条→DeepSeek 答'Milvus';查'知识图谱'→Neo4j(向量检索区分正确)

注: 当前向量单路;Bleve/Neo4j 融合 + rerank + 真实语义 embedding 为后续。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Blizzard
2026-06-10 17:07:36 +08:00
parent 71db0e295f
commit 84d1a1dd3a
11 changed files with 860 additions and 60 deletions
+72
View File
@@ -0,0 +1,72 @@
package rag
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
)
// embedClient 是 OpenAI 兼容的 embeddings 客户端(provider 抽象)。
// 开发期指向第三方在线 API 或本地 mock;生产期换自部署/在线 embedding 模型。
type embedClient struct {
baseURL string
apiKey string
model string
hc *http.Client
}
func newEmbedClient(baseURL, apiKey, model string) *embedClient {
return &embedClient{
baseURL: baseURL,
apiKey: apiKey,
model: model,
hc: &http.Client{Timeout: 30 * time.Second},
}
}
func (e *embedClient) ready() bool { return e != nil && e.baseURL != "" && e.model != "" }
// Embed 把若干文本向量化(OpenAI 兼容 /embeddings)。
func (e *embedClient) Embed(ctx context.Context, texts []string) ([][]float32, error) {
if !e.ready() {
return nil, fmt.Errorf("embedding not configured")
}
body, _ := json.Marshal(map[string]any{"model": e.model, "input": texts})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+"/embeddings", bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if e.apiKey != "" {
req.Header.Set("Authorization", "Bearer "+e.apiKey)
}
resp, err := e.hc.Do(req)
if err != nil {
return nil, fmt.Errorf("embed request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
buf := new(bytes.Buffer)
_, _ = buf.ReadFrom(resp.Body)
return nil, fmt.Errorf("embed http %d: %s", resp.StatusCode, buf.String())
}
var out struct {
Data []struct {
Embedding []float32 `json:"embedding"`
Index int `json:"index"`
} `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("embed decode: %w", err)
}
vecs := make([][]float32, len(out.Data))
for _, d := range out.Data {
if d.Index >= 0 && d.Index < len(vecs) {
vecs[d.Index] = d.Embedding
}
}
return vecs, nil
}