refactor(kb): 文库列表/正文分离 + [[双链]]索引表(可扛大文件)
不再一次拉回整库正文、不再前端扫全文 —— 列表只读元数据,正文按需取,链接走索引。 - store: SaveDoc 维护 size+preview(前 500 字);ListVault 仅 Select 元数据(name/size/preview, 不含 content);GetDoc 取单篇全文;DocLink 表 + ReplaceDocLinks(入库/编辑时按 from 重建出链) + ListLinks。 - gateway: 入库/笔记保存时正则抽 [[链接]]→ReplaceDocLinks 维护索引; /kb/vault 改返元数据+预览;新增 /kb/doc(单篇全文) 与 /kb/links(全库双链)。 - 前端:listVault 返元数据,新增 getDoc/listLinks;VaultPanel 列表只展示名/字数, 选中后 getDoc 按需载正文(带加载态),反链/笔记关系图改用服务端 links 索引(不扫全文)。 验证:curl /kb/vault 仅 name/size/preview;/kb/doc 取单篇;/kb/links 返 3 条双链。 Preview:文库点「架构总览」按需载正文(平台分五层)、反向链接(1)=Dispatcher(来自索引)。tsc+vite+gateway build 通过。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -95,12 +96,28 @@ func (h *Handler) KbSaveNote(c *gin.Context) {
|
||||
}
|
||||
owner := userID(c)
|
||||
_ = h.db.EnsureKB(c.Request.Context(), owner, rawKB(body.KB), "general")
|
||||
_ = h.db.SaveDoc(c.Request.Context(), owner, rawKB(body.KB), body.Name, body.Content)
|
||||
// 落库 + 重建索引由后台 runIngest 统一处理(forceDoc=name 保持笔记身份)。
|
||||
job := newJobID()
|
||||
go h.runIngest(job, owner, rawKB(body.KB), scopedKB(c, body.KB), body.Name, "", nil, body.Content)
|
||||
c.JSON(http.StatusAccepted, gin.H{"job_id": job, "name": body.Name})
|
||||
}
|
||||
|
||||
// wikiLinks 从内容抽取所有 [[名称]](忽略别名)去重,用于维护双链索引。
|
||||
func wikiLinks(s string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range wikiRe.FindAllStringSubmatch(s, -1) {
|
||||
n := strings.TrimSpace(m[1])
|
||||
if n != "" && !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var wikiRe = regexp.MustCompile(`\[\[([^\]|]+)(\|[^\]]*)?\]\]`)
|
||||
|
||||
// KbVault: GET /api/v1/kb/vault?kb= —— 某知识库的全部原始文档(名+内容),供 Obsidian 式文库浏览。
|
||||
func (h *Handler) KbVault(c *gin.Context) {
|
||||
rows, err := h.db.ListVault(c.Request.Context(), userID(c), rawKB(c.Query("kb")))
|
||||
@@ -110,11 +127,35 @@ func (h *Handler) KbVault(c *gin.Context) {
|
||||
}
|
||||
docs := make([]gin.H, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
docs = append(docs, gin.H{"name": r.Name, "content": r.Content})
|
||||
docs = append(docs, gin.H{"name": r.Name, "size": r.Size, "preview": r.Preview})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"docs": docs})
|
||||
}
|
||||
|
||||
// KbDoc: GET /api/v1/kb/doc?kb=&name= —— 取单篇文档全文(按需加载,不在列表里拉全量)。
|
||||
func (h *Handler) KbDoc(c *gin.Context) {
|
||||
d, err := h.db.GetDoc(c.Request.Context(), userID(c), rawKB(c.Query("kb")), c.Query("name"))
|
||||
if err != nil || d == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "文档不存在"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"name": d.Name, "content": d.Content, "size": d.Size})
|
||||
}
|
||||
|
||||
// KbLinks: GET /api/v1/kb/links?kb= —— 某库全部 [[双链]](from→to),供反链/笔记关系图。
|
||||
func (h *Handler) KbLinks(c *gin.Context) {
|
||||
rows, err := h.db.ListLinks(c.Request.Context(), userID(c), rawKB(c.Query("kb")))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
links := make([]gin.H, 0, len(rows))
|
||||
for _, l := range rows {
|
||||
links = append(links, gin.H{"from": l.FromName, "to": l.ToName})
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"links": links})
|
||||
}
|
||||
|
||||
// noteName 取文本首个非空行作笔记名(截断 40 字),用于文本入库的文库留存。
|
||||
func noteName(text string) string {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
@@ -191,7 +232,8 @@ func (h *Handler) runIngest(job, owner, kbName, scoped, forceDoc, filename strin
|
||||
docName = noteName(text)
|
||||
}
|
||||
if text != "" {
|
||||
_ = h.db.SaveDoc(ctx, owner, kbName, docName, text)
|
||||
_ = h.db.SaveDoc(ctx, owner, kbName, docName, text, "", len([]rune(text)))
|
||||
_ = h.db.ReplaceDocLinks(ctx, owner, kbName, docName, wikiLinks(text)) // 维护 [[双链]] 索引
|
||||
}
|
||||
|
||||
// 调 mcp-go kb_ingest(带 job_id):它会发 切块/向量化/写入/完成 事件 + CompleteStream。
|
||||
|
||||
@@ -30,7 +30,9 @@ func New(db *store.Postgres, cache *store.Redis, bus *nats.Bus) *gin.Engine {
|
||||
api.POST("/kb/ingest_file", h.KbIngestFile) // 文件入库(docx/xlsx/pdf… 异步)
|
||||
api.GET("/kb/ingest/:id/stream", h.KbIngestStream) // 入库进度 SSE(实时监控)
|
||||
api.POST("/kb/search", h.KbSearch) // 知识库检索台(→ mcp-go kb_search)
|
||||
api.GET("/kb/vault", h.KbVault) // 文库:原始文档浏览(Obsidian 式)
|
||||
api.GET("/kb/vault", h.KbVault) // 文库:文档列表(仅元数据+预览)
|
||||
api.GET("/kb/doc", h.KbDoc) // 取单篇文档全文(按需加载)
|
||||
api.GET("/kb/links", h.KbLinks) // 某库全部 [[双链]](反链/笔记关系图)
|
||||
api.POST("/kb/note", h.KbSaveNote) // 新建/编辑笔记(落库 + 按 doc 重入库)
|
||||
api.GET("/kb/graph", h.KbGraph) // 知识图谱三元组(→ mcp-go kb_graph,Neo4j)
|
||||
|
||||
|
||||
@@ -111,24 +111,81 @@ type DocLink struct {
|
||||
|
||||
func (DocLink) TableName() string { return "sundynix_doc_link" }
|
||||
|
||||
// SaveDoc 写入/更新一份文档(owner+kb+name 唯一,重名覆盖内容)。
|
||||
func (p *Postgres) SaveDoc(ctx context.Context, owner, kb, name, content string) error {
|
||||
// docHead 取前 n 个 rune 作预览。
|
||||
func docHead(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n])
|
||||
}
|
||||
|
||||
// SaveDoc 写入/更新一份文档(owner+kb+name 唯一,重名覆盖)。
|
||||
// 同时维护 size 与 preview(列表只读它们,不拉全文)。content 入参为内联正文;
|
||||
// objectKey 非空表示正文已转 MinIO(此时 content 传空)。
|
||||
func (p *Postgres) SaveDoc(ctx context.Context, owner, kb, name, content, objectKey string, size int) error {
|
||||
if p.db == nil {
|
||||
return nil
|
||||
}
|
||||
preview := docHead(content, 500)
|
||||
return p.db.WithContext(ctx).Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "owner"}, {Name: "kb"}, {Name: "name"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"content"}),
|
||||
}).Create(&Doc{Owner: owner, KB: kb, Name: name, Content: content}).Error
|
||||
DoUpdates: clause.Assignments(map[string]any{"content": content, "object_key": objectKey, "size": size, "preview": preview, "updated_at": time.Now()}),
|
||||
}).Create(&Doc{Owner: owner, KB: kb, Name: name, Content: content, ObjectKey: objectKey, Size: size, Preview: preview}).Error
|
||||
}
|
||||
|
||||
// ListVault 返回某 owner 某 kb 的全部文档(名+内容),供文库浏览/双链/反链。
|
||||
// ListVault 返回文库列表(仅元数据 + 预览,不含全文),避免一次拉回整库正文。
|
||||
func (p *Postgres) ListVault(ctx context.Context, owner, kb string) ([]Doc, error) {
|
||||
if p.db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var rows []Doc
|
||||
err := p.db.WithContext(ctx).Where("owner = ? AND kb = ?", owner, kb).Order("id").Find(&rows).Error
|
||||
err := p.db.WithContext(ctx).
|
||||
Select("id", "name", "size", "preview", "object_key", "updated_at").
|
||||
Where("owner = ? AND kb = ?", owner, kb).Order("updated_at desc").Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// GetDoc 取单篇文档(含全文 Content 与 ObjectKey,供按需阅读)。
|
||||
func (p *Postgres) GetDoc(ctx context.Context, owner, kb, name string) (*Doc, error) {
|
||||
if p.db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var d Doc
|
||||
if err := p.db.WithContext(ctx).Where("owner = ? AND kb = ? AND name = ?", owner, kb, name).First(&d).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// ReplaceDocLinks 重建某文档的出链(先删旧,再插新)—— 入库/编辑时调用。
|
||||
func (p *Postgres) ReplaceDocLinks(ctx context.Context, owner, kb, from string, tos []string) error {
|
||||
if p.db == nil {
|
||||
return nil
|
||||
}
|
||||
return p.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Where("owner = ? AND kb = ? AND from_name = ?", owner, kb, from).Delete(&DocLink{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, to := range tos {
|
||||
if to == "" || to == from {
|
||||
continue
|
||||
}
|
||||
if err := tx.Create(&DocLink{Owner: owner, KB: kb, FromName: from, ToName: to}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListLinks 返回某 kb 的全部 [[双链]](from→to),供反链/笔记关系图按需查询(数据小)。
|
||||
func (p *Postgres) ListLinks(ctx context.Context, owner, kb string) ([]DocLink, error) {
|
||||
if p.db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var rows []DocLink
|
||||
err := p.db.WithContext(ctx).Where("owner = ? AND kb = ?", owner, kb).Find(&rows).Error
|
||||
return rows, err
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user