feat(gateway): 入库队列接线 + 正文一律MinIO + file_id级联删端点
- 入库改走 JetStream 队列:claim-check 暂存 MinIO → 发作业 → 立即返 job_id; worker 池(StartIngestWorkers)有界并发消费,崩溃重投续跑(幂等),优雅 drain - 存储:正文一律落 MinIO(去 <8000字内联PG 阈值,仅 MinIO 挂时回退兜底); sundynix_doc 删死字段 MD5 - 下游键传稳定 file_id(非展示名);新增 DELETE /api/v1/kb/doc 级联删 (三库 + MinIO 原文 + PG 元数据/双链,owner 隔离) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/sundynix/sundynix-gateway/internal/blob"
|
||||
"github.com/sundynix/sundynix-gateway/internal/nats"
|
||||
"github.com/sundynix/sundynix-gateway/internal/handler"
|
||||
"github.com/sundynix/sundynix-gateway/internal/router"
|
||||
"github.com/sundynix/sundynix-gateway/internal/store"
|
||||
sharedbus "github.com/sundynix/sundynix-shared/bus"
|
||||
@@ -94,6 +95,14 @@ func main() {
|
||||
log.Printf("[gateway] subscribe usage: %v", err)
|
||||
}
|
||||
|
||||
// 入库工作队列:启动有界并发 worker 池消费 JetStream 入库作业(背压防 OOM、崩溃重投兜底)。
|
||||
ingestDrain, err := handler.StartIngestWorkers(context.Background(), db, cache, bus, blobStore)
|
||||
if err != nil {
|
||||
log.Printf("[gateway] 启动入库 worker 失败(入库不可用): %v", err)
|
||||
} else {
|
||||
log.Printf("[gateway] 入库 worker 池就绪(JetStream 持久队列)")
|
||||
}
|
||||
|
||||
r := router.New(db, cache, bus, blobStore)
|
||||
addr := envOr("GATEWAY_ADDR", ":8080")
|
||||
srv := &http.Server{Addr: addr, Handler: r}
|
||||
@@ -118,6 +127,13 @@ func main() {
|
||||
if err := srv.Shutdown(sctx); err != nil {
|
||||
log.Printf("[gateway] HTTP 优雅关闭超时/出错: %v", err)
|
||||
}
|
||||
// HTTP 排空后再 drain 在途入库作业(≤to);超时未完的不丢——AckWait 30min 内重启会重投续跑。
|
||||
if ingestDrain != nil {
|
||||
log.Printf("[gateway] drain 在途入库作业 ≤%s(超时未完者重启后由队列重投)…", to)
|
||||
ictx, icancel := context.WithTimeout(context.Background(), to)
|
||||
ingestDrain(ictx)
|
||||
icancel()
|
||||
}
|
||||
log.Println("[gateway] 已优雅停机")
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user