feat(gateway): 入库队列接线 + 正文一律MinIO + file_id级联删端点
- 入库改走 JetStream 队列:claim-check 暂存 MinIO → 发作业 → 立即返 job_id; worker 池(StartIngestWorkers)有界并发消费,崩溃重投续跑(幂等),优雅 drain - 存储:正文一律落 MinIO(去 <8000字内联PG 阈值,仅 MinIO 挂时回退兜底); sundynix_doc 删死字段 MD5 - 下游键传稳定 file_id(非展示名);新增 DELETE /api/v1/kb/doc 级联删 (三库 + MinIO 原文 + PG 元数据/双链,owner 隔离) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -24,7 +24,10 @@ func MustConnect(url string) *Bus {
|
||||
if err := inner.EnsureTaskStream(context.Background()); err != nil {
|
||||
log.Fatalf("[nats] ensure stream: %v", err)
|
||||
}
|
||||
log.Printf("[nats] connected %s, task stream ready", url)
|
||||
if err := inner.EnsureIngestStream(context.Background()); err != nil {
|
||||
log.Fatalf("[nats] ensure ingest stream: %v", err)
|
||||
}
|
||||
log.Printf("[nats] connected %s, task + ingest streams ready", url)
|
||||
return &Bus{inner: inner}
|
||||
}
|
||||
|
||||
@@ -101,4 +104,19 @@ func (b *Bus) PublishIngest(jobID string, ev *contract.IngestEvent) error {
|
||||
// CompleteStream 发送入库流结束信号。
|
||||
func (b *Bus) CompleteStream(jobID string) error { return b.inner.CompleteStream(jobID) }
|
||||
|
||||
// ---- 入库工作队列(JetStream 持久)----
|
||||
|
||||
// EnsureIngestStream 确保入库作业流存在(启动时调)。
|
||||
func (b *Bus) EnsureIngestStream(ctx context.Context) error { return b.inner.EnsureIngestStream(ctx) }
|
||||
|
||||
// PublishIngestJob 把入库作业持久入队(崩溃不丢,由 worker 消费)。
|
||||
func (b *Bus) PublishIngestJob(ctx context.Context, job *contract.IngestJob) error {
|
||||
return b.inner.PublishIngestJob(ctx, job)
|
||||
}
|
||||
|
||||
// ConsumeIngestJobs 启动入库 worker 池消费作业(有界并发 + 背压 + 崩溃重投)。
|
||||
func (b *Bus) ConsumeIngestJobs(ctx context.Context, h sharedbus.IngestHandler) (func(context.Context), error) {
|
||||
return b.inner.ConsumeIngestJobs(ctx, h)
|
||||
}
|
||||
|
||||
func (b *Bus) Close() { b.inner.Close() }
|
||||
|
||||
Reference in New Issue
Block a user