Files
sundynix-agentix/sundynix-gateway/internal/handler/admin.go
T
Blizzard 55d50417a9 feat: 模型健康/熔断态 surface 到管理端(T4.F 可观测)
failover/熔断的运行时态原来只在 dispatcher 日志、admin 看不到 —— 本次接到概览可见:
- harness: CircuitBreaker.Snapshot() 只读观测访问器(state + fails,不动状态机)
- llm: Pool.ModelHealth() 上报主备链每模型 {provider,model,role,state,fails};
  buildWithFallbacks 把模型名↔breaker 配对(同包直接读 failoverModel.breakers);
  newFailoverModel 改返回具体类型以便读 breakers
- dispatcher 心跳 payload 加 models[]
- gateway /admin/overview 独立超时 Ping dispatcher,合并进 models.health
- admin 概览「模型路由」新增「运行时链路态(实时)」:逐模型状态点
  (🟢在线/🔴熔断中+失败数/🟡半开探测/单点)
- 单测:Snapshot、Pool.ModelHealth(名字↔态配对/单点/空)
- live:配坏主→提交任务打熔断→概览显示 broken-demo「熔断中·失败3」,备用在线

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-06 12:00:13 +08:00

367 lines
13 KiB
Go

package handler
import (
"bytes"
"context"
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/sundynix/sundynix-gateway/internal/store"
"github.com/sundynix/sundynix-shared/contract"
"github.com/sundynix/sundynix-shared/prompts"
"github.com/sundynix/sundynix-shared/secrets"
)
// AuditList: GET /api/v1/admin/audit?limit=&offset= —— 敏感操作审计流(倒序,供运维溯源)。
func (h *Handler) AuditList(c *gin.Context) {
limit, offset := 50, 0
if v := c.Query("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil {
limit = n
}
}
if v := c.Query("offset"); v != "" {
if n, err := strconv.Atoi(v); err == nil {
offset = n
}
}
rows, err := h.db.ListAudit(c.Request.Context(), limit, offset)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
out := make([]gin.H, 0, len(rows))
for _, a := range rows {
out = append(out, gin.H{
"id": a.ID, "actor": a.Actor, "action": a.Action, "route": a.Route,
"path": a.Path, "status": a.Status, "ip": a.IP, "detail": a.Detail, "at": a.CreatedAt,
})
}
c.JSON(http.StatusOK, gin.H{"logs": out})
}
// GuardrailEvents: GET /api/v1/admin/guardrail-events?limit=&offset= —— 护栏命中安全事件流(倒序)。
func (h *Handler) GuardrailEvents(c *gin.Context) {
limit, offset := 50, 0
if v := c.Query("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil {
limit = n
}
}
if v := c.Query("offset"); v != "" {
if n, err := strconv.Atoi(v); err == nil {
offset = n
}
}
rows, err := h.db.ListGuardrailEvents(c.Request.Context(), limit, offset)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
out := make([]gin.H, 0, len(rows))
for _, e := range rows {
out = append(out, gin.H{
"id": e.ID, "actor": e.Actor, "kind": e.Kind, "reason": e.Reason,
"signals": e.Signals, "method": e.Method, "path": e.Path, "ip": e.IP, "at": e.CreatedAt,
})
}
c.JSON(http.StatusOK, gin.H{"events": out})
}
// AdminOverview: GET /api/v1/admin/overview —— 管理端系统级聚合(控制塔口径)。
// 区别于 stats/overview(桌面端个人工作台):这里一律系统级——全平台用户/任务/评测/
// 模型配置态/提示词控制面态/服务健康。Task/Eval 表无 owner 即全量;用户/KB/Doc 走全局计数。
func (h *Handler) AdminOverview(c *gin.Context) {
ctx := c.Request.Context()
ov := h.db.StatsOverview(ctx, "") // owner="" → 跳过个人 KB 口径,仅取全局任务/评测
users, kbs, docs := h.db.SystemCounts(ctx)
// 模型配置态:主模型 + 备用链数 + 各 kind 数量。
chat, _ := h.db.ListModels(ctx, "chat")
emb, _ := h.db.ListModels(ctx, "embedding")
activeChat, activeEmb, fallbacks := "", "", 0
for _, m := range chat {
if m.Active {
activeChat = m.Provider + "/" + m.Model
} else {
fallbacks++
}
}
for _, m := range emb {
if m.Active {
activeEmb = m.Provider + "/" + m.Model
}
}
// 提示词控制面态:受管 key 总数(代码内置)vs 已激活热覆盖数。
promptRows, _ := h.db.ListPrompts(ctx)
overrides := map[string]struct{}{}
for _, pr := range promptRows {
if pr.Active {
overrides[pr.Key] = struct{}{}
}
}
// 服务健康(与 Health / StatsOverview 同口径:本地可判 + milvus/neo4j 经 mcp-go)。
services := gin.H{"gateway": true, "nats": true, "db": h.db.Enabled(), "redis": h.cache.Enabled(), "milvus": false, "neo4j": false}
cctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
if res, err := h.bus.CallTool(cctx, contract.ToolSubjectGo("health"), &contract.ToolCall{Tool: "health"}); err == nil && res != nil && res.OK {
var sub map[string]bool
if json.Unmarshal([]byte(res.Content), &sub) == nil {
services["milvus"], services["neo4j"] = sub["milvus"], sub["neo4j"]
}
}
// 模型运行时健康态:Ping dispatcher 心跳取每模型 failover/熔断态(在线/熔断中/半开)。
// 权威配置来自 DB(上面 chat/emb),运行时态只有 dispatcher 知道 → 二者互补。
modelHealth := []gin.H{}
dctx, dcancel := context.WithTimeout(ctx, 2*time.Second) // 独立超时,不与 mcp-go 探活共用 cctx(避免被挤掉)
defer dcancel()
if data, err := h.bus.Ping(dctx, contract.SubjectHealthDispatcher); err == nil && len(data) > 0 {
var dh struct {
Models []struct {
Provider string `json:"provider"`
Model string `json:"model"`
Role string `json:"role"`
State string `json:"state"`
Fails int `json:"fails"`
} `json:"models"`
}
if json.Unmarshal(data, &dh) == nil {
for _, m := range dh.Models {
modelHealth = append(modelHealth, gin.H{
"provider": m.Provider, "model": m.Model, "role": m.Role, "state": m.State, "fails": m.Fails,
})
}
}
}
c.JSON(http.StatusOK, gin.H{
"users": users, "kb_count": kbs, "kb_docs": docs,
"tasks_today": ov.TasksToday, "tasks_total": ov.TasksTotal,
"status_count": ov.StatusCount, "task_trend": ov.TaskTrend,
"eval_avg": ov.EvalAvg, "faithful_avg": ov.FaithfulAvg, "eval_count": ov.EvalCount,
"models": gin.H{
"chat_count": len(chat), "embedding_count": len(emb),
"active_chat": activeChat, "active_embedding": activeEmb, "fallbacks": fallbacks,
"health": modelHealth, // 运行时每模型 failover/熔断态
},
"prompts": gin.H{"managed": len(prompts.Known), "overrides": len(overrides)},
"services": services,
"checked_at": time.Now().Format(time.RFC3339),
})
}
// maskPrefix 是脱敏展示用的占位前缀;前端把列表里的脱敏 key 原样回传即视为「未改动」。
const maskPrefix = "••••"
// existingModelKey 取某 id 模型库内存储的 api_key(密文,未解密);不存在返回空。
func (h *Handler) existingModelKey(ctx context.Context, id string) string {
if id == "" {
return ""
}
rows, _ := h.db.ListModels(ctx, "")
for _, m := range rows {
if m.ID == id {
return m.APIKey
}
}
return ""
}
// 控制面(运维管理):LLM 模型配置 CRUD + 测试连接 + 变更广播。
// 表 sundynix_model 由 Gateway 持有;Dispatcher 经 NATS 取激活配置。
type modelBody struct {
ID string `json:"id"`
Kind string `json:"kind"`
Provider string `json:"provider"`
BaseURL string `json:"base_url"`
APIKey string `json:"api_key"`
Model string `json:"model"`
}
// ListModels: GET /api/v1/admin/models?kind=chat|embedding —— 列出模型(api_key 脱敏)。
func (h *Handler) ListModels(c *gin.Context) {
rows, err := h.db.ListModels(c.Request.Context(), c.Query("kind"))
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
out := make([]gin.H, 0, len(rows))
for _, m := range rows {
plain, _ := secrets.Decrypt(m.APIKey) // 库内为密文,脱敏前先还原以展示真实尾 4 位
out = append(out, gin.H{
"id": m.ID, "kind": m.Kind, "provider": m.Provider, "base_url": m.BaseURL,
"model": m.Model, "active": m.Active, "api_key": mask(plain),
})
}
c.JSON(http.StatusOK, gin.H{"models": out})
}
// ListPricing: GET /api/v1/admin/pricing —— 列出各模型的计价配置(token↔真钱)。
func (h *Handler) ListPricing(c *gin.Context) {
rows, err := h.db.ListPricing(c.Request.Context())
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
out := make([]gin.H, 0, len(rows))
for _, p := range rows {
out = append(out, gin.H{
"model_id": p.ModelID, "input_per_1k": p.InputPer1K, "output_per_1k": p.OutputPer1K, "currency": p.Currency,
})
}
c.JSON(http.StatusOK, gin.H{"pricing": out})
}
// SavePricing: PUT /api/v1/admin/pricing —— 设置某模型的输入/输出单价(每 1K token)。
func (h *Handler) SavePricing(c *gin.Context) {
var b struct {
ModelID string `json:"model_id"`
InputPer1K float64 `json:"input_per_1k"`
OutputPer1K float64 `json:"output_per_1k"`
Currency string `json:"currency"`
}
if err := c.ShouldBindJSON(&b); err != nil || b.ModelID == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "model_id required"})
return
}
if b.InputPer1K < 0 || b.OutputPer1K < 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "单价不能为负"})
return
}
if b.Currency == "" {
b.Currency = "CNY"
}
if err := h.db.UpsertPricing(c.Request.Context(), b.ModelID, b.InputPer1K, b.OutputPer1K, b.Currency); err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
// SaveModel: POST /api/v1/admin/models —— 新增/更新一条模型配置。
func (h *Handler) SaveModel(c *gin.Context) {
var b modelBody
if err := c.ShouldBindJSON(&b); err != nil || b.BaseURL == "" || b.Model == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "provider/base_url/model required"})
return
}
provider := b.Provider
if provider == "" {
provider = "openai-compatible"
}
kind := b.Kind
if kind == "" {
kind = contract.ConfigKindChat
}
// api_key 处理:空或脱敏占位 => 沿用库内既有密文(更新时未改 key);否则视为新明文,加密落库。
apiKey := b.APIKey
if apiKey == "" || strings.HasPrefix(apiKey, maskPrefix) {
apiKey = h.existingModelKey(c.Request.Context(), b.ID) // 已是密文,原样保留(不存在则空)
} else {
enc, err := secrets.Encrypt(apiKey)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "encrypt api_key: " + err.Error()})
return
}
apiKey = enc
}
m := &store.LLMModel{BaseModel: store.BaseModel{ID: b.ID}, Kind: kind, Provider: provider, BaseURL: b.BaseURL, APIKey: apiKey, Model: b.Model}
if err := h.db.SaveModel(c.Request.Context(), m); err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
h.broadcastActive(c.Request.Context())
c.JSON(http.StatusOK, gin.H{"id": m.ID})
}
// SetActiveModel: POST /api/v1/admin/models/:id/active —— 设为激活并广播。
func (h *Handler) SetActiveModel(c *gin.Context) {
id := c.Param("id")
if err := h.db.SetActiveModel(c.Request.Context(), id); err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
h.broadcastActive(c.Request.Context())
c.JSON(http.StatusOK, gin.H{"status": "ok", "active": id})
}
// DeleteModel: DELETE /api/v1/admin/models/:id
func (h *Handler) DeleteModel(c *gin.Context) {
id := c.Param("id")
if err := h.db.DeleteModel(c.Request.Context(), id); err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
return
}
h.broadcastActive(c.Request.Context())
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
// TestModel: POST /api/v1/admin/models/test —— 探测 OpenAI 兼容端点连通性。
func (h *Handler) TestModel(c *gin.Context) {
var b modelBody
if err := c.ShouldBindJSON(&b); err != nil || b.BaseURL == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "base_url required"})
return
}
// 若未带 key(或回传脱敏占位),用库里的真实 key。
key := b.APIKey
if key == "" || strings.HasPrefix(key, maskPrefix) {
key = h.existingModelKey(c.Request.Context(), b.ID)
}
// key 此刻可能是库内密文,也可能是用户新填的明文;Decrypt 对无前缀明文透传,两种都还原成可用明文。
if plain, err := secrets.Decrypt(key); err == nil {
key = plain
}
ctx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Second)
defer cancel()
var req *http.Request
if b.Kind == contract.ConfigKindEmbedding {
// embedding 端点多无 /models,发一个最小 /embeddings 探测。
payload, _ := json.Marshal(map[string]any{"model": b.Model, "input": []string{"ping"}})
req, _ = http.NewRequestWithContext(ctx, http.MethodPost, b.BaseURL+"/embeddings", bytes.NewReader(payload))
req.Header.Set("Content-Type", "application/json")
} else {
req, _ = http.NewRequestWithContext(ctx, http.MethodGet, b.BaseURL+"/models", nil)
}
if key != "" {
req.Header.Set("Authorization", "Bearer "+key)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
c.JSON(http.StatusOK, gin.H{"ok": false, "message": err.Error()})
return
}
defer resp.Body.Close()
c.JSON(http.StatusOK, gin.H{"ok": resp.StatusCode < 400, "message": "HTTP " + resp.Status})
}
// broadcastActive 重新广播各 kind 当前激活配置,触发对应消费方热更新。
// chat 配置带 Fallbacks(其它已登记 chat 模型作备用),dispatcher 据此重建 failover 链。
func (h *Handler) broadcastActive(ctx context.Context) {
for _, kind := range []string{contract.ConfigKindChat, contract.ConfigKindEmbedding} {
if cfg := h.db.ActiveConfig(ctx, kind); cfg != nil {
_ = h.bus.PublishConfigUpdated(kind, cfg)
}
}
}
func mask(s string) string {
if len(s) <= 4 {
if s == "" {
return ""
}
return "••••"
}
return "••••" + s[len(s)-4:]
}