feat: add multi-source breed encyclopedia and two-hour refresh sessions
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# 犬种目录与百科
|
||||
|
||||
## 数据结构
|
||||
|
||||
- `sundynix_breed`:UUID 主键、中文名、英文名、别名、拼音与首字母、启用状态;起源地、FCI 编号和分组、肩高、体重、寿命、介绍、性格、运动、毛发护理、相处笔记、指数。`field_sources` 使用 PostgreSQL JSONB 保存展示字段的来源机构。
|
||||
- `sundynix_breed_reference`:UUID 主键,`breed_id` 关联犬种;机构、机构内编号、原始页面链接、核对日期、事实字段(JSONB)。每个犬种与机构唯一。参考尺寸和指数按机构分别保存,不合并为统一标准。
|
||||
- `sundynix_pet.breed_id`:可空。选目录品种时关联 UUID;手动填写时仅保留 `breed` 文本。
|
||||
- `editorial_locked`:管理员保存资料后锁定展示内容,重复导入不会覆盖人工修改或清空的字段。来源资料只读,导入仅在其事实内容发生变化时更新。
|
||||
|
||||
## 展示
|
||||
|
||||
管理端 `/admin/breeds` 支持每页 10 条分页、名称/别名/拼音搜索、A–Z 筛选、启停与新增编辑。编辑窗切换“展示资料 / 官方来源”,可以对照各机构标准与评分。展示字段可手动完善,缺失字段留空。
|
||||
|
||||
小程序“知识”提供搜索、中文拼音 A–Z 筛选和详情;详情分为基本信息、性格指数、散步与护理、资料出处,字段注明来源,引用链接可复制。所有文案为简短中文整理,不保留来源长篇文章或图片。品种选择组件复用于新增/编辑档案,也允许手填。
|
||||
|
||||
## 采集与导入
|
||||
|
||||
2026-10-05:CKU 公开列表 348 项,按中文名称去重后 347 项。补充中华田园犬、混种犬、贵宾犬三项实用分类,合计 350 项。
|
||||
|
||||
- CKU:https://www.cku.org.cn/oa-web/api/v1/open/getFciDogAll
|
||||
88 个详情接口有完整资料,整理基本事实与简短性格标签。
|
||||
- FCI:https://www.fci.be/en/nomenclature/
|
||||
使用编号准确关联目录,保存官方名称、原产地和分组;变种保留自己的中文档案,来源标明基础标准范围。分组页面归属国字段用于起源地参考,尺寸不从目录推测。
|
||||
- AKC:https://www.akc.org/dog-breeds/
|
||||
只使用准确英文名或已核对的等价名称匹配,提取结构化尺寸、寿命和指数。尺寸展示统一换算 cm/kg,原始单位保留在来源资料。变种不套用父犬种的 AKC 展示数据;父犬种参考资料明确标注标准范围。
|
||||
- 英国皇家犬业俱乐部:https://www.royalkennelclub.com/breeds-a-z
|
||||
本机直连超时,先从官方页面核对 8 个犬种的运动、护理和寿命信息,记录于 `rkc_facts.json`。
|
||||
|
||||
`data.json` 保存中文目录与已有摘要;`enrichment.json` 保存多来源的事实快照和展示补充。`rkc_facts.json` 是人工核对的事实记录。
|
||||
|
||||
从仓库根目录执行 `python3 pets-be/scripts/collect_breed_facts.py` 可重新生成补充数据。采集脚本仅用 Python 标准库,缓存置于 `/private/tmp/pawprint-breed-sources`;机构连接失败不伪造数据。
|
||||
|
||||
服务器迁移时自动导入;也可在 `pets-be` 目录执行 `go run ./cmd/seedbreeds`。内容无变化时跳过资料更新。导入不会恢复管理员改名或停用的品种,人工编辑后的展示内容不会被刷新覆盖。
|
||||
@@ -0,0 +1,102 @@
|
||||
package breeds
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/mozillazg/go-pinyin"
|
||||
"github.com/sundynix/pets-be/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
//go:embed data.json
|
||||
var data []byte
|
||||
|
||||
// SortFields provides both full pinyin and initials, retaining Latin breed names.
|
||||
func SortFields(name string) (string, string, string) {
|
||||
args := pinyin.NewArgs()
|
||||
args.Fallback = func(r rune, _ pinyin.Args) []string {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
return []string{strings.ToLower(string(r))}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
words := pinyin.LazyPinyin(name, args)
|
||||
// Breed-name phrases with common alternative pronunciations.
|
||||
if strings.HasPrefix(name, "重庆") && len(words) > 0 {
|
||||
words[0] = "chong"
|
||||
}
|
||||
if strings.HasPrefix(name, "长毛") && len(words) > 0 {
|
||||
words[0] = "chang"
|
||||
}
|
||||
full := strings.Join(words, "")
|
||||
var initials strings.Builder
|
||||
for _, word := range words {
|
||||
if word != "" {
|
||||
initials.WriteByte(word[0])
|
||||
}
|
||||
}
|
||||
initial := "#"
|
||||
if len(full) > 0 && full[0] >= 'a' && full[0] <= 'z' {
|
||||
initial = strings.ToUpper(full[:1])
|
||||
}
|
||||
return full, initials.String(), initial
|
||||
}
|
||||
|
||||
func Defaults() ([]model.Breed, error) {
|
||||
var rows []struct {
|
||||
Origin string `json:"origin"`
|
||||
BreedGroup string `json:"breed_group"`
|
||||
FCINumber string `json:"fci_number"`
|
||||
EnglishName string `json:"english_name"`
|
||||
Height string `json:"height"`
|
||||
Weight string `json:"weight"`
|
||||
Summary string `json:"summary"`
|
||||
Temperament string `json:"temperament"`
|
||||
Traits string `json:"traits"`
|
||||
CareNotes string `json:"care_notes"`
|
||||
EncyclopediaUpdatedAt string `json:"encyclopedia_updated_at"`
|
||||
Name string `json:"name"`
|
||||
Aliases string `json:"aliases"`
|
||||
Source string `json:"source"`
|
||||
SourceKey string `json:"source_key"`
|
||||
SourceURL string `json:"source_url"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]model.Breed, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
full, short, initial := SortFields(row.Name)
|
||||
key := row.SourceKey
|
||||
result = append(result, model.Breed{Origin: row.Origin, BreedGroup: row.BreedGroup, FCINumber: row.FCINumber, EnglishName: row.EnglishName, Height: row.Height, Weight: row.Weight, Summary: row.Summary, Temperament: row.Temperament, Traits: row.Traits, CareNotes: row.CareNotes, EncyclopediaUpdatedAt: row.EncyclopediaUpdatedAt, Name: row.Name, Aliases: row.Aliases, Pinyin: full, PinyinInitials: short, Initial: initial, Enabled: true, Source: row.Source, SourceKey: &key, SourceURL: row.SourceURL})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Existing entries (including edits and disabled breeds) are never overwritten.
|
||||
func Seed(db *gorm.DB) (int64, error) {
|
||||
rows, err := Defaults()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("load breeds: %w", err)
|
||||
}
|
||||
result := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(rows, 100)
|
||||
if result.Error != nil {
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
// Backfill newly introduced encyclopedia fields exactly once on untouched seed rows.
|
||||
// Future starts never overwrite administrator changes.
|
||||
for _, row := range rows {
|
||||
if row.Summary == "" {
|
||||
continue
|
||||
}
|
||||
if err := db.Model(&model.Breed{}).Where("source_key = ? AND name = ? AND (encyclopedia_updated_at IS NULL OR encyclopedia_updated_at = '') AND updated_at = created_at", row.SourceKey, row.Name).Updates(map[string]any{"origin": row.Origin, "breed_group": row.BreedGroup, "fci_number": row.FCINumber, "english_name": row.EnglishName, "height": row.Height, "weight": row.Weight, "summary": row.Summary, "temperament": row.Temperament, "traits": row.Traits, "care_notes": row.CareNotes, "encyclopedia_updated_at": row.EncyclopediaUpdatedAt}).Error; err != nil {
|
||||
return result.RowsAffected, err
|
||||
}
|
||||
}
|
||||
return result.RowsAffected, enrich(db)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package breeds
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCatalogAndPinyin(t *testing.T) {
|
||||
rows, err := Defaults()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 350 {
|
||||
t.Fatalf("catalog count %d", len(rows))
|
||||
}
|
||||
names, sources := map[string]bool{}, map[string]bool{}
|
||||
for _, row := range rows {
|
||||
if names[row.Name] || row.SourceKey == nil || sources[*row.SourceKey] || row.Pinyin == "" || row.Initial == "" {
|
||||
t.Fatalf("invalid catalog row: %s", row.Name)
|
||||
}
|
||||
names[row.Name] = true
|
||||
sources[*row.SourceKey] = true
|
||||
}
|
||||
for name, want := range map[string]string{"柯基": "K", "重庆犬": "C", "长毛吉娃娃": "C", "中华田园犬": "Z", "Border Collie": "B"} {
|
||||
full, short, initial := SortFields(name)
|
||||
if initial != want || full == "" || short == "" {
|
||||
t.Fatalf("%s: %s/%s/%s", name, full, short, initial)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfficialSources(t *testing.T) {
|
||||
rows, err := Enrichments()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, row := range rows {
|
||||
seen := map[string]bool{}
|
||||
for _, ref := range row.References {
|
||||
if seen[ref.Provider] || len(ref.Facts) == 0 || ref.URL == "" || ref.RetrievedAt == "" {
|
||||
t.Fatalf("invalid reference for %s", row.Name)
|
||||
}
|
||||
seen[ref.Provider] = true
|
||||
counts[ref.Provider]++
|
||||
if ref.Provider == "fci" && "cku:"+ref.ExternalID != strings.TrimRight(row.SourceKey, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") {
|
||||
t.Fatalf("wrong FCI mapping %s/%s", row.SourceKey, ref.ExternalID)
|
||||
}
|
||||
}
|
||||
for key, provider := range row.FieldSources {
|
||||
if provider != "editorial" && !seen[provider] {
|
||||
t.Fatalf("%s %s has missing source %s", row.Name, key, provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rows) != 350 || counts["fci"] != 347 || counts["akc"] < 160 || counts["rkc"] != 8 {
|
||||
t.Fatalf("incomplete import: %v", counts)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,100 @@
|
||||
package breeds
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/sundynix/pets-be/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
//go:embed enrichment.json
|
||||
var enrichmentData []byte
|
||||
|
||||
type Enrichment struct {
|
||||
SourceKey string `json:"source_key"`
|
||||
Name string `json:"name"`
|
||||
Fields map[string]string `json:"fields"`
|
||||
FieldSources map[string]string `json:"field_sources"`
|
||||
References []model.BreedReference `json:"references"`
|
||||
}
|
||||
|
||||
func Enrichments() ([]Enrichment, error) {
|
||||
var rows []Enrichment
|
||||
err := json.Unmarshal(enrichmentData, &rows)
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// Reference facts remain available even when an editor locks the display content.
|
||||
func enrich(db *gorm.DB) error {
|
||||
rows, err := Enrichments()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load breed sources: %w", err)
|
||||
}
|
||||
defaults, err := Defaults()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
baseline := map[string]model.Breed{}
|
||||
for _, breed := range defaults {
|
||||
baseline[*breed.SourceKey] = breed
|
||||
}
|
||||
for _, item := range rows {
|
||||
var breed model.Breed
|
||||
if err := db.Where("source_key = ?", item.SourceKey).First(&breed).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, source := range item.References {
|
||||
source.BreedID = breed.ID
|
||||
// Source snapshots are read-only in admin; update only when factual metadata changes.
|
||||
if err := db.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "breed_id"}, {Name: "provider"}}, DoUpdates: clause.AssignmentColumns([]string{"external_id", "url", "retrieved_at", "facts"}), Where: clause.Where{Exprs: []clause.Expression{clause.Expr{SQL: "sundynix_breed_reference.facts IS DISTINCT FROM EXCLUDED.facts OR sundynix_breed_reference.url IS DISTINCT FROM EXCLUDED.url OR sundynix_breed_reference.retrieved_at IS DISTINCT FROM EXCLUDED.retrieved_at"}}}}).Create(&source).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if breed.EditorialLocked || breed.Name != item.Name {
|
||||
continue
|
||||
}
|
||||
original := baseline[item.SourceKey]
|
||||
updates := map[string]any{"english_name": original.EnglishName, "origin": original.Origin, "breed_group": original.BreedGroup, "fci_number": original.FCINumber, "height": original.Height, "weight": original.Weight, "summary": original.Summary, "temperament": original.Temperament, "traits": original.Traits, "care_notes": original.CareNotes, "life_span": "", "exercise": "", "grooming": ""}
|
||||
for field, value := range item.Fields {
|
||||
switch field {
|
||||
case "english_name", "origin", "breed_group", "fci_number", "height", "weight", "life_span", "exercise", "grooming", "temperament", "traits", "summary", "care_notes":
|
||||
updates[field] = value
|
||||
default:
|
||||
return fmt.Errorf("unsupported breed field: %s", field)
|
||||
}
|
||||
}
|
||||
origins, err := json.Marshal(item.FieldSources)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updates["field_sources"] = string(origins)
|
||||
latest := original.EncyclopediaUpdatedAt
|
||||
for _, ref := range item.References {
|
||||
if ref.RetrievedAt > latest {
|
||||
latest = ref.RetrievedAt
|
||||
}
|
||||
}
|
||||
updates["encyclopedia_updated_at"] = latest
|
||||
// Skip unchanged columns so ordinary starts do not rewrite every breed.
|
||||
current := map[string]string{"english_name": breed.EnglishName, "origin": breed.Origin, "breed_group": breed.BreedGroup, "fci_number": breed.FCINumber, "height": breed.Height, "weight": breed.Weight, "life_span": breed.LifeSpan, "exercise": breed.Exercise, "grooming": breed.Grooming, "temperament": breed.Temperament, "traits": breed.Traits, "summary": breed.Summary, "care_notes": breed.CareNotes, "encyclopedia_updated_at": breed.EncyclopediaUpdatedAt}
|
||||
for key, value := range updates {
|
||||
if key != "field_sources" && current[key] == value {
|
||||
delete(updates, key)
|
||||
}
|
||||
}
|
||||
if reflect.DeepEqual(breed.FieldSources, item.FieldSources) {
|
||||
delete(updates, "field_sources")
|
||||
}
|
||||
if len(updates) == 0 {
|
||||
continue
|
||||
}
|
||||
// UpdateColumns avoids advancing UpdatedAt on each application start.
|
||||
if err := db.Model(&breed).Where("editorial_locked = false AND name = ?", item.Name).UpdateColumns(updates).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,114 @@
|
||||
[
|
||||
{
|
||||
"source_key": "cku:39",
|
||||
"external_id": "welsh-corgi-pembroke",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/pastoral/welsh-corgi-pembroke/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每周梳理超过一次",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "短毛",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:38",
|
||||
"external_id": "welsh-corgi-cardigan",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/pastoral/welsh-corgi-cardigan/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每周梳理超过一次",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "短毛",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:86",
|
||||
"external_id": "yorkshire-terrier",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/toy/yorkshire-terrier/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 30 分钟",
|
||||
"grooming": "每天梳理",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "长毛",
|
||||
"sheds": "掉毛较少"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:162",
|
||||
"external_id": "whippet",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/hound/whippet/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每周梳理一次",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "短毛",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:209",
|
||||
"external_id": "tibetan-terrier",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/utility/tibetan-terrier/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每天梳理",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "中型",
|
||||
"coat_length": "长毛",
|
||||
"sheds": "掉毛较少"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:231",
|
||||
"external_id": "tibetan-spaniel",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/utility/tibetan-spaniel/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每周梳理超过一次",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "中等毛长",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:85",
|
||||
"external_id": "west-highland-white-terrier",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/terrier/west-highland-white-terrier/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:不超过 1 小时",
|
||||
"grooming": "每周梳理超过一次",
|
||||
"life_span": "通常超过 12 年",
|
||||
"size": "小型",
|
||||
"coat_length": "中等毛长",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
},
|
||||
{
|
||||
"source_key": "cku:347",
|
||||
"external_id": "white-swiss-shepherd-dog-imp",
|
||||
"url": "https://www.royalkennelclub.com/search/breeds-a-to-z/breeds/pastoral/white-swiss-shepherd-dog-imp/",
|
||||
"facts": {
|
||||
"exercise": "每日运动量参考:超过 2 小时",
|
||||
"grooming": "每周梳理超过一次",
|
||||
"life_span": "通常超过 10 年",
|
||||
"size": "大型",
|
||||
"coat_length": "中长毛",
|
||||
"sheds": "会掉毛"
|
||||
},
|
||||
"retrieved_at": "2026-10-05"
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user