Compare commits

..
2 Commits
Author SHA1 Message Date
Blizzard 825d049e32 Merge pull request 'ci: 复用服务器构建缓存与 HTTPS 登录配置' (#2) from dev into master
Veyra CI/CD / verify (push) Successful in 8m30s
Veyra CI/CD / deploy (push) Successful in 1m34s
2026-10-09 13:12:09 +00:00
Blizzard ac936d2a1a ci: 复用部署服务器缓存并支持 HTTPS 同源登录 2026-10-09 21:11:44 +08:00
14 changed files with 158 additions and 47 deletions
+22 -43
View File
@@ -4,6 +4,8 @@ on:
branches: [master]
env:
REMOTE_DIR: /home/workspace/sundynix-veyra
DEPLOY_HOST: 192.168.100.132
DEPLOY_USER: root
jobs:
verify:
runs-on: ubuntu-latest
@@ -18,14 +20,28 @@ jobs:
curl -fLsS -H "Authorization: token ${GITEA_TOKEN}" \
"https://git.sundynix.cn/${REPOSITORY}/archive/${COMMIT_SHA}.tar.gz" \
| tar -xz --strip-components=1
- name: Check App, embedded admin, Go and PostgreSQL integration
run: bash deploy/check.sh
- name: Install SSH tools
run: |
set -eu
if command -v sshpass >/dev/null && command -v ssh >/dev/null; then exit 0; fi
if command -v apk >/dev/null; then
apk add --no-cache sshpass openssh-client
else
apt-get update -qq
apt-get install -y -qq --no-install-recommends sshpass openssh-client
fi
- name: Verify and build on cached deployment host
env:
SSHPASS: ${{ secrets.DEPLOY_PASSWORD }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
COMMIT_SHA: ${{ github.sha }}
run: bash deploy/ci-ssh.sh verify
deploy:
needs: verify
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Checkout verified commit from Gitea
- name: Checkout exact commit from Gitea
env:
GITEA_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
@@ -35,56 +51,19 @@ jobs:
curl -fLsS -H "Authorization: token ${GITEA_TOKEN}" \
"https://git.sundynix.cn/${REPOSITORY}/archive/${COMMIT_SHA}.tar.gz" \
| tar -xz --strip-components=1
- name: Build and package immutable image
env:
COMMIT_SHA: ${{ github.sha }}
run: |
set -euo pipefail
docker build -t "sundynix/veyra:${COMMIT_SHA}" .
docker save "sundynix/veyra:${COMMIT_SHA}" | gzip > image.tar.gz
sha256sum image.tar.gz > image.tar.gz.sha256
- name: Install SSH tools
run: |
set -eu
if command -v sshpass >/dev/null && command -v ssh >/dev/null; then exit 0; fi
if command -v apk >/dev/null; then
apk add --no-cache sshpass openssh-client
elif command -v apt-get >/dev/null; then
else
apt-get update -qq
apt-get install -y -qq --no-install-recommends sshpass openssh-client
else
echo 'Runner requires ssh and sshpass' >&2
exit 1
fi
- name: Transfer and deploy
- name: Deploy verified image
env:
SSHPASS: ${{ secrets.DEPLOY_PASSWORD }}
DEPLOY_HOST: 192.168.100.132
DEPLOY_USER: root
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
VEYRA_ENV_FILE: ${{ secrets.VEYRA_ENV_FILE }}
COMMIT_SHA: ${{ github.sha }}
run: |
set -euo pipefail
: "${SSHPASS:?DEPLOY_PASSWORD is required}"
: "${DEPLOY_HOST:?DEPLOY_HOST is required}"
: "${DEPLOY_USER:?DEPLOY_USER is required}"
: "${DEPLOY_KNOWN_HOSTS:?DEPLOY_KNOWN_HOSTS is required}"
[[ "$DEPLOY_HOST" =~ ^[a-zA-Z0-9.-]+$ && "$DEPLOY_USER" =~ ^[a-zA-Z0-9_-]+$ ]] || exit 2
[[ "$COMMIT_SHA" =~ ^[0-9a-f]{40}$ ]] || exit 2
known_hosts="$(mktemp)"
trap 'rm -f "$known_hosts"' EXIT
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$known_hosts"
chmod 600 "$known_hosts"
options=(-o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts" -o ConnectTimeout=15)
target="$DEPLOY_USER@$DEPLOY_HOST"
release="$REMOTE_DIR/releases/$COMMIT_SHA"
sshpass -e ssh "${options[@]}" "$target" "umask 077; mkdir -p '$release' '$REMOTE_DIR/configs'"
sshpass -e scp "${options[@]}" deploy/bootstrap-config.sh "$target:$release/"
# Bootstrap once via secret; never replace an existing server configuration.
if [[ -n "$VEYRA_ENV_FILE" ]]; then
printf '%s\n' "$VEYRA_ENV_FILE" | sshpass -e ssh "${options[@]}" "$target" \
"bash '$release/bootstrap-config.sh' '$REMOTE_DIR'"
fi
sshpass -e scp "${options[@]}" image.tar.gz image.tar.gz.sha256 deploy/docker-compose.yml deploy/release.sh "$target:$release/"
sshpass -e ssh "${options[@]}" "$target" "bash '$release/release.sh' '$REMOTE_DIR' '$COMMIT_SHA'"
run: bash deploy/ci-ssh.sh deploy
-1
View File
@@ -1,4 +1,3 @@
# syntax=docker/dockerfile:1
FROM node:22-bookworm-slim AS admin
WORKDIR /src/web/admin
COPY web/admin/package*.json ./
+3
View File
@@ -30,3 +30,6 @@ LOG_MAX_AGE_DAYS=14
# Python 3,用于同进程 Worker 调用隔离的研究引擎。
PYTHON_EXECUTABLE=python3
# HTTPS reverse proxy public origin, without trailing slash. Blank for local development.
PUBLIC_ORIGIN=
+6 -2
View File
@@ -10,7 +10,7 @@
- 应用配置:服务器 `configs/.env`,权限 600;日志:服务器 `logs/`,由容器 UID 10001 写入并轮转。
- PostgreSQL、Redis 使用已有内网服务;不额外部署数据库容器,不覆盖数据、身份或加密主密钥。
仓库 Actions 已启用,部署使用 `DEPLOY_PASSWORD`、`DEPLOY_KNOWN_HOSTS` 两个仓库 Secret。服务器地址、用户和目录在工作流中明确配置;SSH 严格校验已知服务器公钥。可选 `VEYRA_ENV_FILE` 仅用于新服务器首次创建配置,已有文件永不覆盖。当前服务器已通过 SSH 安全初始化,无需设置此可选 Secret。
仓库 Actions 已启用,部署使用 `DEPLOY_PASSWORD`、`DEPLOY_KNOWN_HOSTS` 两个仓库 Secret。服务器地址、用户和目录在工作流中明确配置;SSH 严格校验已知服务器公钥。服务器配置需在首次部署前初始化;当前服务器已通过 SSH 安全初始化。发布不传输或覆盖配置。
应用环境文件必须采用无外层引号的 `KEY=VALUE` 格式。Compose 使用 `env_file.format: raw`,保留密码中的 `$`、`#` 等字符,不把应用配置作为 Compose 插值文件。需要 Docker、Compose >=2.30、bash、flock、gzip、sha256sum;当前服务器 Compose 2.36.2 满足要求。
@@ -18,7 +18,7 @@
1. 使用 Gitea job token 下载当前提交归档,与参考项目保持一致,避免 Git HTTP pack 检出问题。
2. Docker 内检查 App 类型、12 项客户端测试、iOS/Android Hermes 导出;管理端类型检查、构建、Go embed;gofmt、go vet 与 Go race 测试。构建阶段使用已验证可达的 `https://goproxy.cn,direct` 下载 Go 依赖,保留默认 checksum 校验;运行镜像不包含此构建设置。PostgreSQL 集成测试使用独立、用后清理的 CI 容器,不访问开发库。部署脚本也有成功、校验失败、迁移失败、健康失败回滚、配置保留测试。
3. 所有检查通过才构建 `sundynix/veyra:<完整提交 SHA>`,打包压缩并生成 SHA-256 校验文件,通过 SSH 传入服务器的独立发布目录。
3. Runner 只传提交源码,检查和构建在部署服务器独立 builds/<SHA> 目录中执行,复用服务器 Docker 依赖层缓存;使用独立构建锁避免并发资源争用。所有检查通过才构建 `sundynix/veyra:<完整提交 SHA>`,在服务器本地打包并生成 SHA-256 校验文件。不在 Runner 重复下载基础镜像,也不跨主机传输大型镜像。
4. 服务器加锁串行发布,校验压缩包、加载镜像,执行同一可执行文件的 `-command db-create` 和 `-command db-migrate`,再启动 Compose。
5. `/readyz` 检查 PostgreSQL 与 Redis。新版本就绪后才更新 `.current-image`;失败则恢复上一版镜像与 Compose,首次部署失败则停止应用。迁移失败不替换运行中的服务。不会自动回退数据库迁移;后续迁移应保持与上一版应用兼容,破坏性迁移需单独设计。
@@ -39,3 +39,7 @@ go build -trimpath -o bin/veyra .
服务器排查使用 `docker compose --env-file /dev/null -f docker-compose.yml`,同时将 `VEYRA_IMAGE` 设为 `.current-image` 中的值。不要将含凭据的 `docker compose config`、环境文件或容器完整 inspect 输出到日志;配置语法检查使用 `config --quiet`。
配置语义参考 [Gitea Actions](https://docs.gitea.com/usage/actions/) 与 [Docker Compose env_file](https://docs.docker.com/reference/compose-file/services/#env_file)。
## HTTPS 反向代理
域名 `veyra.sundynix.cn` 的 Nginx 将全部页面与 API 代理至 `http://192.168.100.132:8080`,传递原始 Host。服务器 configs/.env 设置 `PUBLIC_ORIGIN=https://veyra.sundynix.cn`,使 HTTPS Origin 校验和 Secure Cookie 正确工作;应用不信任任意 X-Forwarded-Proto。域名 Nginx 由用户按已提供配置操作。
+1
View File
@@ -22,5 +22,6 @@ for ((i=0; i<60; i++)); do
done
if [[ "$ready" != true ]]; then echo 'CI PostgreSQL did not become ready' >&2; exit 1; fi
docker run --rm --network "$network" \
-v veyra-go-test-cache:/root/.cache/go-build \
-e VEYRA_TEST_DATABASE_URL='postgres://veyra_test:test_only@postgres:5432/veyra_test?sslmode=disable' \
"veyra-test:$run_id"
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
: "${SSHPASS:?DEPLOY_PASSWORD is required}"
: "${DEPLOY_KNOWN_HOSTS:?DEPLOY_KNOWN_HOSTS is required}"
[[ "$DEPLOY_HOST" =~ ^[a-zA-Z0-9.-]+$ && "$DEPLOY_USER" =~ ^[a-zA-Z0-9_-]+$ ]] || exit 2
[[ "$COMMIT_SHA" =~ ^[0-9a-f]{40}$ && "$REMOTE_DIR" =~ ^/[a-zA-Z0-9_/-]+$ && "$REMOTE_DIR" != / && "$REMOTE_DIR" != *..* ]] || exit 2
known_hosts="$(mktemp)"
archive="$(mktemp)"
trap 'rm -f "$known_hosts" "$archive"' EXIT
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$known_hosts"
chmod 600 "$known_hosts"
options=(-o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$known_hosts" -o ConnectTimeout=15)
target="$DEPLOY_USER@$DEPLOY_HOST"
case "${1:-}" in
verify)
build="$REMOTE_DIR/builds/$COMMIT_SHA"
tar --exclude=.git --exclude=node_modules --exclude=.env --exclude=logs --exclude=bin -czf "$archive" .
sshpass -e ssh "${options[@]}" "$target" "umask 077; mkdir -p '$build'"
sshpass -e scp "${options[@]}" "$archive" "$target:$build/source.tar.gz"
sshpass -e ssh "${options[@]}" "$target" "cd '$build'; tar -xzf source.tar.gz; rm source.tar.gz; bash deploy/remote-build.sh '$REMOTE_DIR' '$COMMIT_SHA'"
;;
deploy)
sshpass -e ssh "${options[@]}" "$target" "bash '$REMOTE_DIR/releases/$COMMIT_SHA/release.sh' '$REMOTE_DIR' '$COMMIT_SHA'"
;;
*) exit 2 ;;
esac
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
root="${1:?Deployment directory required}"
sha="${2:?Commit SHA required}"
[[ "$root" =~ ^/[a-zA-Z0-9_/-]+$ && "$root" != / && "$root" != *..* && "$sha" =~ ^[0-9a-f]{40}$ ]] || exit 2
exec 9>"$root/.build.lock"
flock -w 1800 9
cd "$root/builds/$sha"
bash deploy/check.sh
docker build -t "sundynix/veyra:$sha" .
release="$root/releases/$sha"
umask 077
mkdir -p "$release"
cp deploy/docker-compose.yml deploy/release.sh "$release/"
docker save "sundynix/veyra:$sha" | gzip > "$release/image.tar.gz"
(cd "$release" && sha256sum image.tar.gz > image.tar.gz.sha256)
echo "Verified image prepared: $sha"
+7
View File
@@ -2,6 +2,13 @@
更新日期:2026-10-09。
## CI 构建优化与 HTTPS(2026-10-09,进行中)
- PR #1 已正常合并至 master,提交 abcb1c99;触发 Actions run 38。未直接推送 master。
- Runner 下载 Docker 外部 frontend 约 14 MB 耗时数分钟,属于 Docker Hub 网络瓶颈。移除不必要的 docker/dockerfile:1 frontend 引用,改为 Runner 传源码、192.168.100.132 在独立 builds/<SHA> 目录执行完整容器检查与构建,复用已有依赖缓存;构建锁与发布锁分离,镜像本地打包,不跨主机传输大型镜像。
- 新增 PUBLIC_ORIGIN 配置,明确允许指定 HTTPS 公共域名,校验 Host 和 Origin,不盲信转发协议头;HTTPS 公共域名启用 Secure Cookie。配置与 handler 验证、完整容器内 Go race(独立 PostgreSQL)已通过。服务器公共域名配置已保存,原凭据与主密钥保留。
- Nginx 实际不在部署主机,用户明确自行配置;已提供 veyra.sundynix.cn 完整代理配置,目标为 192.168.100.132:8080。是否生效须实际验证,不声称已经配置。
## 最新 CI/CD 配置(2026-10-09)
- 用户要求参考 sundynix-pets 接入自动部署,并指定先创建 dev、禁止直接提交 master、CI/CD 只由 master push(包括 PR 合并)触发。已创建 dev 并保留当天全部修改;AGENTS.md 已记录分支规则。
+8
View File
@@ -14,6 +14,7 @@ import (
type Config struct {
PythonExecutable string
PublicOrigin string
Logging logging.Options
Environment, HTTPAddr string
PGHost, PGPort, PGDatabase, PGUser, PGPassword, PGSSLMode, PGAdminDatabase string
@@ -62,6 +63,13 @@ func LoadEnvFile(path string) error {
func Load() (Config, error) {
c := Config{Environment: env("APP_ENV", "local"), HTTPAddr: env("HTTP_ADDR", "127.0.0.1:8080"), PGHost: env("PGHOST", "192.168.100.127"), PGPort: env("PGPORT", "5432"), PGDatabase: env("PGDATABASE", "veyra"), PGUser: os.Getenv("PGUSER"), PGPassword: os.Getenv("PGPASSWORD"), PGSSLMode: env("PGSSLMODE", "disable"), PGAdminDatabase: env("PGADMIN_DATABASE", "postgres"), RedisAddr: env("REDIS_ADDR", "192.168.100.127:6379"), RedisUsername: os.Getenv("REDIS_USERNAME"), RedisPassword: os.Getenv("REDIS_PASSWORD")}
c.PythonExecutable = env("PYTHON_EXECUTABLE", "python3")
c.PublicOrigin = os.Getenv("PUBLIC_ORIGIN")
if c.PublicOrigin != "" {
origin, err := url.Parse(c.PublicOrigin)
if err != nil || (origin.Scheme != "http" && origin.Scheme != "https") || origin.Hostname() == "" || origin.User != nil || origin.Path != "" || origin.RawQuery != "" || origin.Fragment != "" || origin.Opaque != "" || origin.Scheme+"://"+origin.Host != c.PublicOrigin {
return Config{}, fmt.Errorf("PUBLIC_ORIGIN must be an http or https origin without a path")
}
}
c.Logging = logging.Options{Directory: env("LOG_DIR", "logs"), Level: env("LOG_LEVEL", "info"), MaxSizeMB: 50, MaxBackups: 10, MaxAgeDays: 14}
for key, dest := range map[string]*int{"LOG_MAX_SIZE_MB": &c.Logging.MaxSizeMB, "LOG_MAX_BACKUPS": &c.Logging.MaxBackups, "LOG_MAX_AGE_DAYS": &c.Logging.MaxAgeDays} {
if value := os.Getenv(key); value != "" {
+25
View File
@@ -23,3 +23,28 @@ func TestRejectInvalidRedisDB(t *testing.T) {
t.Fatal("negative database must be rejected")
}
}
func TestPublicOriginConfiguration(t *testing.T) {
for _, tc := range []struct {
origin string
valid bool
}{
{"", true},
{"https://veyra.sundynix.cn", true},
{"http://localhost:8080", true},
{"https://veyra.sundynix.cn/", false},
{"https://veyra.sundynix.cn?", false},
{"https://veyra.sundynix.cn#", false},
{"https://user:password@veyra.sundynix.cn", false},
{"ftp://veyra.sundynix.cn", false},
{"https:///missing-host", false},
} {
t.Run(tc.origin, func(t *testing.T) {
t.Setenv("PUBLIC_ORIGIN", tc.origin)
_, err := Load()
if (err == nil) != tc.valid {
t.Fatalf("valid=%v error=%v", tc.valid, err)
}
})
}
}
+6
View File
@@ -35,10 +35,13 @@ type IdentityOptions struct {
Limiter LoginLimiter
DevelopmentLogin bool
SecureCookie bool
PublicOrigin string
}
const sessionCookie = "veyra_admin_session"
type publicOriginContextKey struct{}
func bindIdentity(r *gin.Engine, options IdentityOptions) {
r.SetTrustedProxies(nil)
adminAuth := func(c *gin.Context) { authenticate(c, options, "admin") }
@@ -210,6 +213,9 @@ func sameOrigin(r *http.Request) bool {
if err != nil || origin.Host == "" || origin.User != nil || origin.Path != "" || origin.RawQuery != "" || origin.Fragment != "" {
return false
}
if expected, _ := r.Context().Value(publicOriginContextKey{}).(string); expected != "" {
return origin.Host == r.Host && origin.Scheme+"://"+origin.Host == expected
}
scheme := "http"
if r.TLS != nil {
scheme = "https"
+32
View File
@@ -49,3 +49,35 @@ func TestDevelopmentLoginDisabledByDefault(t *testing.T) {
t.Fatal("development login open outside explicitly enabled environment")
}
}
func TestHTTPSPublicOriginBehindProxy(t *testing.T) {
auth, err := service.NewAuthService(nil)
if err != nil {
t.Fatal(err)
}
probe := func(context.Context) error { return nil }
for _, tc := range []struct {
name, publicOrigin, host, origin, forwardedProto string
status int
}{
{"configured HTTPS", "https://veyra.sundynix.cn", "veyra.sundynix.cn", "https://veyra.sundynix.cn", "https", 503},
{"no reliance on proxy header", "https://veyra.sundynix.cn", "veyra.sundynix.cn", "https://veyra.sundynix.cn", "", 503},
{"foreign origin", "https://veyra.sundynix.cn", "veyra.sundynix.cn", "https://evil.example", "https", 403},
{"HTTP origin rejected", "https://veyra.sundynix.cn", "veyra.sundynix.cn", "http://veyra.sundynix.cn", "https", 403},
{"wrong host", "https://veyra.sundynix.cn", "evil.example", "https://veyra.sundynix.cn", "https", 403},
{"spoofed proxy header", "", "veyra.sundynix.cn", "https://veyra.sundynix.cn", "https", 403},
} {
t.Run(tc.name, func(t *testing.T) {
router := NewRouter(service.NewHealthService(probe, probe), zap.NewNop(), IdentityOptions{Auth: auth, DevelopmentLogin: true, PublicOrigin: tc.publicOrigin})
req := httptest.NewRequest(http.MethodPost, "http://"+tc.host+"/api/admin/v1/auth/login", strings.NewReader(`{"login_name":"admin","password":"test-password"}`))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Origin", tc.origin)
req.Header.Set("X-Forwarded-Proto", tc.forwardedProto)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != tc.status {
t.Fatalf("got %d want %d", rec.Code, tc.status)
}
})
}
}
+3
View File
@@ -21,6 +21,9 @@ func NewRouter(health *service.HealthService, logger *zap.Logger, identity Ident
r.Use(func(c *gin.Context) {
ctx, cancel := context.WithTimeout(c.Request.Context(), 5*time.Second)
defer cancel()
if identity.PublicOrigin != "" {
ctx = context.WithValue(ctx, publicOriginContextKey{}, identity.PublicOrigin)
}
c.Request = c.Request.WithContext(ctx)
requestID := uuid.NewString()
c.Header("X-Request-ID", requestID)
+2 -1
View File
@@ -12,6 +12,7 @@ import (
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
@@ -148,7 +149,7 @@ func serve(ctx context.Context, c config.Config, logger *zap.Logger) error {
screeningDone := make(chan struct{})
go func() { defer close(screeningDone); screenings.Run(workerContext, logger) }()
defer func() { workerCancel(); <-workerDone; <-researchDone; <-marketDone; <-screeningDone }()
identityOptions := handler.IdentityOptions{Screenings: screenings, Market: marketService, Backtests: backtests, Strategies: service.NewStrategyService(strategyRepo, development), Notifications: notifications, Auth: auth, Fills: service.NewTradeFillService(fillRepo, development), Plans: service.NewTradePlanService(planRepo, development), Candidates: service.NewCandidateService(candidateRepo, development), Providers: service.NewProviderService(providerRepo, vault), Accounts: service.NewAccountService(accountRepo), Limiter: ratelimit.NewRedis(redisClient), DevelopmentLogin: development, SecureCookie: !development}
identityOptions := handler.IdentityOptions{Screenings: screenings, Market: marketService, Backtests: backtests, Strategies: service.NewStrategyService(strategyRepo, development), Notifications: notifications, Auth: auth, Fills: service.NewTradeFillService(fillRepo, development), Plans: service.NewTradePlanService(planRepo, development), Candidates: service.NewCandidateService(candidateRepo, development), Providers: service.NewProviderService(providerRepo, vault), Accounts: service.NewAccountService(accountRepo), Limiter: ratelimit.NewRedis(redisClient), DevelopmentLogin: development, SecureCookie: !development || strings.HasPrefix(c.PublicOrigin, "https://"), PublicOrigin: c.PublicOrigin}
server := &http.Server{Addr: c.HTTPAddr, Handler: handler.NewRouter(health, logger, identityOptions), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 1 << 20}
result := make(chan error, 1)
go func() { result <- server.ListenAndServe() }()