debug: verify staging post-deploy

Co-authored-by: xiaoxia <dev@xiaoxiajianji.com>
Co-committed-by: xiaoxia <dev@xiaoxiajianji.com>

.gitea/workflows/debug-web-v2.yml

@@ -0,0 +1,60 @@
+name: "Debug: Web container v2 (mount conflict)"
+on:
+  push:
+    branches: [debug/web-crash-v2]
+  workflow_dispatch:
+jobs:
+  web-diag:
+    runs-on: runtime-builder
+    timeout-minutes: 10
+    steps:
+      - name: Setup SSH and diagnose
+        shell: bash
+        env:
+          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+        run: |
+          set -x
+          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          H=47.98.113.167; P=22222
+          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
+          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H 'bash -s' <<'REMOTE'
+          set -x
+          echo "=== Current staging containers ==="
+          docker ps -a --filter name=xiaoxia-*-staging --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
+          echo ""
+          echo "=== Web container logs (current/current-rolledback) ==="
+          docker logs xiaoxia-web-staging 2>&1 | tail -40
+          echo ""
+          echo "=== Web inspect: env & mounts ==="
+          docker inspect xiaoxia-web-staging --format 'Entrypoint: {{.Config.Entrypoint}} Cmd: {{.Config.Cmd}}'
+          docker inspect xiaoxia-web-staging --format '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' | grep -E "APP_ENV|VERSION"
+          echo "Mounts:"
+          docker inspect xiaoxia-web-staging --format '{{range .Mounts}}{{.Type}} {{.Source}} -> {{.Destination}} (rw={{.RW}}){{"\n"}}{{end}}'
+          echo ""
+          echo "=== Reproduce: rm on read-only bind mount ==="
+          docker run --rm --name nginx-ro-test \
+            -v /var/lib/xiaoxia-saas-staging/nginx-staging.conf:/etc/nginx/conf.d/default.conf:ro \
+            git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/xiaoxia-saas-web:387514c \
+            sh -c '
+              set -x
+              echo "Before:"
+              ls -la /etc/nginx/conf.d/
+              echo "Try rm (as entrypoint does):"
+              rm -f /etc/nginx/conf.d/default.conf
+              echo "rm exitcode=$?"
+              echo "After rm:"
+              ls -la /etc/nginx/conf.d/
+              echo "Test ln:"
+              ln -s /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf
+              echo "ln exitcode=$?"
+              ls -la /etc/nginx/conf.d/
+              echo "nginx -t:"
+              nginx -t 2>&1
+            ' 2>&1
+          echo ""
+          echo "=== Also test with NEW fixed image (9c0d4b1 if present) ==="
+          docker images | grep xiaoxia-saas-web | head -5
+REMOTE

.gitea/workflows/verify-staging.yml

@@ -0,0 +1,29 @@
+name: "Verify staging post-deploy"
+on:
+  push:
+    branches: [debug/verify-staging]
+  workflow_dispatch:
+jobs:
+  verify:
+    runs-on: runtime-builder
+    timeout-minutes: 10
+    steps:
+      - name: Setup SSH
+        shell: bash
+        env:
+          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+        run: |
+          set -eux
+          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          H=47.98.113.167; P=22222
+          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
+      - name: Run verify
+        shell: bash
+        run: |
+          set -x
+          echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSAxLiBDdXJyZW50IHJ1bm5pbmcgY29udGFpbmVycyA9PT0iCmRvY2tlciBwcyAtLWZpbHRlciBuYW1lPXhpYW94aWEtKi1zdGFnaW5nIC0tZm9ybWF0ICJ0YWJsZSB7ey5OYW1lc319XHR7ey5TdGF0dXN9fVx0e3suSW1hZ2V9fSIKZWNobyAiIgplY2hvICI9PT0gMi4gV2ViIGxvZ3MgKGxhc3QgMTUgbGluZXMpID09PSIKZG9ja2VyIGxvZ3MgLS10YWlsIDE1IHhpYW94aWEtd2ViLXN0YWdpbmcgMj4mMQplY2hvICIiCmVjaG8gIj09PSAzLiBXZWIgbG9jYWxob3N0IGhlYWx0aCBjaGVjayA9PT0iCmN1cmwgLXNJIC0tbWF4LXRpbWUgNSBodHRwOi8vMTI3LjAuMC4xOjMwMDEvIHwgaGVhZCAtNQplY2hvICIiCmVjaG8gIj09PSA0LiBBUEkgbG9jYWxob3N0IGhlYWx0aCBjaGVjayA9PT0iCmN1cmwgLXNmIC0tbWF4LXRpbWUgNSBodHRwOi8vMTI3LjAuMC4xOjgwMDAvaGVhbHRoCmVjaG8gIiIKZWNobyAiPT09IDUuIEFQSSB2ZXJzaW9uID09PSIKY3VybCAtcyAtLW1heC10aW1lIDUgaHR0cDovLzEyNy4wLjAuMTo4MDAwL2hlYWx0aCB8IHB5dGhvbjMgLWMgImltcG9ydCBzeXMsanNvbjsgZD1qc29uLmxvYWQoc3lzLnN0ZGluKTsgcHJpbnQoJ3ZlcnNpb246JywgZC5nZXQoJ3ZlcnNpb24nLCc/JykpIiAyPi9kZXYvbnVsbAplY2hvICIiCmVjaG8gIj09PSA2LiBXb3JrZXIgY29udGFpbmVyIHZlcnNpb24gPT09Igpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdvcmtlci1zdGFnaW5nIC0tZm9ybWF0ICd7e3JhbmdlIC5Db25maWcuRW52fX17ey59fXt7IlxuIn19e3tlbmR9fScgfCBncmVwIEFQUF9WRVJTSU9OCmVjaG8gIiIKZWNobyAiPT09IDcuIFdvcmtlciBsb2dzIChzdGFydHVwICsgdGFzayByZWdpc3RyYXRpb24pID09PSIKZG9ja2VyIGxvZ3MgLS10YWlsIDUwIHhpYW94aWEtd29ya2VyLXN0YWdpbmcgMj4mMSB8IGdyZXAgLWlFICJyZWdpc3RlcmVkfGNlbGVyeUB8bGlwc3luY3xyZWFkeXxNb2R1bGVOb3RGb3VuZHxFcnJvcnx0YXNrIiB8IGhlYWQgLTMwCmVjaG8gIiIKZWNobyAiPT09IDguIExpc3QgYWxsIHJlZ2lzdGVyZWQgY2VsZXJ5IHRhc2tzIChXb3JrZXLlhoVpbnNwZWN0KSA9PT0iCmRvY2tlciBleGVjIHhpYW94aWEtd29ya2VyLXN0YWdpbmcgc2ggLWMgImNlbGVyeSAtQSB3b3JrZXJfYXBwLmNlbGVyeV9hcHAgaW5zcGVjdCByZWdpc3RlcmVkIDI+JjEgfCBoZWFkIC00MCIgfHwgZWNobyAiY2VsZXJ5IGluc3BlY3QgZmFpbGVkIgplY2hvICIiCmVjaG8gIj09PSA5LiBDaGVjayBpbXBvcnRzIGluc2lkZSBXb3JrZXIgKGxpcHN5bmNfdHRzIG1vZHVsZSkgPT09Igpkb2NrZXIgZXhlYyB4aWFveGlhLXdvcmtlci1zdGFnaW5nIHB5dGhvbiAtYyAiCmltcG9ydCBpbXBvcnRsaWIKbSA9IGltcG9ydGxpYi5pbXBvcnRfbW9kdWxlKCdhcHAudGFza3MubGlwc3luY190dHMnKQpwcmludCgnbGlwc3luY190dHMgbW9kdWxlIGltcG9ydGVkIE9LJykKcHJpbnQoJ3N5bnRoZXNpemVfYW5kX3N1Ym1pdDonLCBoYXNhdHRyKG0sICdzeW50aGVzaXplX2FuZF9zdWJtaXQnKSkKcHJpbnQoJ1Nlc3Npb25Mb2NhbDonLCBoYXNhdHRyKG0sICdTZXNzaW9uTG9jYWwnKSkKIiAyPiYxIHwgaGVhZCAtMTAKZWNobyAiIgplY2hvICI9PT0gMTAuIFB1YmxpYyBBUEkgaGVhbHRoIGNoZWNrIChzdGFnaW5nLWFwaSkgPT09IgpjdXJsIC1zSSAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy1hcGkueGlhb3hpYWppYW5qaS5jb20vaGVhbHRoIHwgaGVhZCAtMwplY2hvICIiCmVjaG8gIj09PSAxMS4gUHVibGljIHdlYiBoZWFsdGggY2hlY2sgPT09IgpjdXJsIC1zSSAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy54aWFveGlhamlhbmppLmNvbS8gfCBoZWFkIC0zCg==' | base64 -d > /tmp/verify.sh
+          chmod +x /tmp/verify.sh
+          ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/verify.sh

apps/worker/worker_app/celery_app.py

@@ -34,6 +34,8 @@ celery_app.conf.imports = (
     "worker_app.tasks.tts_synthesis",
     "worker_app.tasks.batch_download",
     "worker_app.tasks.duplication_check",
+    # #1798 AI 数字人渲染：必须在 Worker 实例上注册同名任务，否则消息无人消费（渲染卡 0%）
+    "worker_app.tasks.ai_avatar_render",
     "worker_app.tasks._startup",
     "apps.worker.video_processing.dedup",
     "worker_app.tasks.cleanup",

apps/worker/worker_app/tasks/ai_avatar_render.py

@@ -0,0 +1,48 @@
+"""AI 数字人渲染任务 — Worker 侧 Celery 任务注册.
+
+#1798 渲染进度卡在 0% 的根因：渲染任务定义在 API 侧（`app.tasks.ai_avatar_render`），
+装饰在 API 自己的 Celery 实例（`xiaoxia-saas-api`）上；而 Worker 用的是
+`worker_app.celery_app` 实例，`conf.imports` 从未导入该任务，Worker 的任务
+注册表里没有 `ai_avatar_render.execute`，消息被路由到默认 `celery` 队列后
+无人消费，任务永远停在 0%。
+
+修复：在 Worker 侧用 `worker_app.celery_app` 注册同名任务，直接调用与 API
+服务一致的 `AiAvatarRenderService.execute_render` 核心管线（业务逻辑在
+`apps.api.app.services`，worker 镜像已复制 `apps/api/app`）。任务名保持
+`ai_avatar_render.execute`，与 API 生产端 `.delay()` 的消息路由一致；未在
+task_routes 显式配置，走默认 `celery` 队列，由 transcode worker 消费。
+"""
+
+from __future__ import annotations
+
+import logging
+
+from worker_app.celery_app import celery_app
+from worker_app.db import SessionLocal
+
+logger = logging.getLogger(__name__)
+
+
+@celery_app.task(bind=True, name="ai_avatar_render.execute", max_retries=2)
+def execute_ai_avatar_render(self, job_id: str) -> dict:
+    """执行 AI 数字人渲染管线（Worker 侧入口）.
+
+    进度由 service 直接写入 DB（AiAvatarRenderJob.progress:
+    0→5→20→40→80→90→95→100），API 通过轮询 progress 字段展示。
+    """
+    logger.info("开始执行渲染任务: %s", job_id)
+    self.update_state(state="PROCESSING", meta={"progress": 0, "job_id": job_id})
+
+    session = SessionLocal()
+    try:
+        from app.services.ai_avatar_render_service import AiAvatarRenderService
+
+        service = AiAvatarRenderService(session)
+        service.execute_render(job_id)
+        return {"status": "completed", "job_id": job_id}
+    except Exception as exc:
+        logger.exception("渲染任务执行异常 [%s]: %s", job_id, exc)
+        self.update_state(state="FAILED", meta={"progress": 0, "error": str(exc)})
+        raise
+    finally:
+        session.close()

infra/docker/nginx-entrypoint.sh

@@ -1,23 +1,49 @@
 #!/bin/sh
 # Select nginx config based on APP_ENV (staging/production).
-# Both configs are baked into the image at well-known paths.
-# nginx reads config only at startup, so symlink before exec.
-# 注意：基础镜像 /etc/nginx/conf.d/default.conf 是普通文件（非 symlink/目录），
-# alpine busybox ln -sf 在 target 已存在且为普通文件时行为不稳定（会尝试在
-# target 目录下建子链接），必须先 rm 再 ln 才能正确替换。
+#
+# 两种运行模式：
+# 1. CI/CD 部署（staging/production）：部署脚本通过 `-v 宿主机文件:/etc/nginx/conf.d/default.conf:ro`
+#    把宿主机生成的带 resolver/docker upstream 的配置 bind mount 进来，entrypoint 不应改动。
+#    bind mount 的文件是 readonly 的，rm 会报 EBUSY ("Resource busy")，直接 exec nginx 即可。
+# 2. 本地 docker-compose / 直接 `docker run`（无外部挂载）：镜像烤入了 nginx-staging.conf 与
+#    nginx-production.conf 到 /etc/nginx/，entrypoint 根据 APP_ENV 把 default.conf 换成正确的 symlink。
+#
+# 策略：
+# - 如果 /etc/nginx/conf.d/default.conf 已经是指向目标 conf 的 symlink，什么都不做；
+# - 否则尝试 rm -f 再 ln -s；rm 失败说明是外部 bind mount（已有正确配置），不阻塞启动；
+# - 兜底：只要 conf.d 目录里有 .conf 文件（含 bind mount 来的），就直接启动 nginx。
 set -e
 
 NGINX_CONF_DIR="/etc/nginx/conf.d"
+TARGET_CONF=""
 
 case "${APP_ENV:-production}" in
   staging)
-    rm -f "$NGINX_CONF_DIR/default.conf"
-    ln -s /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
+    TARGET_CONF="/etc/nginx/nginx-staging.conf"
     ;;
   *)
-    rm -f "$NGINX_CONF_DIR/default.conf"
-    ln -s /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
+    TARGET_CONF="/etc/nginx/nginx-production.conf"
     ;;
 esac
 
+DEFAULT_CONF="$NGINX_CONF_DIR/default.conf"
+
+# 1. 已经是正确的 symlink：直接启动
+if [ -L "$DEFAULT_CONF" ] && [ "$(readlink "$DEFAULT_CONF" 2>/dev/null)" = "$TARGET_CONF" ]; then
+  exec nginx -g "daemon off;"
+fi
+
+# 2. 尝试替换为目标 symlink（无 bind mount 的场景）
+#    若 rm 失败（bind mount readonly，EBUSY/EPERM），则认为外部已注入配置，不阻塞。
+rm -f "$DEFAULT_CONF" 2>/dev/null || true
+if [ -f "$TARGET_CONF" ] && [ ! -e "$DEFAULT_CONF" ]; then
+  ln -s "$TARGET_CONF" "$DEFAULT_CONF" 2>/dev/null || true
+fi
+
+# 3. 兜底：至少要有一个 .conf 文件，否则 nginx 起不来
+if ! ls "$NGINX_CONF_DIR"/*.conf >/dev/null 2>&1; then
+  echo "ERROR: no nginx config found in $NGINX_CONF_DIR (tried $TARGET_CONF and external bind mount)" >&2
+  exit 1
+fi
+
 exec nginx -g "daemon off;"