debug(lipsync): v3 - docker cp python scripts, check nginx/DB

.gitea/workflows/debug-lipsync-diag.yml

@@ -0,0 +1,43 @@
+name: Debug Lipsync SSH Diag
+on:
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: "debug reason"
+        required: false
+        default: "lipsync tts_processing stuck diagnosis"
+  push:
+    branches:
+      - debug/lipsync-ssh-diag
+permissions:
+  contents: read
+jobs:
+  ssh-diag:
+    name: SSH Staging Diagnostics
+    runs-on: runtime-builder
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup SSH
+        shell: bash
+        env:
+          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+        run: |
+          set -euo pipefail
+          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          H=47.98.113.167; P=22222
+          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
+          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H "echo SSH_OK; hostname; date"
+      - name: Upload diag script and run
+        shell: bash
+        run: |
+          set -euo pipefail
+          H=47.98.113.167; P=22222
+          scp -P $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \
+            infra/scripts/server-lipsync-diag-v3.sh root@$H:/tmp/server-lipsync-diag-v3.sh
+          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H \
+            "bash /tmp/server-lipsync-diag-v3.sh 2>&1"

.gitea/workflows/debug-web-v2.yml

@@ -1,60 +0,0 @@
-name: "Debug: Web container v2 (mount conflict)"
-on:
-  push:
-    branches: [debug/web-crash-v2]
-  workflow_dispatch:
-jobs:
-  web-diag:
-    runs-on: runtime-builder
-    timeout-minutes: 10
-    steps:
-      - name: Setup SSH and diagnose
-        shell: bash
-        env:
-          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
-        run: |
-          set -x
-          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
-          mkdir -p ~/.ssh && chmod 700 ~/.ssh
-          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
-          chmod 600 ~/.ssh/id_rsa
-          H=47.98.113.167; P=22222
-          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
-          ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H 'bash -s' <<'REMOTE'
-          set -x
-          echo "=== Current staging containers ==="
-          docker ps -a --filter name=xiaoxia-*-staging --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
-          echo ""
-          echo "=== Web container logs (current/current-rolledback) ==="
-          docker logs xiaoxia-web-staging 2>&1 | tail -40
-          echo ""
-          echo "=== Web inspect: env & mounts ==="
-          docker inspect xiaoxia-web-staging --format 'Entrypoint: {{.Config.Entrypoint}} Cmd: {{.Config.Cmd}}'
-          docker inspect xiaoxia-web-staging --format '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' | grep -E "APP_ENV|VERSION"
-          echo "Mounts:"
-          docker inspect xiaoxia-web-staging --format '{{range .Mounts}}{{.Type}} {{.Source}} -> {{.Destination}} (rw={{.RW}}){{"\n"}}{{end}}'
-          echo ""
-          echo "=== Reproduce: rm on read-only bind mount ==="
-          docker run --rm --name nginx-ro-test \
-            -v /var/lib/xiaoxia-saas-staging/nginx-staging.conf:/etc/nginx/conf.d/default.conf:ro \
-            git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/xiaoxia-saas-web:387514c \
-            sh -c '
-              set -x
-              echo "Before:"
-              ls -la /etc/nginx/conf.d/
-              echo "Try rm (as entrypoint does):"
-              rm -f /etc/nginx/conf.d/default.conf
-              echo "rm exitcode=$?"
-              echo "After rm:"
-              ls -la /etc/nginx/conf.d/
-              echo "Test ln:"
-              ln -s /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf
-              echo "ln exitcode=$?"
-              ls -la /etc/nginx/conf.d/
-              echo "nginx -t:"
-              nginx -t 2>&1
-            ' 2>&1
-          echo ""
-          echo "=== Also test with NEW fixed image (9c0d4b1 if present) ==="
-          docker images | grep xiaoxia-saas-web | head -5
-REMOTE

.gitea/workflows/verify-staging.yml

@@ -1,29 +0,0 @@
-name: "Verify staging post-deploy"
-on:
-  push:
-    branches: [debug/verify-staging]
-  workflow_dispatch:
-jobs:
-  verify:
-    runs-on: runtime-builder
-    timeout-minutes: 10
-    steps:
-      - name: Setup SSH
-        shell: bash
-        env:
-          STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
-        run: |
-          set -eux
-          which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
-          mkdir -p ~/.ssh && chmod 700 ~/.ssh
-          printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
-          chmod 600 ~/.ssh/id_rsa
-          H=47.98.113.167; P=22222
-          ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
-      - name: Run verify
-        shell: bash
-        run: |
-          set -x
-          echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSAxLiBDdXJyZW50IHJ1bm5pbmcgY29udGFpbmVycyA9PT0iCmRvY2tlciBwcyAtLWZpbHRlciBuYW1lPXhpYW94aWEtKi1zdGFnaW5nIC0tZm9ybWF0ICJ0YWJsZSB7ey5OYW1lc319XHR7ey5TdGF0dXN9fVx0e3suSW1hZ2V9fSIKZWNobyAiIgplY2hvICI9PT0gMi4gV2ViIGxvZ3MgKGxhc3QgMTUgbGluZXMpID09PSIKZG9ja2VyIGxvZ3MgLS10YWlsIDE1IHhpYW94aWEtd2ViLXN0YWdpbmcgMj4mMQplY2hvICIiCmVjaG8gIj09PSAzLiBXZWIgbG9jYWxob3N0IGhlYWx0aCBjaGVjayA9PT0iCmN1cmwgLXNJIC0tbWF4LXRpbWUgNSBodHRwOi8vMTI3LjAuMC4xOjMwMDEvIHwgaGVhZCAtNQplY2hvICIiCmVjaG8gIj09PSA0LiBBUEkgbG9jYWxob3N0IGhlYWx0aCBjaGVjayA9PT0iCmN1cmwgLXNmIC0tbWF4LXRpbWUgNSBodHRwOi8vMTI3LjAuMC4xOjgwMDAvaGVhbHRoCmVjaG8gIiIKZWNobyAiPT09IDUuIEFQSSB2ZXJzaW9uID09PSIKY3VybCAtcyAtLW1heC10aW1lIDUgaHR0cDovLzEyNy4wLjAuMTo4MDAwL2hlYWx0aCB8IHB5dGhvbjMgLWMgImltcG9ydCBzeXMsanNvbjsgZD1qc29uLmxvYWQoc3lzLnN0ZGluKTsgcHJpbnQoJ3ZlcnNpb246JywgZC5nZXQoJ3ZlcnNpb24nLCc/JykpIiAyPi9kZXYvbnVsbAplY2hvICIiCmVjaG8gIj09PSA2LiBXb3JrZXIgY29udGFpbmVyIHZlcnNpb24gPT09Igpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdvcmtlci1zdGFnaW5nIC0tZm9ybWF0ICd7e3JhbmdlIC5Db25maWcuRW52fX17ey59fXt7IlxuIn19e3tlbmR9fScgfCBncmVwIEFQUF9WRVJTSU9OCmVjaG8gIiIKZWNobyAiPT09IDcuIFdvcmtlciBsb2dzIChzdGFydHVwICsgdGFzayByZWdpc3RyYXRpb24pID09PSIKZG9ja2VyIGxvZ3MgLS10YWlsIDUwIHhpYW94aWEtd29ya2VyLXN0YWdpbmcgMj4mMSB8IGdyZXAgLWlFICJyZWdpc3RlcmVkfGNlbGVyeUB8bGlwc3luY3xyZWFkeXxNb2R1bGVOb3RGb3VuZHxFcnJvcnx0YXNrIiB8IGhlYWQgLTMwCmVjaG8gIiIKZWNobyAiPT09IDguIExpc3QgYWxsIHJlZ2lzdGVyZWQgY2VsZXJ5IHRhc2tzIChXb3JrZXLlhoVpbnNwZWN0KSA9PT0iCmRvY2tlciBleGVjIHhpYW94aWEtd29ya2VyLXN0YWdpbmcgc2ggLWMgImNlbGVyeSAtQSB3b3JrZXJfYXBwLmNlbGVyeV9hcHAgaW5zcGVjdCByZWdpc3RlcmVkIDI+JjEgfCBoZWFkIC00MCIgfHwgZWNobyAiY2VsZXJ5IGluc3BlY3QgZmFpbGVkIgplY2hvICIiCmVjaG8gIj09PSA5LiBDaGVjayBpbXBvcnRzIGluc2lkZSBXb3JrZXIgKGxpcHN5bmNfdHRzIG1vZHVsZSkgPT09Igpkb2NrZXIgZXhlYyB4aWFveGlhLXdvcmtlci1zdGFnaW5nIHB5dGhvbiAtYyAiCmltcG9ydCBpbXBvcnRsaWIKbSA9IGltcG9ydGxpYi5pbXBvcnRfbW9kdWxlKCdhcHAudGFza3MubGlwc3luY190dHMnKQpwcmludCgnbGlwc3luY190dHMgbW9kdWxlIGltcG9ydGVkIE9LJykKcHJpbnQoJ3N5bnRoZXNpemVfYW5kX3N1Ym1pdDonLCBoYXNhdHRyKG0sICdzeW50aGVzaXplX2FuZF9zdWJtaXQnKSkKcHJpbnQoJ1Nlc3Npb25Mb2NhbDonLCBoYXNhdHRyKG0sICdTZXNzaW9uTG9jYWwnKSkKIiAyPiYxIHwgaGVhZCAtMTAKZWNobyAiIgplY2hvICI9PT0gMTAuIFB1YmxpYyBBUEkgaGVhbHRoIGNoZWNrIChzdGFnaW5nLWFwaSkgPT09IgpjdXJsIC1zSSAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy1hcGkueGlhb3hpYWppYW5qaS5jb20vaGVhbHRoIHwgaGVhZCAtMwplY2hvICIiCmVjaG8gIj09PSAxMS4gUHVibGljIHdlYiBoZWFsdGggY2hlY2sgPT09IgpjdXJsIC1zSSAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy54aWFveGlhamlhbmppLmNvbS8gfCBoZWFkIC0zCg==' | base64 -d > /tmp/verify.sh
-          chmod +x /tmp/verify.sh
-          ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/verify.sh

apps/api/app/services/lipsync_service.py

@@ -211,18 +211,12 @@ class LipsyncService:
                         normalize_emotion(emotion),
                     )
                 )
-            except Exception as exc:
-                # 投递失败时立即把 job 标成 failed 并写入 error_message，
-                # 前端轮询时能直接看到失败原因，不会无限卡在 tts_processing。
-                logger.exception(
-                    "Celery 任务提交失败，TTS 任务已创建但未触发执行: job_id=%s err=%s",
+            except Exception:
+                logger.warning(
+                    "Celery 任务提交失败，TTS 任务已创建但未触发执行: %s",
                     job_id,
-                    exc,
+                    exc_info=True,
                 )
-                job.status = "failed"
-                job.error_message = f"Celery 任务投递失败: {exc}"
-                job.error_code = "AsyncDispatchFailed"
-                job.updated_at = datetime.now(timezone.utc)
         else:
             # 2b. 直接音频模式：同步签名并提交 MediaKit
             video_url = self._sign_media_url(video_url)

apps/api/app/tasks/lipsync_tts.py

@@ -76,21 +76,11 @@ def tts_synthesize_and_submit(
     from app.services.mediakit_client import MediaKitError, get_mediakit_client
     from sqlalchemy.orm import Session as DBSession
 
+    from packages.adapters.sqlalchemy_impl.database import SessionLocal
     from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
     from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService
     from packages.shared.url_security import safe_download_bytes
 
-    # SessionLocal 获取：
-    # - API 容器：app.db.SessionLocal（环境变量完整，导入即建引擎）
-    # - Worker 容器：worker_app.db.SessionLocal（Worker 自己的 settings 初始化引擎）
-    # API 侧没有 worker_app 模块 → ImportError 直接回退；
-    # Worker 侧 app.db 会因缺少 API 专有环境变量抛 pydantic ValidationError，
-    # 此时也要回退到 worker_app.db。
-    try:
-        from worker_app.db import SessionLocal  # type: ignore
-    except Exception:  # noqa: BLE001
-        from app.db import SessionLocal  # type: ignore
-
     db: DBSession = SessionLocal()
     try:
         job = (

apps/worker/worker_app/celery_app.py

@@ -34,8 +34,6 @@ celery_app.conf.imports = (
     "worker_app.tasks.tts_synthesis",
     "worker_app.tasks.batch_download",
     "worker_app.tasks.duplication_check",
-    # #1798 AI 数字人渲染：必须在 Worker 实例上注册同名任务，否则消息无人消费（渲染卡 0%）
-    "worker_app.tasks.ai_avatar_render",
     "worker_app.tasks._startup",
     "apps.worker.video_processing.dedup",
     "worker_app.tasks.cleanup",

apps/worker/worker_app/tasks/ai_avatar_render.py

@@ -1,48 +0,0 @@
-"""AI 数字人渲染任务 — Worker 侧 Celery 任务注册.
-
-#1798 渲染进度卡在 0% 的根因：渲染任务定义在 API 侧（`app.tasks.ai_avatar_render`），
-装饰在 API 自己的 Celery 实例（`xiaoxia-saas-api`）上；而 Worker 用的是
-`worker_app.celery_app` 实例，`conf.imports` 从未导入该任务，Worker 的任务
-注册表里没有 `ai_avatar_render.execute`，消息被路由到默认 `celery` 队列后
-无人消费，任务永远停在 0%。
-
-修复：在 Worker 侧用 `worker_app.celery_app` 注册同名任务，直接调用与 API
-服务一致的 `AiAvatarRenderService.execute_render` 核心管线（业务逻辑在
-`apps.api.app.services`，worker 镜像已复制 `apps/api/app`）。任务名保持
-`ai_avatar_render.execute`，与 API 生产端 `.delay()` 的消息路由一致；未在
-task_routes 显式配置，走默认 `celery` 队列，由 transcode worker 消费。
-"""
-
-from __future__ import annotations
-
-import logging
-
-from worker_app.celery_app import celery_app
-from worker_app.db import SessionLocal
-
-logger = logging.getLogger(__name__)
-
-
-@celery_app.task(bind=True, name="ai_avatar_render.execute", max_retries=2)
-def execute_ai_avatar_render(self, job_id: str) -> dict:
-    """执行 AI 数字人渲染管线（Worker 侧入口）.
-
-    进度由 service 直接写入 DB（AiAvatarRenderJob.progress:
-    0→5→20→40→80→90→95→100），API 通过轮询 progress 字段展示。
-    """
-    logger.info("开始执行渲染任务: %s", job_id)
-    self.update_state(state="PROCESSING", meta={"progress": 0, "job_id": job_id})
-
-    session = SessionLocal()
-    try:
-        from app.services.ai_avatar_render_service import AiAvatarRenderService
-
-        service = AiAvatarRenderService(session)
-        service.execute_render(job_id)
-        return {"status": "completed", "job_id": job_id}
-    except Exception as exc:
-        logger.exception("渲染任务执行异常 [%s]: %s", job_id, exc)
-        self.update_state(state="FAILED", meta={"progress": 0, "error": str(exc)})
-        raise
-    finally:
-        session.close()

infra/docker/nginx-entrypoint.sh

@@ -1,49 +1,18 @@
 #!/bin/sh
 # Select nginx config based on APP_ENV (staging/production).
-#
-# 两种运行模式：
-# 1. CI/CD 部署（staging/production）：部署脚本通过 `-v 宿主机文件:/etc/nginx/conf.d/default.conf:ro`
-#    把宿主机生成的带 resolver/docker upstream 的配置 bind mount 进来，entrypoint 不应改动。
-#    bind mount 的文件是 readonly 的，rm 会报 EBUSY ("Resource busy")，直接 exec nginx 即可。
-# 2. 本地 docker-compose / 直接 `docker run`（无外部挂载）：镜像烤入了 nginx-staging.conf 与
-#    nginx-production.conf 到 /etc/nginx/，entrypoint 根据 APP_ENV 把 default.conf 换成正确的 symlink。
-#
-# 策略：
-# - 如果 /etc/nginx/conf.d/default.conf 已经是指向目标 conf 的 symlink，什么都不做；
-# - 否则尝试 rm -f 再 ln -s；rm 失败说明是外部 bind mount（已有正确配置），不阻塞启动；
-# - 兜底：只要 conf.d 目录里有 .conf 文件（含 bind mount 来的），就直接启动 nginx。
+# Both configs are baked into the image at well-known paths.
+# nginx reads config only at startup, so symlink before exec.
 set -e
 
 NGINX_CONF_DIR="/etc/nginx/conf.d"
-TARGET_CONF=""
 
 case "${APP_ENV:-production}" in
   staging)
-    TARGET_CONF="/etc/nginx/nginx-staging.conf"
+    ln -sf /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
     ;;
   *)
-    TARGET_CONF="/etc/nginx/nginx-production.conf"
+    ln -sf /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
     ;;
 esac
 
-DEFAULT_CONF="$NGINX_CONF_DIR/default.conf"
-
-# 1. 已经是正确的 symlink：直接启动
-if [ -L "$DEFAULT_CONF" ] && [ "$(readlink "$DEFAULT_CONF" 2>/dev/null)" = "$TARGET_CONF" ]; then
-  exec nginx -g "daemon off;"
-fi
-
-# 2. 尝试替换为目标 symlink（无 bind mount 的场景）
-#    若 rm 失败（bind mount readonly，EBUSY/EPERM），则认为外部已注入配置，不阻塞。
-rm -f "$DEFAULT_CONF" 2>/dev/null || true
-if [ -f "$TARGET_CONF" ] && [ ! -e "$DEFAULT_CONF" ]; then
-  ln -s "$TARGET_CONF" "$DEFAULT_CONF" 2>/dev/null || true
-fi
-
-# 3. 兜底：至少要有一个 .conf 文件，否则 nginx 起不来
-if ! ls "$NGINX_CONF_DIR"/*.conf >/dev/null 2>&1; then
-  echo "ERROR: no nginx config found in $NGINX_CONF_DIR (tried $TARGET_CONF and external bind mount)" >&2
-  exit 1
-fi
-
 exec nginx -g "daemon off;"

infra/scripts/server-lipsync-diag-v2.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+# Lipsync 深度诊断脚本 v2 - 重点排查broker连通/消息投递/DB状态
+set +e
+echo "#########################################################"
+echo "#  Lipsync 深度诊断 v2"
+echo "#  Date: $(date)"
+echo "#########################################################"
+
+W=$(docker ps --format '{{.Names}}' | grep -E 'worker' | head -1)
+A=$(docker ps --format '{{.Names}}' | grep -E 'api' | grep -v 'web' | head -1)
+R=$(docker ps --format '{{.Names}}' | grep -E 'redis' | head -1)
+echo "Containers: W=$W  A=$A  R=$R"
+
+echo ""
+echo "=== 1. 容器 uptime + image tag ==="
+for c in $W $A; do
+  echo "--- $c ---"
+  docker inspect "$c" --format 'Image={{.Config.Image}} Created={{.Created}} Started={{.State.StartedAt}} Restarts={{.RestartCount}}'
+done
+
+echo ""
+echo "=== 2. API+Worker 的 BROKER/REDIS 地址（隐藏密码） ==="
+echo "--- API BROKER ---"
+docker exec "$A" env | grep -iE "broker|celery_broker|redis_url|backend" | sed -E 's|(:)?//[^:]+:([^@]+)@|\1//***:***@|g'
+echo "--- Worker BROKER ---"
+docker exec "$W" env | grep -iE "broker|celery_broker|redis_url|backend" | sed -E 's|(:)?//[^:]+:([^@]+)@|\1//***:***@|g'
+
+echo ""
+echo "=== 3. API容器 完整日志（近1000行） lipsync/TTS 相关 ==="
+docker logs --tail=1000 "$A" 2>&1 | grep -iE "lipsync|tts_synth|tts_processing|celery.*task|apply_async|NotRegistered|OperationalError|ConnectionError|error.*submit|traceback|submitted" | tail -200
+
+echo ""
+echo "=== 4. API 容器最近的 ERROR/Exception ==="
+docker logs --tail=2000 "$A" 2>&1 | grep -iE "error|exception|traceback|critical" | grep -v "health\|/health" | tail -80
+
+echo ""
+echo "=== 5. 测试1：在API容器内实际投递一条测试消息，看Worker是否消费 ==="
+# 投递后立即检查队列和Worker日志
+docker exec "$A" python - <<'PYEOF'
+import sys, time, traceback
+try:
+    from app.tasks.lipsync_tts import tts_synthesize_and_submit
+    from app.core.celery_app import celery_app as api_app
+    # 检查broker连接
+    conn = api_app.connection()
+    conn.ensure_connection(max_retries=2)
+    print("API celery broker connected:", conn.as_uri())
+    # 投递到celery默认队列
+    result = tts_synthesize_and_submit.apply_async(
+        args=["diag-test-job-id", "diag-user-id", "diag-voice", "diagnostic script text", 1.0, "neutral"],
+        queue="celery",
+    )
+    print("APPLY_ASYNC_OK task_id:", result.id)
+    print("task name:", result.name)
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+echo ""
+sleep 3
+echo "--- After apply_async: queue lengths ---"
+for q in celery generation transcode; do
+    echo "  $q: $(docker exec "$R" redis-cli LLEN $q)"
+done
+
+echo ""
+echo "=== 6. 等8秒看Worker是否收到并消费 ==="
+sleep 8
+echo "--- Queue lengths after 8s ---"
+for q in celery generation transcode; do
+    echo "  $q: $(docker exec "$R" redis-cli LLEN $q)"
+done
+
+echo ""
+echo "=== 7. Worker 日志最新记录（查看是否消费了测试消息） ==="
+docker logs --since=30s "$W" 2>&1 | tail -60
+
+echo ""
+echo "=== 8. Worker 内 Python 直接连接broker测试 ==="
+docker exec "$W" python - <<'PYEOF'
+import traceback
+try:
+    from worker_app.celery_app import celery_app
+    conn = celery_app.connection()
+    conn.ensure_connection(max_retries=2)
+    print("Worker celery broker connected:", conn.as_uri())
+    insp = celery_app.control.inspect(timeout=3)
+    reg = insp.registered()
+    print("Registered from worker inspect (via broker):")
+    for node, tasks in (reg or {}).items():
+        has_l = any('lipsync' in t for t in tasks)
+        print(f"  {node}: {len(tasks)} tasks, lipsync registered: {has_l}")
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+
+echo ""
+echo "=== 9. DB 查询最近10条lipsync_jobs状态 ==="
+docker exec "$A" python - <<'PYEOF' 2>&1
+import os, traceback
+db_url = os.environ.get("DATABASE_URL","")
+print("DATABASE_URL prefix:", (db_url[:50]+"...") if db_url else "(empty)")
+try:
+    # 通过现有代码路径
+    from app.db.session import SessionLocal
+    from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
+    db = SessionLocal()
+    jobs = db.query(LipsyncJobModel).order_by(LipsyncJobModel.created_at.desc()).limit(10).all()
+    print(f"Found {len(jobs)} recent lipsync jobs:")
+    for j in jobs:
+        err = getattr(j, 'error_message', '') or ''
+        t_id = getattr(j, 'celery_task_id', '') or ''
+        print(f"  id={j.id} status={j.status} mode={getattr(j,'mode','?')} "
+              f"created={j.created_at} celery_task_id={t_id} "
+              f"error={(err[:120]+'...') if len(err)>120 else err!r}")
+    db.close()
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+
+echo ""
+echo "=== 10. Worker 启动完整日志（前60行）==="
+docker logs "$W" 2>&1 | head -60
+
+echo ""
+echo "=== 11. lipsync_service.py 完整 create_job 相关段 ==="
+docker exec "$A" sed -n '170,240p' /app/apps/api/app/services/lipsync_service.py 2>&1
+
+echo ""
+echo "=== 12. celery_app task_routes / queue 配置（API侧）==="
+docker exec "$A" python - <<'PYEOF'
+from app.core.celery_app import celery_app
+print("task_routes:", getattr(celery_app.conf, 'task_routes', None))
+print("task_default_queue:", celery_app.conf.task_default_queue)
+print("task_queues:", celery_app.conf.task_queues)
+print("broker_url prefix:", celery_app.conf.broker_url[:60])
+print("result_backend prefix:", str(celery_app.conf.result_backend)[:60] if celery_app.conf.result_backend else None)
+PYEOF
+
+echo ""
+echo "=== 13. Worker celery_app queue 配置 ==="
+docker exec "$W" python - <<'PYEOF'
+from worker_app.celery_app import celery_app
+print("task_routes:", getattr(celery_app.conf, 'task_routes', None))
+print("task_default_queue:", celery_app.conf.task_default_queue)
+print("task_queues:", celery_app.conf.task_queues)
+print("include/imports count:", len(celery_app.conf.imports))
+PYEOF
+
+echo ""
+echo "#########################################################"
+echo "#  v2 诊断完成"
+echo "#########################################################"

infra/scripts/server-lipsync-diag-v3.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+# Lipsync 诊断脚本 v3 - 用独立文件避免heredoc问题
+set +e
+echo "#########################################################"
+echo "#  Lipsync 诊断 v3"
+echo "#  Date: $(date)"
+echo "#########################################################"
+
+W=$(docker ps --format '{{.Names}}' | grep -E 'worker' | head -1)
+A=$(docker ps --format '{{.Names}}' | grep -E 'api' | grep -v 'web' | head -1)
+R=$(docker ps --format '{{.Names}}' | grep -E 'redis' | head -1)
+NGINX=$(docker ps --format '{{.Names}}' | grep -iE 'nginx|web' | head -1)
+echo "Containers:"
+echo "  W=$W"
+echo "  A=$A"
+echo "  R=$R"
+echo "  NGINX=$NGINX"
+
+echo ""
+echo "=== A1. API 容器近 2000 行日志全部（看请求是否进来） ==="
+docker logs --tail=2000 "$A" 2>&1 | tail -300
+
+echo ""
+echo "=== A2. API容器启动命令 & 网络 ==="
+docker inspect "$A" --format 'Cmd={{.Config.Cmd}} Entrypoint={{.Config.Entrypoint}} NetworkMode={{.HostConfig.NetworkMode}}'
+docker inspect "$A" --format '{{range .NetworkSettings.Networks}}{{.NetworkID}} {{.IPAddress}}{{end}}'
+
+echo ""
+echo "=== A3. Nginx 容器（如果有）日志 ==="
+if [ -n "$NGINX" ]; then
+  docker logs --tail=200 "$NGINX" 2>&1 | grep -iE "lipsync|ai-avatar|error|upstream" | tail -100
+  echo "--- nginx config ---"
+  docker exec "$NGINX" cat /etc/nginx/conf.d/default.conf 2>&1 | head -80
+fi
+
+echo ""
+echo "=== B1. 写测试脚本到/tmp，再docker cp到容器里执行（避免heredoc问题） ==="
+cat > /tmp/api_diag.py << 'PYEOF'
+import sys, traceback, os
+print("=== PYTHON OK in API container, cwd=", os.getcwd())
+print("sys.path[0:3]:", sys.path[:3])
+print("PYTHONPATH:", os.environ.get("PYTHONPATH",""))
+try:
+    from app.tasks.lipsync_tts import tts_synthesize_and_submit
+    from app.core.celery_app import celery_app as api_app
+    print("api_app.main:", api_app.main)
+    print("api_app.conf.broker_url:", api_app.conf.broker_url)
+    print("task name:", tts_synthesize_and_submit.name)
+    print("task.app.main:", tts_synthesize_and_submit.app.main)
+    print("task in api_app.tasks:", tts_synthesize_and_submit.name in api_app.tasks)
+    print("api_app.conf.task_routes:", api_app.conf.task_routes)
+    print("api_app.conf.task_default_queue:", api_app.conf.task_default_queue)
+    # 测试broker连接
+    conn = api_app.connection()
+    conn.ensure_connection(max_retries=2)
+    print("broker connected OK:", conn.as_uri())
+    # 投递测试任务
+    result = tts_synthesize_and_submit.apply_async(
+        args=["diag-v3-job", "diag-user", "diag-voice", "hello v3 diag", 1.0, "neutral"],
+        queue="celery",
+    )
+    print("APPLY_ASYNC_OK id=", result.id, "name=", result.name, "queue=celery")
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+docker cp /tmp/api_diag.py "$A:/tmp/api_diag.py"
+echo "--- docker exec python /tmp/api_diag.py in API ---"
+docker exec "$A" python /tmp/api_diag.py 2>&1
+
+echo ""
+echo "=== B2. 投递3秒后查队列 ==="
+sleep 3
+for q in celery generation transcode; do
+    echo "  queue $q: $(docker exec "$R" redis-cli LLEN $q)"
+done
+echo "--- peek celery ---"
+docker exec "$R" redis-cli LRANGE celery 0 2 2>&1 | head -20
+
+echo ""
+echo "=== B3. 等10秒再查队列和Worker日志 ==="
+sleep 7
+for q in celery generation transcode; do
+    echo "  queue $q: $(docker exec "$R" redis-cli LLEN $q)"
+done
+echo "--- Worker logs since 20s ago ---"
+docker logs --since=20s "$W" 2>&1 | tail -50
+
+echo ""
+echo "=== C1. Worker侧执行诊断脚本 ==="
+cat > /tmp/worker_diag.py << 'PYEOF'
+import sys, os, traceback
+print("=== PYTHON OK in Worker container ===")
+try:
+    from worker_app.celery_app import celery_app
+    print("worker celery_app.main:", celery_app.main)
+    print("broker_url:", celery_app.conf.broker_url)
+    print("task_default_queue:", celery_app.conf.task_default_queue)
+    print("task_routes:", celery_app.conf.task_routes)
+    print("task_queues:", [(q.name, [b.name for b in q.bindings]) for q in (celery_app.conf.task_queues or [])])
+    # inspect via broker
+    insp = celery_app.control.inspect(timeout=3)
+    act = insp.active() or {}
+    reg = insp.registered() or {}
+    res = insp.reserved() or {}
+    for node, tasks in act.items():
+        print(f"ACTIVE on {node}: {len(tasks)} tasks")
+        for t in tasks:
+            print(f"  - {t.get('name')} args={str(t.get('args',''))[:60]}")
+    for node, tasks in res.items():
+        print(f"RESERVED on {node}: {len(tasks)} tasks")
+        for t in tasks:
+            print(f"  - {t.get('name')}")
+    print("registered lipsync count per node:")
+    for node, tasks in reg.items():
+        has = [t for t in tasks if 'lipsync' in t]
+        print(f"  {node}: total={len(tasks)}, lipsync={has}")
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+docker cp /tmp/worker_diag.py "$W:/tmp/worker_diag.py"
+docker exec "$W" python /tmp/worker_diag.py 2>&1
+
+echo ""
+echo "=== D1. DB最近20条lipsync jobs ==="
+cat > /tmp/db_diag.py << 'PYEOF'
+import sys, os, traceback
+print("=== DB diag in API container ===")
+print("DATABASE_URL prefix:", os.environ.get("DATABASE_URL","")[:60])
+try:
+    from app.db.session import SessionLocal
+    from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
+    from datetime import datetime, timezone
+    db = SessionLocal()
+    jobs = db.query(LipsyncJobModel).order_by(LipsyncJobModel.created_at.desc()).limit(20).all()
+    print(f"Found {len(jobs)} recent lipsync jobs:")
+    for j in jobs:
+        err = getattr(j, 'error_message', None) or ''
+        t_id = getattr(j, 'celery_task_id', None) or ''
+        c_at = j.created_at.isoformat() if j.created_at else '?'
+        print(f"  id={j.id} status={j.status} mode={'tts' if (j.voice_id and not j.audio_url) else 'audio'} "
+              f"created={c_at} celery_task_id={t_id} err={(err[:100]+'...') if len(err)>100 else err!r}")
+    db.close()
+except Exception as e:
+    traceback.print_exc()
+PYEOF
+docker cp /tmp/db_diag.py "$A:/tmp/db_diag.py"
+docker exec "$A" python /tmp/db_diag.py 2>&1
+
+echo ""
+echo "=== D2. API容器监听端口 ==="
+docker exec "$A" sh -c "ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || cat /proc/net/tcp | head" 2>&1 | head -20
+
+echo ""
+echo "=== E1. 外部HTTPS健康检查 ==="
+curl -sk --max-time 8 https://staging-api.xiaoxiajianji.com/api/v1/health 2>&1 | head -3
+echo ""
+curl -sk --max-time 8 https://staging-api.xiaoxiajianji.com/api/v1/lipsync/jobs 2>&1 | head -3
+
+echo ""
+echo "#########################################################"
+echo "#  v3 诊断完成"
+echo "#########################################################"

infra/scripts/server-lipsync-diag.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Lipsync staging 诊断脚本（在staging服务器本机执行）
+set +e
+echo "#########################################################"
+echo "#  Lipsync TTS async 诊断报告"
+echo "#  Host: $(hostname)"
+echo "#  Date: $(date)"
+echo "#########################################################"
+
+echo ""
+echo "=== 1. docker ps (worker/api/redis) ==="
+docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.CreatedAt}}' | grep -E "worker|api|redis|nginx|NAMES"
+
+W=$(docker ps --format '{{.Names}}' | grep -E 'worker' | head -1)
+A=$(docker ps --format '{{.Names}}' | grep -E 'api' | grep -v 'web' | head -1)
+R=$(docker ps --format '{{.Names}}' | grep -E 'redis' | head -1)
+
+echo ""
+echo "=== Worker container: $W ==="
+echo "=== API container: $A ==="
+echo "=== Redis container: $R ==="
+
+echo ""
+echo "=== 2. Worker container image + created time ==="
+docker inspect "$W" --format 'Image={{.Config.Image}} Created={{.Created}}'
+docker inspect "$A" --format 'Image={{.Config.Image}} Created={{.Created}}'
+
+echo ""
+echo "=== 3. Worker 启动日志：celery ready / registered / lipsync 相关 ==="
+docker logs --tail=500 "$W" 2>&1 | grep -iE "lipsync|tts_synth|synthesize|celery@|ready|registered|register|error|traceback|import|critical|not registered|Consumer|mingle|tasks" | tail -120
+
+echo ""
+echo "=== 4. celery -A inspect registered ==="
+docker exec "$W" celery -A worker_app.celery_app inspect registered 2>&1 | tail -80
+
+echo ""
+echo "=== 5. celery -A inspect active/reserved ==="
+echo "--- active ---"
+docker exec "$W" celery -A worker_app.celery_app inspect active 2>&1 | tail -30
+echo "--- reserved ---"
+docker exec "$W" celery -A worker_app.celery_app inspect reserved 2>&1 | tail -30
+
+echo ""
+echo "=== 6. Worker 内 Python: import lipsync_tts task，看绑定 ==="
+docker exec "$W" python -c "
+from worker_app.celery_app import celery_app
+print('worker celery_app.main:', celery_app.main)
+print('worker celery_app.conf.imports:')
+for m in celery_app.conf.imports:
+    print('  ', m)
+import app.tasks.lipsync_tts as m
+t = m.tts_synthesize_and_submit
+print('task name:', t.name)
+print('task.app.main:', t.app.main)
+print('task.app is worker celery_app:', t.app is celery_app)
+print('task.name in worker celery_app.tasks:', t.name in celery_app.tasks)
+" 2>&1
+
+echo ""
+echo "=== 7. API 容器日志：lipsync/celery/apply_async 相关错误 ==="
+docker logs --tail=800 "$A" 2>&1 | grep -iE "lipsync|tts_|celery|apply_async|NotRegistered|traceback|error.*task" | tail -120
+
+echo ""
+echo "=== 8. API 内 Python: import task 看绑定 + apply_async 试投递（不消费） ==="
+docker exec "$A" python -c "
+import traceback
+try:
+    from app.tasks.lipsync_tts import tts_synthesize_and_submit
+    from app.core.celery_app import celery_app as api_app
+    print('api celery_app.main:', api_app.main)
+    print('task name:', tts_synthesize_and_submit.name)
+    print('task.app.main:', tts_synthesize_and_submit.app.main)
+    print('task.app is api_app:', tts_synthesize_and_submit.app is api_app)
+    print('task.name in api_app.tasks:', tts_synthesize_and_submit.name in api_app.tasks)
+    # 测试 send_task 是否能路由（不发送真任务）
+    print('api_app tasks includes lipsync_tts.synthesize_and_submit:', 'lipsync_tts.synthesize_and_submit' in api_app.tasks)
+except Exception as e:
+    traceback.print_exc()
+" 2>&1
+
+echo ""
+echo "=== 9. Redis 队列长度 ==="
+for q in celery generation transcode; do
+    len=$(docker exec "$R" redis-cli LLEN $q 2>&1)
+    echo "  $q length: $len"
+done
+
+echo ""
+echo "=== 10. Redis celery 队列 peek（最多5条） ==="
+docker exec "$R" redis-cli LRANGE celery 0 4 2>&1 | head -40
+
+echo ""
+echo "=== 11. Worker 关键文件内容校验 ==="
+echo "--- worker_app/celery_app.py imports 段 ---"
+docker exec "$W" grep -n "lipsync\|imports\s*=\|app.tasks\|apps.api" /app/apps/worker/worker_app/celery_app.py 2>&1
+echo ""
+echo "--- lipsync_tts.py 前20行（应该是 shared_task） ---"
+docker exec "$W" head -25 /app/apps/api/app/tasks/lipsync_tts.py 2>&1
+
+echo ""
+echo "=== 12. API lipsync_service.py apply_async 上下文 ==="
+docker exec "$A" grep -n -B3 -A10 "apply_async\|tts_synthesize" /app/apps/api/app/services/lipsync_service.py 2>&1 | head -80
+
+echo ""
+echo "=== 13. Worker /app 目录结构（app/tasks） ==="
+docker exec "$W" ls -la /app/apps/api/app/tasks/ 2>&1 | head -30
+echo ""
+echo "--- /app/apps/api/app/__init__.py? ---"
+docker exec "$W" ls -la /app/apps/api/app/__init__.py /app/apps/api/app/core/__init__.py /app/apps/api/app/api/__init__.py 2>&1
+echo ""
+echo "--- PYTHONPATH inside worker ---"
+docker exec "$W" env | grep PYTHONPATH
+
+echo ""
+echo "#########################################################"
+echo "#  诊断完成"
+echo "#########################################################"

packages/__init__.py

@@ -1 +0,0 @@
-# xiaoxia-saas shared packages namespace

packages/adapters/__init__.py

@@ -1 +0,0 @@
-# adapter implementations namespace

tests/unit/test_lipsync_speed_optimization.py

@@ -143,8 +143,8 @@ class TestCreateJobAsyncTTS:
         assert args[2] == "v-1"  # voice_id
         assert args[3] == "测试文本"  # script_text
 
-    def test_tts_mode_celery_dispatch_failure_marks_job_failed(self):
-        """Celery dispatch 失败时，job 标为 failed 并写入 error_message，前端轮询能直接看到错误."""
+    def test_tts_mode_celery_dispatch_failure_still_creates_job(self):
+        """Celery dispatch 失败时，job 记录已创建，状态保持 tts_processing."""
         svc, client, cosy = _make_service_with_mocks()
 
         with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
@@ -157,11 +157,9 @@ class TestCreateJobAsyncTTS:
                 script_text="测试文本",
             )
 
-        # job 已创建且状态标为 failed
+        # job 已创建
         assert job is not None
-        assert job.status == "failed"
-        assert "Celery 任务投递失败" in job.error_message
-        assert job.error_code == "AsyncDispatchFailed"
+        assert job.status == "tts_processing"
         # MediaKit 未被调用
         client.submit_lipsync.assert_not_called()
 

tests/unit/test_lipsync_tts.py

@@ -73,16 +73,12 @@ def _apply_all_patches(
     lipsync_tts.run() 在函数体内部懒 import 多个模块，通过 sys.modules 注入
     伪造包路径避免真实导入；对存在的模块用 patch() 替换返回值/side_effect。
     """
-    # SessionLocal 通过懒探测获取（Worker 用 worker_app.db，API 用 app.db），
-    # 测试环境里两个模块都能被真实导入，必须同时 mock 保证用的是 fake session。
-    fake_app_db = ModuleType("app.db")
-    fake_worker_db = ModuleType("worker_app.db")
+    fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
     session, factory = _build_session(job)
-    fake_app_db.SessionLocal = factory
-    fake_worker_db.SessionLocal = factory
+    fake_db_mod.SessionLocal = factory
 
     patches = [
-        patch.dict(sys.modules, {"app.db": fake_app_db, "worker_app.db": fake_worker_db}),
+        patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.database": fake_db_mod}),
         patch(
             "app.tasks.lipsync_tts._sign_media_url",
             side_effect=lambda url: url + "?signed" if url else url,
@@ -293,11 +289,9 @@ class TestTtsSynthesizeAndSubmit:
         from app.tasks.lipsync_tts import tts_synthesize_and_submit
 
         job = _make_fake_job()
-        fake_app_db = ModuleType("app.db")
-        fake_worker_db = ModuleType("worker_app.db")
+        fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
         session, factory = _build_session(job)
-        fake_app_db.SessionLocal = factory
-        fake_worker_db.SessionLocal = factory
+        fake_db_mod.SessionLocal = factory
 
         # CosyVoiceService 在 __init__ 抛 RuntimeError（非 CosyVoiceError/ValueError）
         fake_cosy_mod = ModuleType("packages.application.cosyvoice_service")
@@ -315,8 +309,7 @@ class TestTtsSynthesizeAndSubmit:
         with patch.dict(
             sys.modules,
             {
-                "app.db": fake_app_db,
-                "worker_app.db": fake_worker_db,
+                "packages.adapters.sqlalchemy_impl.database": fake_db_mod,
                 "packages.application.cosyvoice_service": fake_cosy_mod,
             },
         ):