Compare commits

..

4 Commits

Author SHA1 Message Date
xiaoxia-agent 643183ecb0 debug(web): use docker run instead of docker create
Debug: Web container crash diag / Web Crash Diagnostics (push) Successful in 54s
2026-09-11 10:16:05 +08:00
xiaoxia-agent 3e7273b8cd debug(web): deeper diag of nginx entrypoint crash
Debug: Web container crash diag / Web Crash Diagnostics (push) Successful in 49s
2026-09-11 10:12:15 +08:00
xiaoxia-agent 5659fddc97 debug(web): fix runs-on + add web diag script
Debug: Web container crash diag / Web Crash Diagnostics (push) Successful in 19s
2026-09-11 10:07:21 +08:00
xiaoxia-agent 59095f2f79 debug(web): nginx crash diag
Debug: Web container crash diag / web-diag (push) Failing after 3s
2026-09-11 10:02:47 +08:00
13 changed files with 80 additions and 215 deletions
+37
View File
@@ -0,0 +1,37 @@
name: "Debug: Web container crash diag"
on:
push:
branches:
- 'debug/web-crash-diag'
workflow_dispatch:
jobs:
web-diag:
name: Web Crash Diagnostics
runs-on: runtime-builder
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Setup SSH
shell: bash
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -euo pipefail
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
mkdir -p ~/.ssh && chmod 700 ~/.ssh
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
H=47.98.113.167; P=22222
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H "echo SSH_OK; hostname; date"
- name: Run web diag
shell: bash
run: |
set -euo pipefail
H=47.98.113.167; P=22222
scp -P $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no \
infra/scripts/server-web-diag.sh root@$H:/tmp/server-web-diag.sh
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H \
"bash /tmp/server-web-diag.sh 2>&1"
-60
View File
@@ -1,60 +0,0 @@
name: "Debug: Web container v2 (mount conflict)"
on:
push:
branches: [debug/web-crash-v2]
workflow_dispatch:
jobs:
web-diag:
runs-on: runtime-builder
timeout-minutes: 10
steps:
- name: Setup SSH and diagnose
shell: bash
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -x
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
mkdir -p ~/.ssh && chmod 700 ~/.ssh
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
H=47.98.113.167; P=22222
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H 'bash -s' <<'REMOTE'
set -x
echo "=== Current staging containers ==="
docker ps -a --filter name=xiaoxia-*-staging --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
echo ""
echo "=== Web container logs (current/current-rolledback) ==="
docker logs xiaoxia-web-staging 2>&1 | tail -40
echo ""
echo "=== Web inspect: env & mounts ==="
docker inspect xiaoxia-web-staging --format 'Entrypoint: {{.Config.Entrypoint}} Cmd: {{.Config.Cmd}}'
docker inspect xiaoxia-web-staging --format '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' | grep -E "APP_ENV|VERSION"
echo "Mounts:"
docker inspect xiaoxia-web-staging --format '{{range .Mounts}}{{.Type}} {{.Source}} -> {{.Destination}} (rw={{.RW}}){{"\n"}}{{end}}'
echo ""
echo "=== Reproduce: rm on read-only bind mount ==="
docker run --rm --name nginx-ro-test \
-v /var/lib/xiaoxia-saas-staging/nginx-staging.conf:/etc/nginx/conf.d/default.conf:ro \
git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/xiaoxia-saas-web:387514c \
sh -c '
set -x
echo "Before:"
ls -la /etc/nginx/conf.d/
echo "Try rm (as entrypoint does):"
rm -f /etc/nginx/conf.d/default.conf
echo "rm exitcode=$?"
echo "After rm:"
ls -la /etc/nginx/conf.d/
echo "Test ln:"
ln -s /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf
echo "ln exitcode=$?"
ls -la /etc/nginx/conf.d/
echo "nginx -t:"
nginx -t 2>&1
' 2>&1
echo ""
echo "=== Also test with NEW fixed image (9c0d4b1 if present) ==="
docker images | grep xiaoxia-saas-web | head -5
REMOTE
-28
View File
@@ -1,28 +0,0 @@
name: "E2E lipsync verify v3"
on:
push:
branches: [debug/e2e-lipsync]
workflow_dispatch:
jobs:
e2e:
runs-on: runtime-builder
timeout-minutes: 10
steps:
- name: Setup SSH
shell: bash
env:
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
run: |
set -eux
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
mkdir -p ~/.ssh && chmod 700 ~/.ssh
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -p 22222 -H 47.98.113.167 >> ~/.ssh/known_hosts 2>/dev/null
- name: Run
shell: bash
run: |
set -x
echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSAxLiBSZWNlbnQgbGlwc3luYyBqb2JzIChjb3JyZWN0IG1vZGVsIHBhdGgpID09PSIKZG9ja2VyIGV4ZWMgeGlhb3hpYS1hcGktc3RhZ2luZyBzaCAtYyAiY2QgL2FwcCAmJiBweXRob24gLWMgXCIKZnJvbSBhcHAuZGIgaW1wb3J0IFNlc3Npb25Mb2NhbApmcm9tIHBhY2thZ2VzLmFkYXB0ZXJzLnNxbGFsY2hlbXlfaW1wbC5tb2RlbHMgaW1wb3J0IExpcHN5bmNKb2JNb2RlbCBhcyBMSgpkYiA9IFNlc3Npb25Mb2NhbCgpCnRvdGFsID0gZGIucXVlcnkoTEopLmNvdW50KCkKcHJpbnQoZidUb3RhbCBsaXBzeW5jIGpvYnM6IHt0b3RhbH0nKQpqb2JzID0gZGIucXVlcnkoTEopLm9yZGVyX2J5KExKLmNyZWF0ZWRfYXQuZGVzYygpKS5saW1pdCgxMCkuYWxsKCkKZm9yIGogaW4gam9iczoKICAgIGVyciA9IChqLmVycm9yX21lc3NhZ2Ugb3IgJycpWzoxMDBdCiAgICBwcmludChmJyAgaWQ9e2ouaWR9IHVzZXI9e2oudXNlcl9pZH0gc3RhdHVzPXtqLnN0YXR1c30gZXJyX2NvZGU9e2ouZXJyb3JfY29kZX0gZXJyPXtlcnJ9JykKZGIuY2xvc2UoKQpcIiIgMj4mMSB8IHRhaWwgLTMwCmVjaG8gIiIKZWNobyAiPT09IDIuIFRlc3QgdXNlcnMgPT09Igpkb2NrZXIgZXhlYyB4aWFveGlhLWFwaS1zdGFnaW5nIHNoIC1jICJjZCAvYXBwICYmIHB5dGhvbiAtYyBcIgpmcm9tIGFwcC5kYiBpbXBvcnQgU2Vzc2lvbkxvY2FsCmZyb20gcGFja2FnZXMuYWRhcHRlcnMuc3FsYWxjaGVteV9pbXBsLm1vZGVscyBpbXBvcnQgVXNlck1vZGVsIGFzIFUKZGIgPSBTZXNzaW9uTG9jYWwoKQp1c2VycyA9IGRiLnF1ZXJ5KFUpLm9yZGVyX2J5KFUuY3JlYXRlZF9hdC5kZXNjKCkpLmxpbWl0KDUpLmFsbCgpCmZvciB1IGluIHVzZXJzOgogICAgcHJpbnQoZicgIGlkPXt1LmlkfSBlbWFpbD17dS5lbWFpbH0gcGhvbmU9e2dldGF0dHIodSxcInBob25lX251bWJlclwiLE5vbmUpfSBhY3RpdmU9e2dldGF0dHIodSxcImlzX2FjdGl2ZVwiLE5vbmUpfScpCmRiLmNsb3NlKCkKXCIiIDI+JjEgfCB0YWlsIC0xNQplY2hvICIiCmVjaG8gIj09PSAzLiBBUEkgd29ya2VyIGxvZ3M6IGFueSBsaXBzeW5jIHRhc2sgZXhlY3V0aW9uIGhpc3Rvcnk/ID09PSIKZG9ja2VyIGxvZ3MgeGlhb3hpYS13b3JrZXItc3RhZ2luZyAyPiYxIHwgZ3JlcCAtaUUgImxpcHN5bmNfdHRzfHN5bnRoZXNpemVfYW5kX3N1Ym1pdHxBc3luY0Rpc3BhdGNoRmFpbGVkfHR0c19wcm9jZXNzaW5nfHR0c19wcm94eSIgfCB0YWlsIC0yMAplY2hvICIiCmVjaG8gIj09PSA0LiBQdWJsaWMgR0VUIC9oZWFsdGggd2l0aCBHRVQgbWV0aG9kID09PSIKY3VybCAtc2YgLS1tYXgtdGltZSAxMCBodHRwczovL3N0YWdpbmctYXBpLnhpYW94aWFqaWFuamkuY29tL2hlYWx0aCAyPiYxIHwgaGVhZCAtNQplY2hvICIiCmVjaG8gIj09PSA1LiBQdWJsaWMgd2ViIHJvb3QgPT09IgpjdXJsIC1zZiAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy54aWFveGlhamlhbmppLmNvbS8gMj4mMSB8IGhlYWQgLTUK' | base64 -d > /tmp/e2e.sh
chmod +x /tmp/e2e.sh
ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/e2e.sh
+4 -10
View File
@@ -211,18 +211,12 @@ class LipsyncService:
normalize_emotion(emotion),
)
)
except Exception as exc:
# 投递失败时立即把 job 标成 failed 并写入 error_message
# 前端轮询时能直接看到失败原因,不会无限卡在 tts_processing。
logger.exception(
"Celery 任务提交失败,TTS 任务已创建但未触发执行: job_id=%s err=%s",
except Exception:
logger.warning(
"Celery 任务提交失败,TTS 任务已创建但未触发执行: %s",
job_id,
exc,
exc_info=True,
)
job.status = "failed"
job.error_message = f"Celery 任务投递失败: {exc}"
job.error_code = "AsyncDispatchFailed"
job.updated_at = datetime.now(timezone.utc)
else:
# 2b. 直接音频模式:同步签名并提交 MediaKit
video_url = self._sign_media_url(video_url)
+1 -11
View File
@@ -76,21 +76,11 @@ def tts_synthesize_and_submit(
from app.services.mediakit_client import MediaKitError, get_mediakit_client
from sqlalchemy.orm import Session as DBSession
from packages.adapters.sqlalchemy_impl.database import SessionLocal
from packages.adapters.sqlalchemy_impl.models import LipsyncJobModel
from packages.application.cosyvoice_service import CosyVoiceError, CosyVoiceService
from packages.shared.url_security import safe_download_bytes
# SessionLocal 获取:
# - API 容器:app.db.SessionLocal(环境变量完整,导入即建引擎)
# - Worker 容器:worker_app.db.SessionLocalWorker 自己的 settings 初始化引擎)
# API 侧没有 worker_app 模块 → ImportError 直接回退;
# Worker 侧 app.db 会因缺少 API 专有环境变量抛 pydantic ValidationError
# 此时也要回退到 worker_app.db。
try:
from worker_app.db import SessionLocal # type: ignore
except Exception: # noqa: BLE001
from app.db import SessionLocal # type: ignore
db: DBSession = SessionLocal()
try:
job = (
-2
View File
@@ -34,8 +34,6 @@ celery_app.conf.imports = (
"worker_app.tasks.tts_synthesis",
"worker_app.tasks.batch_download",
"worker_app.tasks.duplication_check",
# #1798 AI 数字人渲染:必须在 Worker 实例上注册同名任务,否则消息无人消费(渲染卡 0%)
"worker_app.tasks.ai_avatar_render",
"worker_app.tasks._startup",
"apps.worker.video_processing.dedup",
"worker_app.tasks.cleanup",
@@ -1,48 +0,0 @@
"""AI 数字人渲染任务 — Worker 侧 Celery 任务注册.
#1798 渲染进度卡在 0% 的根因:渲染任务定义在 API 侧(`app.tasks.ai_avatar_render`),
装饰在 API 自己的 Celery 实例(`xiaoxia-saas-api`)上;而 Worker 用的是
`worker_app.celery_app` 实例,`conf.imports` 从未导入该任务,Worker 的任务
注册表里没有 `ai_avatar_render.execute`,消息被路由到默认 `celery` 队列后
无人消费,任务永远停在 0%
修复:在 Worker 侧用 `worker_app.celery_app` 注册同名任务,直接调用与 API
服务一致的 `AiAvatarRenderService.execute_render` 核心管线(业务逻辑在
`apps.api.app.services`worker 镜像已复制 `apps/api/app`)。任务名保持
`ai_avatar_render.execute`,与 API 生产端 `.delay()` 的消息路由一致;未在
task_routes 显式配置,走默认 `celery` 队列,由 transcode worker 消费。
"""
from __future__ import annotations
import logging
from worker_app.celery_app import celery_app
from worker_app.db import SessionLocal
logger = logging.getLogger(__name__)
@celery_app.task(bind=True, name="ai_avatar_render.execute", max_retries=2)
def execute_ai_avatar_render(self, job_id: str) -> dict:
"""执行 AI 数字人渲染管线(Worker 侧入口).
进度由 service 直接写入 DBAiAvatarRenderJob.progress:
0→5→20→40→80→90→95→100),API 通过轮询 progress 字段展示。
"""
logger.info("开始执行渲染任务: %s", job_id)
self.update_state(state="PROCESSING", meta={"progress": 0, "job_id": job_id})
session = SessionLocal()
try:
from app.services.ai_avatar_render_service import AiAvatarRenderService
service = AiAvatarRenderService(session)
service.execute_render(job_id)
return {"status": "completed", "job_id": job_id}
except Exception as exc:
logger.exception("渲染任务执行异常 [%s]: %s", job_id, exc)
self.update_state(state="FAILED", meta={"progress": 0, "error": str(exc)})
raise
finally:
session.close()
+4 -35
View File
@@ -1,49 +1,18 @@
#!/bin/sh
# Select nginx config based on APP_ENV (staging/production).
#
# 两种运行模式:
# 1. CI/CD 部署(staging/production):部署脚本通过 `-v 宿主机文件:/etc/nginx/conf.d/default.conf:ro`
# 把宿主机生成的带 resolver/docker upstream 的配置 bind mount 进来,entrypoint 不应改动。
# bind mount 的文件是 readonly 的,rm 会报 EBUSY ("Resource busy"),直接 exec nginx 即可。
# 2. 本地 docker-compose / 直接 `docker run`(无外部挂载):镜像烤入了 nginx-staging.conf 与
# nginx-production.conf 到 /etc/nginx/entrypoint 根据 APP_ENV 把 default.conf 换成正确的 symlink。
#
# 策略:
# - 如果 /etc/nginx/conf.d/default.conf 已经是指向目标 conf 的 symlink,什么都不做;
# - 否则尝试 rm -f 再 ln -srm 失败说明是外部 bind mount(已有正确配置),不阻塞启动;
# - 兜底:只要 conf.d 目录里有 .conf 文件(含 bind mount 来的),就直接启动 nginx。
# Both configs are baked into the image at well-known paths.
# nginx reads config only at startup, so symlink before exec.
set -e
NGINX_CONF_DIR="/etc/nginx/conf.d"
TARGET_CONF=""
case "${APP_ENV:-production}" in
staging)
TARGET_CONF="/etc/nginx/nginx-staging.conf"
ln -sf /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
;;
*)
TARGET_CONF="/etc/nginx/nginx-production.conf"
ln -sf /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
;;
esac
DEFAULT_CONF="$NGINX_CONF_DIR/default.conf"
# 1. 已经是正确的 symlink:直接启动
if [ -L "$DEFAULT_CONF" ] && [ "$(readlink "$DEFAULT_CONF" 2>/dev/null)" = "$TARGET_CONF" ]; then
exec nginx -g "daemon off;"
fi
# 2. 尝试替换为目标 symlink(无 bind mount 的场景)
# 若 rm 失败(bind mount readonlyEBUSY/EPERM),则认为外部已注入配置,不阻塞。
rm -f "$DEFAULT_CONF" 2>/dev/null || true
if [ -f "$TARGET_CONF" ] && [ ! -e "$DEFAULT_CONF" ]; then
ln -s "$TARGET_CONF" "$DEFAULT_CONF" 2>/dev/null || true
fi
# 3. 兜底:至少要有一个 .conf 文件,否则 nginx 起不来
if ! ls "$NGINX_CONF_DIR"/*.conf >/dev/null 2>&1; then
echo "ERROR: no nginx config found in $NGINX_CONF_DIR (tried $TARGET_CONF and external bind mount)" >&2
exit 1
fi
exec nginx -g "daemon off;"
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
set +e
echo "=== 1. Check base nginx image conf.d ==="
docker run --rm --entrypoint sh git.xiaoxiajianji.com/xiaoxia/base/nginx:alpine -c \
"ls -la /etc/nginx/conf.d/ 2>&1; echo '---'; file /etc/nginx/conf.d/default.conf 2>&1; echo '---'; ls -la /etc/nginx/nginx*.conf 2>&1"
echo ""
echo "=== 2. Check staging web image (currently running) ==="
WEB_IMG=$(docker inspect xiaoxia-web-staging --format '{{.Config.Image}}' 2>/dev/null)
echo "Staging web image: $WEB_IMG"
docker run --rm --entrypoint sh "$WEB_IMG" -c \
"ls -la /etc/nginx/conf.d/ 2>&1; echo '---'; cat /docker-entrypoint.sh 2>&1; echo '---'; ls -la /etc/nginx/nginx-*.conf 2>&1"
echo ""
echo "=== 3. Run entrypoint with APP_ENV=staging to reproduce ==="
docker run --rm -e APP_ENV=staging --entrypoint sh "$WEB_IMG" -c \
"set -x; sh -x /docker-entrypoint.sh 2>&1 | head -30"
echo ""
echo "=== 4. What if we rm -f default.conf first then ln? ==="
docker run --rm -e APP_ENV=staging --entrypoint sh "$WEB_IMG" -c \
"rm -f /etc/nginx/conf.d/default.conf && ln -sf /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf && ls -la /etc/nginx/conf.d/default.conf && echo OK"
echo ""
echo "=== 5. Check running web logs for crash reason ==="
docker logs xiaoxia-web-staging 2>&1 | tail -30
echo ""
echo "=== Done ==="
-1
View File
@@ -1 +0,0 @@
# xiaoxia-saas shared packages namespace
-1
View File
@@ -1 +0,0 @@
# adapter implementations namespace
@@ -143,8 +143,8 @@ class TestCreateJobAsyncTTS:
assert args[2] == "v-1" # voice_id
assert args[3] == "测试文本" # script_text
def test_tts_mode_celery_dispatch_failure_marks_job_failed(self):
"""Celery dispatch 失败时,job 标为 failed 并写入 error_message,前端轮询能直接看到错误."""
def test_tts_mode_celery_dispatch_failure_still_creates_job(self):
"""Celery dispatch 失败时,job 记录已创建,状态保持 tts_processing."""
svc, client, cosy = _make_service_with_mocks()
with patch("app.services.lipsync_service.tts_synthesize_and_submit") as mock_task:
@@ -157,11 +157,9 @@ class TestCreateJobAsyncTTS:
script_text="测试文本",
)
# job 已创建且状态标为 failed
# job 已创建
assert job is not None
assert job.status == "failed"
assert "Celery 任务投递失败" in job.error_message
assert job.error_code == "AsyncDispatchFailed"
assert job.status == "tts_processing"
# MediaKit 未被调用
client.submit_lipsync.assert_not_called()
+6 -13
View File
@@ -73,16 +73,12 @@ def _apply_all_patches(
lipsync_tts.run() 在函数体内部懒 import 多个模块,通过 sys.modules 注入
伪造包路径避免真实导入;对存在的模块用 patch() 替换返回值/side_effect。
"""
# SessionLocal 通过懒探测获取(Worker 用 worker_app.dbAPI 用 app.db),
# 测试环境里两个模块都能被真实导入,必须同时 mock 保证用的是 fake session。
fake_app_db = ModuleType("app.db")
fake_worker_db = ModuleType("worker_app.db")
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
session, factory = _build_session(job)
fake_app_db.SessionLocal = factory
fake_worker_db.SessionLocal = factory
fake_db_mod.SessionLocal = factory
patches = [
patch.dict(sys.modules, {"app.db": fake_app_db, "worker_app.db": fake_worker_db}),
patch.dict(sys.modules, {"packages.adapters.sqlalchemy_impl.database": fake_db_mod}),
patch(
"app.tasks.lipsync_tts._sign_media_url",
side_effect=lambda url: url + "?signed" if url else url,
@@ -293,11 +289,9 @@ class TestTtsSynthesizeAndSubmit:
from app.tasks.lipsync_tts import tts_synthesize_and_submit
job = _make_fake_job()
fake_app_db = ModuleType("app.db")
fake_worker_db = ModuleType("worker_app.db")
fake_db_mod = ModuleType("packages.adapters.sqlalchemy_impl.database")
session, factory = _build_session(job)
fake_app_db.SessionLocal = factory
fake_worker_db.SessionLocal = factory
fake_db_mod.SessionLocal = factory
# CosyVoiceService 在 __init__ 抛 RuntimeError(非 CosyVoiceError/ValueError
fake_cosy_mod = ModuleType("packages.application.cosyvoice_service")
@@ -315,8 +309,7 @@ class TestTtsSynthesizeAndSubmit:
with patch.dict(
sys.modules,
{
"app.db": fake_app_db,
"worker_app.db": fake_worker_db,
"packages.adapters.sqlalchemy_impl.database": fake_db_mod,
"packages.application.cosyvoice_service": fake_cosy_mod,
},
):