Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4fcedfd5a9 | |||
| cc335673a9 | |||
| 029267cb46 |
@@ -8,53 +8,23 @@ jobs:
|
||||
runs-on: runtime-builder
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Setup SSH and diagnose
|
||||
- name: Setup SSH
|
||||
shell: bash
|
||||
env:
|
||||
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
|
||||
run: |
|
||||
set -x
|
||||
set -eux
|
||||
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
|
||||
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
H=47.98.113.167; P=22222
|
||||
ssh-keyscan -p $P -H $H >> ~/.ssh/known_hosts 2>/dev/null
|
||||
ssh -p $P -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@$H 'bash -s' <<'REMOTE'
|
||||
- name: Run diag via base64
|
||||
shell: bash
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Current staging containers ==="
|
||||
docker ps -a --filter name=xiaoxia-*-staging --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
|
||||
echo ""
|
||||
echo "=== Web container logs (current/current-rolledback) ==="
|
||||
docker logs xiaoxia-web-staging 2>&1 | tail -40
|
||||
echo ""
|
||||
echo "=== Web inspect: env & mounts ==="
|
||||
docker inspect xiaoxia-web-staging --format 'Entrypoint: {{.Config.Entrypoint}} Cmd: {{.Config.Cmd}}'
|
||||
docker inspect xiaoxia-web-staging --format '{{range .Config.Env}}{{.}}{{"\n"}}{{end}}' | grep -E "APP_ENV|VERSION"
|
||||
echo "Mounts:"
|
||||
docker inspect xiaoxia-web-staging --format '{{range .Mounts}}{{.Type}} {{.Source}} -> {{.Destination}} (rw={{.RW}}){{"\n"}}{{end}}'
|
||||
echo ""
|
||||
echo "=== Reproduce: rm on read-only bind mount ==="
|
||||
docker run --rm --name nginx-ro-test \
|
||||
-v /var/lib/xiaoxia-saas-staging/nginx-staging.conf:/etc/nginx/conf.d/default.conf:ro \
|
||||
git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/xiaoxia-saas-web:387514c \
|
||||
sh -c '
|
||||
set -x
|
||||
echo "Before:"
|
||||
ls -la /etc/nginx/conf.d/
|
||||
echo "Try rm (as entrypoint does):"
|
||||
rm -f /etc/nginx/conf.d/default.conf
|
||||
echo "rm exitcode=$?"
|
||||
echo "After rm:"
|
||||
ls -la /etc/nginx/conf.d/
|
||||
echo "Test ln:"
|
||||
ln -s /etc/nginx/nginx-staging.conf /etc/nginx/conf.d/default.conf
|
||||
echo "ln exitcode=$?"
|
||||
ls -la /etc/nginx/conf.d/
|
||||
echo "nginx -t:"
|
||||
nginx -t 2>&1
|
||||
' 2>&1
|
||||
echo ""
|
||||
echo "=== Also test with NEW fixed image (9c0d4b1 if present) ==="
|
||||
docker images | grep xiaoxia-saas-web | head -5
|
||||
REMOTE
|
||||
echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSBDdXJyZW50IHN0YWdpbmcgY29udGFpbmVycyA9PT0iCmRvY2tlciBwcyAtYSAtLWZpbHRlciBuYW1lPXhpYW94aWEtKi1zdGFnaW5nIC0tZm9ybWF0ICJ0YWJsZSB7ey5OYW1lc319XHR7ey5TdGF0dXN9fVx0e3suSW1hZ2V9fSIKZWNobyAiIgplY2hvICI9PT0gV2ViIGNvbnRhaW5lciBsb2dzIChjdXJyZW50L3JvbGxlZC1iYWNrKSA9PT0iCmRvY2tlciBsb2dzIHhpYW94aWEtd2ViLXN0YWdpbmcgMj4mMSB8IHRhaWwgLTQwCmVjaG8gIiIKZWNobyAiPT09IFdlYiBpbnNwZWN0OiBlbnYgJiBtb3VudHMgPT09Igpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdlYi1zdGFnaW5nIC0tZm9ybWF0ICdFbnRyeXBvaW50OiB7ey5Db25maWcuRW50cnlwb2ludH19IENtZDoge3suQ29uZmlnLkNtZH19Jwpkb2NrZXIgaW5zcGVjdCB4aWFveGlhLXdlYi1zdGFnaW5nIC0tZm9ybWF0ICd7e3JhbmdlIC5Db25maWcuRW52fX17ey59fXt7IlxuIn19e3tlbmR9fScgfCBncmVwIC1FICJBUFBfRU5WfFZFUlNJT04iCmVjaG8gIk1vdW50czoiCmRvY2tlciBpbnNwZWN0IHhpYW94aWEtd2ViLXN0YWdpbmcgLS1mb3JtYXQgJ3t7cmFuZ2UgLk1vdW50c319e3suVHlwZX19IHt7LlNvdXJjZX19IC0+IHt7LkRlc3RpbmF0aW9ufX0gKHJ3PXt7LlJXfX0pe3siXG4ifX17e2VuZH19JwplY2hvICIiCmVjaG8gIj09PSBSZXByb2R1Y2U6IHJtIG9uIHJlYWQtb25seSBiaW5kIG1vdW50IChpbWFnZSAzODc1MTRjKSA9PT0iCmRvY2tlciBydW4gLS1ybSAtLW5hbWUgbmdpbngtcm8tdGVzdCBcCiAgLXYgL3Zhci9saWIveGlhb3hpYS1zYWFzLXN0YWdpbmcvbmdpbngtc3RhZ2luZy5jb25mOi9ldGMvbmdpbngvY29uZi5kL2RlZmF1bHQuY29uZjpybyBcCiAgZ2l0LnhpYW94aWFqaWFuamkuY29tL3hpYW94aWEveGlhb3hpYS1zYWFzL3hpYW94aWEtc2Fhcy13ZWI6Mzg3NTE0YyBcCiAgL2Jpbi9zaCAtYyAnc2V0IC14OyBlY2hvIEJFRk9SRTsgbHMgLWxhIC9ldGMvbmdpbngvY29uZi5kLzsgcm0gLWYgL2V0Yy9uZ2lueC9jb25mLmQvZGVmYXVsdC5jb25mOyBlY2hvICJybSBleGl0PSQ/IjsgZWNobyBBRlRFUjsgbHMgLWxhIC9ldGMvbmdpbngvY29uZi5kLycgMj4mMQplY2hvICIiCmVjaG8gIj09PSBMb2NhbCB3ZWIgaW1hZ2VzIGF2YWlsYWJsZSA9PT0iCmRvY2tlciBpbWFnZXMgfCBncmVwIHhpYW94aWEtc2Fhcy13ZWIgfCBoZWFkIC01Cg==' | base64 -d > /tmp/diag.sh
|
||||
chmod +x /tmp/diag.sh
|
||||
cat /tmp/diag.sh
|
||||
ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/diag.sh
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
name: "E2E lipsync verify v3"
|
||||
on:
|
||||
push:
|
||||
branches: [debug/e2e-lipsync]
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
e2e:
|
||||
runs-on: runtime-builder
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Setup SSH
|
||||
shell: bash
|
||||
env:
|
||||
STAGING_SSH_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
|
||||
run: |
|
||||
set -eux
|
||||
which ssh || (apt-get update -qq && apt-get install -y -qq openssh-client)
|
||||
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
printf "%s" "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/id_rsa
|
||||
ssh-keyscan -p 22222 -H 47.98.113.167 >> ~/.ssh/known_hosts 2>/dev/null
|
||||
- name: Run
|
||||
shell: bash
|
||||
run: |
|
||||
set -x
|
||||
echo 'IyEvYmluL2Jhc2gKc2V0IC14CmVjaG8gIj09PSAxLiBSZWNlbnQgbGlwc3luYyBqb2JzIChjb3JyZWN0IG1vZGVsIHBhdGgpID09PSIKZG9ja2VyIGV4ZWMgeGlhb3hpYS1hcGktc3RhZ2luZyBzaCAtYyAiY2QgL2FwcCAmJiBweXRob24gLWMgXCIKZnJvbSBhcHAuZGIgaW1wb3J0IFNlc3Npb25Mb2NhbApmcm9tIHBhY2thZ2VzLmFkYXB0ZXJzLnNxbGFsY2hlbXlfaW1wbC5tb2RlbHMgaW1wb3J0IExpcHN5bmNKb2JNb2RlbCBhcyBMSgpkYiA9IFNlc3Npb25Mb2NhbCgpCnRvdGFsID0gZGIucXVlcnkoTEopLmNvdW50KCkKcHJpbnQoZidUb3RhbCBsaXBzeW5jIGpvYnM6IHt0b3RhbH0nKQpqb2JzID0gZGIucXVlcnkoTEopLm9yZGVyX2J5KExKLmNyZWF0ZWRfYXQuZGVzYygpKS5saW1pdCgxMCkuYWxsKCkKZm9yIGogaW4gam9iczoKICAgIGVyciA9IChqLmVycm9yX21lc3NhZ2Ugb3IgJycpWzoxMDBdCiAgICBwcmludChmJyAgaWQ9e2ouaWR9IHVzZXI9e2oudXNlcl9pZH0gc3RhdHVzPXtqLnN0YXR1c30gZXJyX2NvZGU9e2ouZXJyb3JfY29kZX0gZXJyPXtlcnJ9JykKZGIuY2xvc2UoKQpcIiIgMj4mMSB8IHRhaWwgLTMwCmVjaG8gIiIKZWNobyAiPT09IDIuIFRlc3QgdXNlcnMgPT09Igpkb2NrZXIgZXhlYyB4aWFveGlhLWFwaS1zdGFnaW5nIHNoIC1jICJjZCAvYXBwICYmIHB5dGhvbiAtYyBcIgpmcm9tIGFwcC5kYiBpbXBvcnQgU2Vzc2lvbkxvY2FsCmZyb20gcGFja2FnZXMuYWRhcHRlcnMuc3FsYWxjaGVteV9pbXBsLm1vZGVscyBpbXBvcnQgVXNlck1vZGVsIGFzIFUKZGIgPSBTZXNzaW9uTG9jYWwoKQp1c2VycyA9IGRiLnF1ZXJ5KFUpLm9yZGVyX2J5KFUuY3JlYXRlZF9hdC5kZXNjKCkpLmxpbWl0KDUpLmFsbCgpCmZvciB1IGluIHVzZXJzOgogICAgcHJpbnQoZicgIGlkPXt1LmlkfSBlbWFpbD17dS5lbWFpbH0gcGhvbmU9e2dldGF0dHIodSxcInBob25lX251bWJlclwiLE5vbmUpfSBhY3RpdmU9e2dldGF0dHIodSxcImlzX2FjdGl2ZVwiLE5vbmUpfScpCmRiLmNsb3NlKCkKXCIiIDI+JjEgfCB0YWlsIC0xNQplY2hvICIiCmVjaG8gIj09PSAzLiBBUEkgd29ya2VyIGxvZ3M6IGFueSBsaXBzeW5jIHRhc2sgZXhlY3V0aW9uIGhpc3Rvcnk/ID09PSIKZG9ja2VyIGxvZ3MgeGlhb3hpYS13b3JrZXItc3RhZ2luZyAyPiYxIHwgZ3JlcCAtaUUgImxpcHN5bmNfdHRzfHN5bnRoZXNpemVfYW5kX3N1Ym1pdHxBc3luY0Rpc3BhdGNoRmFpbGVkfHR0c19wcm9jZXNzaW5nfHR0c19wcm94eSIgfCB0YWlsIC0yMAplY2hvICIiCmVjaG8gIj09PSA0LiBQdWJsaWMgR0VUIC9oZWFsdGggd2l0aCBHRVQgbWV0aG9kID09PSIKY3VybCAtc2YgLS1tYXgtdGltZSAxMCBodHRwczovL3N0YWdpbmctYXBpLnhpYW94aWFqaWFuamkuY29tL2hlYWx0aCAyPiYxIHwgaGVhZCAtNQplY2hvICIiCmVjaG8gIj09PSA1LiBQdWJsaWMgd2ViIHJvb3QgPT09IgpjdXJsIC1zZiAtLW1heC10aW1lIDEwIGh0dHBzOi8vc3RhZ2luZy54aWFveGlhamlhbmppLmNvbS8gMj4mMSB8IGhlYWQgLTUK' | base64 -d > /tmp/e2e.sh
|
||||
chmod +x /tmp/e2e.sh
|
||||
ssh -p 22222 -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no root@47.98.113.167 'bash -s' < /tmp/e2e.sh
|
||||
@@ -34,8 +34,6 @@ celery_app.conf.imports = (
|
||||
"worker_app.tasks.tts_synthesis",
|
||||
"worker_app.tasks.batch_download",
|
||||
"worker_app.tasks.duplication_check",
|
||||
# #1798 AI 数字人渲染:必须在 Worker 实例上注册同名任务,否则消息无人消费(渲染卡 0%)
|
||||
"worker_app.tasks.ai_avatar_render",
|
||||
"worker_app.tasks._startup",
|
||||
"apps.worker.video_processing.dedup",
|
||||
"worker_app.tasks.cleanup",
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
"""AI 数字人渲染任务 — Worker 侧 Celery 任务注册.
|
||||
|
||||
#1798 渲染进度卡在 0% 的根因:渲染任务定义在 API 侧(`app.tasks.ai_avatar_render`),
|
||||
装饰在 API 自己的 Celery 实例(`xiaoxia-saas-api`)上;而 Worker 用的是
|
||||
`worker_app.celery_app` 实例,`conf.imports` 从未导入该任务,Worker 的任务
|
||||
注册表里没有 `ai_avatar_render.execute`,消息被路由到默认 `celery` 队列后
|
||||
无人消费,任务永远停在 0%。
|
||||
|
||||
修复:在 Worker 侧用 `worker_app.celery_app` 注册同名任务,直接调用与 API
|
||||
服务一致的 `AiAvatarRenderService.execute_render` 核心管线(业务逻辑在
|
||||
`apps.api.app.services`,worker 镜像已复制 `apps/api/app`)。任务名保持
|
||||
`ai_avatar_render.execute`,与 API 生产端 `.delay()` 的消息路由一致;未在
|
||||
task_routes 显式配置,走默认 `celery` 队列,由 transcode worker 消费。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from worker_app.celery_app import celery_app
|
||||
from worker_app.db import SessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="ai_avatar_render.execute", max_retries=2)
|
||||
def execute_ai_avatar_render(self, job_id: str) -> dict:
|
||||
"""执行 AI 数字人渲染管线(Worker 侧入口).
|
||||
|
||||
进度由 service 直接写入 DB(AiAvatarRenderJob.progress:
|
||||
0→5→20→40→80→90→95→100),API 通过轮询 progress 字段展示。
|
||||
"""
|
||||
logger.info("开始执行渲染任务: %s", job_id)
|
||||
self.update_state(state="PROCESSING", meta={"progress": 0, "job_id": job_id})
|
||||
|
||||
session = SessionLocal()
|
||||
try:
|
||||
from app.services.ai_avatar_render_service import AiAvatarRenderService
|
||||
|
||||
service = AiAvatarRenderService(session)
|
||||
service.execute_render(job_id)
|
||||
return {"status": "completed", "job_id": job_id}
|
||||
except Exception as exc:
|
||||
logger.exception("渲染任务执行异常 [%s]: %s", job_id, exc)
|
||||
self.update_state(state="FAILED", meta={"progress": 0, "error": str(exc)})
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
@@ -1,49 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Select nginx config based on APP_ENV (staging/production).
|
||||
#
|
||||
# 两种运行模式:
|
||||
# 1. CI/CD 部署(staging/production):部署脚本通过 `-v 宿主机文件:/etc/nginx/conf.d/default.conf:ro`
|
||||
# 把宿主机生成的带 resolver/docker upstream 的配置 bind mount 进来,entrypoint 不应改动。
|
||||
# bind mount 的文件是 readonly 的,rm 会报 EBUSY ("Resource busy"),直接 exec nginx 即可。
|
||||
# 2. 本地 docker-compose / 直接 `docker run`(无外部挂载):镜像烤入了 nginx-staging.conf 与
|
||||
# nginx-production.conf 到 /etc/nginx/,entrypoint 根据 APP_ENV 把 default.conf 换成正确的 symlink。
|
||||
#
|
||||
# 策略:
|
||||
# - 如果 /etc/nginx/conf.d/default.conf 已经是指向目标 conf 的 symlink,什么都不做;
|
||||
# - 否则尝试 rm -f 再 ln -s;rm 失败说明是外部 bind mount(已有正确配置),不阻塞启动;
|
||||
# - 兜底:只要 conf.d 目录里有 .conf 文件(含 bind mount 来的),就直接启动 nginx。
|
||||
# Both configs are baked into the image at well-known paths.
|
||||
# nginx reads config only at startup, so symlink before exec.
|
||||
# 注意:基础镜像 /etc/nginx/conf.d/default.conf 是普通文件(非 symlink/目录),
|
||||
# alpine busybox ln -sf 在 target 已存在且为普通文件时行为不稳定(会尝试在
|
||||
# target 目录下建子链接),必须先 rm 再 ln 才能正确替换。
|
||||
set -e
|
||||
|
||||
NGINX_CONF_DIR="/etc/nginx/conf.d"
|
||||
TARGET_CONF=""
|
||||
|
||||
case "${APP_ENV:-production}" in
|
||||
staging)
|
||||
TARGET_CONF="/etc/nginx/nginx-staging.conf"
|
||||
rm -f "$NGINX_CONF_DIR/default.conf"
|
||||
ln -s /etc/nginx/nginx-staging.conf "$NGINX_CONF_DIR/default.conf"
|
||||
;;
|
||||
*)
|
||||
TARGET_CONF="/etc/nginx/nginx-production.conf"
|
||||
rm -f "$NGINX_CONF_DIR/default.conf"
|
||||
ln -s /etc/nginx/nginx-production.conf "$NGINX_CONF_DIR/default.conf"
|
||||
;;
|
||||
esac
|
||||
|
||||
DEFAULT_CONF="$NGINX_CONF_DIR/default.conf"
|
||||
|
||||
# 1. 已经是正确的 symlink:直接启动
|
||||
if [ -L "$DEFAULT_CONF" ] && [ "$(readlink "$DEFAULT_CONF" 2>/dev/null)" = "$TARGET_CONF" ]; then
|
||||
exec nginx -g "daemon off;"
|
||||
fi
|
||||
|
||||
# 2. 尝试替换为目标 symlink(无 bind mount 的场景)
|
||||
# 若 rm 失败(bind mount readonly,EBUSY/EPERM),则认为外部已注入配置,不阻塞。
|
||||
rm -f "$DEFAULT_CONF" 2>/dev/null || true
|
||||
if [ -f "$TARGET_CONF" ] && [ ! -e "$DEFAULT_CONF" ]; then
|
||||
ln -s "$TARGET_CONF" "$DEFAULT_CONF" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# 3. 兜底:至少要有一个 .conf 文件,否则 nginx 起不来
|
||||
if ! ls "$NGINX_CONF_DIR"/*.conf >/dev/null 2>&1; then
|
||||
echo "ERROR: no nginx config found in $NGINX_CONF_DIR (tried $TARGET_CONF and external bind mount)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec nginx -g "daemon off;"
|
||||
|
||||
Reference in New Issue
Block a user