xiaoxia-saas/scripts/ci/run_unit_tests.sh

#!/bin/bash
# CI Unit Tests Job 主脚本
# 包含：依赖缓存、增量测试选择、覆盖率测试、diff覆盖率门禁
set -eu

JOB_NAME="${1:-Unit Tests}"

echo "=== CI Unit Tests 开始 ==="

# --- Python 版本选择（必须 >= 3.11，代码使用 datetime.UTC）---
if ! python3 -c "import sys; assert sys.version_info >= (3, 11)" 2>/dev/null; then
  for cand in python3.12 python3.11 /opt/python3.12/bin/python3; do
    if command -v "$cand" >/dev/null 2>&1 && "$cand" -c "import sys; assert sys.version_info >= (3, 11)" 2>/dev/null; then
      PY3_DIR="$(dirname "$(command -v "$cand")")"
      export PATH="$PY3_DIR:$PATH"
      hash -r
      echo "✅ python3 版本过低，改用 $cand ($("$cand" -c "import sys; print(sys.version.split()[0])"))"
      break
    fi
  done
  if ! python3 -c "import sys; assert sys.version_info >= (3, 11)" 2>/dev/null; then
    echo "❌ 未找到 Python >= 3.11，本项目要求 Python 3.11+（使用 datetime.UTC）" >&2
    exit 1
  fi
fi

PYVER=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
echo "使用 Python 版本: $(python3 --version)"

# --- 依赖缓存检查（按 Python 版本区分缓存文件，避免跨版本复用）---
# 如果 requirements 文件未变化且依赖已安装，跳过 pip install（持久 runner 优化）
REQ_HASH_FILE="/tmp/.ci_unit_tests_req_hash_py${PYVER}"
CURRENT_REQ_HASH=""
if [ -f requirements-base.txt ] && [ -f requirements.txt ] && [ -f requirements-dev.txt ]; then
  CURRENT_REQ_HASH=$(cat requirements-base.txt requirements.txt requirements-dev.txt | md5sum | cut -d' ' -f1)
fi

SKIP_PIP_INSTALL=false
if [ -n "$CURRENT_REQ_HASH" ] && [ -f "$REQ_HASH_FILE" ]; then
  CACHED_HASH=$(cat "$REQ_HASH_FILE")
  if [ "$CACHED_HASH" = "$CURRENT_REQ_HASH" ]; then
    # 验证关键包是否还在
    if python3 -c "import pytest; import celery" 2>/dev/null; then
      echo "✅ 依赖无变化 (hash=$CURRENT_REQ_HASH)，跳过 pip install"
      SKIP_PIP_INSTALL=true
    else
      echo "⚠️ 依赖 hash 匹配但关键包缺失，重新安装"
    fi
  fi
fi

# --- 安装依赖 ---
if [ "$SKIP_PIP_INSTALL" = "false" ]; then
  echo ""
  echo "=== 安装 Python 依赖 ==="
  # pip install 带重试（网络不稳定时自动重试），合并为一次调用减少开销
  for i in 1 2 3; do
    python3 -m pip install -q -r requirements-base.txt -r requirements.txt -r requirements-dev.txt && break
    echo "pip install 失败，重试 $i/3..."
    [ $i -eq 3 ] && exit 1
    sleep 5
  done
  # 保存 hash 标记
  if [ -n "$CURRENT_REQ_HASH" ]; then
    echo "$CURRENT_REQ_HASH" > "$REQ_HASH_FILE"
  fi
fi
pytest --version

# 双保险：确保numpy已安装
echo "=== 验证 numpy 安装 ==="
SKIP_NUMPY_TESTS=0
if python3 -c "import numpy; assert numpy.__version__ == '1.26.4'" 2>/dev/null; then
  echo "✅ numpy 1.26.4 已就绪（缓存命中）"
else
  echo "需要安装 numpy 1.26.4..."
  python3 -m pip install numpy==1.26.4 || {
    echo "❌ numpy 首次安装失败，尝试不使用缓存重新安装..."
    python3 -m pip install --no-cache-dir numpy==1.26.4 || {
      echo "⚠️ numpy 安装失败，跳过需要 numpy 的测试"
      SKIP_NUMPY_TESTS=1
    }
  }
fi
if [ "$SKIP_NUMPY_TESTS" = "0" ]; then
  python3 -c "import numpy; print(f'✅ numpy {numpy.__version__} 就绪')" || {
    echo "⚠️ numpy 导入失败，跳过需要 numpy 的测试"
    SKIP_NUMPY_TESTS=1
  }
fi

# --- 增量测试选择（PR + push 均支持） ---
UNIT_TEST_MODE="full"
SELECTED_TEST_FILES="tests/unit"

IS_PULL_REQUEST=false
IS_PUSH=false
[ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && IS_PULL_REQUEST=true
[ "${GITHUB_EVENT_NAME:-}" = "push" ] && IS_PUSH=true

if ($IS_PULL_REQUEST || $IS_PUSH) && [ -n "${GITHUB_TOKEN:-}" ]; then
  echo ""
  echo "=== 增量测试选择 ==="

  CHANGED_FILES=""

  if $IS_PULL_REQUEST; then
    PR_NUMBER=$(echo "$GITHUB_REF" | sed 's|refs/pull/||; s|/.*||')
    API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
    CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" \
      | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin) if f['status'] != 'removed']")
  elif $IS_PUSH && [ -n "${GITHUB_SHA:-}" ]; then
    # Push 事件：通过 GitHub API 获取本次 push 改动的文件
    API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}"
    RESPONSE=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
      -H "Accept: application/vnd.github.v3.diff" "$API_URL" 2>/dev/null || echo "")

    if [ -n "$RESPONSE" ]; then
      CHANGED_FILES=$(echo "$RESPONSE" | grep '^diff --git' | sed 's|diff --git a/\(.*\) b/.*|\1|' || echo "")
    fi

    # 备用方案：获取 previous commit SHA 再查 API
    if [ -z "$CHANGED_FILES" ]; then
      PREV_SHA=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
        "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/commits?sha=${GITHUB_SHA}&per_page=2" \
        | python3 -c "import sys,json; commits=json.load(sys.stdin); print(commits[1]['sha'] if len(commits)>1 else '')" 2>/dev/null || echo "")
      if [ -n "$PREV_SHA" ]; then
        COMPARE_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/compare/${PREV_SHA}...${GITHUB_SHA}"
        CHANGED_FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$COMPARE_URL" \
          | python3 -c "import sys,json; data=json.load(sys.stdin); [print(f['filename']) for f in data.get('files',[]) if f['status'] != 'removed']" 2>/dev/null || echo "")
      fi
    fi
  fi

  echo "改动文件数: $(echo "$CHANGED_FILES" | grep -c . || echo 0)"

  if [ -n "$CHANGED_FILES" ]; then
    set +e
    CHANGED_FILES="$CHANGED_FILES" \
    SELECTED_TESTS_OUTPUT=/tmp/selected_tests.txt \
    python3 scripts/ci/select_unit_tests.py
    SELECT_EXIT=$?
    set -e
    if [ $SELECT_EXIT -eq 0 ]; then
      UNIT_TEST_MODE="incremental"
      TEST_FILES=$(cat /tmp/selected_tests.txt | tr '\n' ' ')
      SELECTED_TEST_FILES="$TEST_FILES"
      echo "增量模式: $(cat /tmp/selected_tests.txt | wc -l) 个测试文件"
    else
      echo "全量模式（增量选择失败）"
    fi
  else
    echo "无法获取改动文件列表，使用全量模式"
  fi
fi

# --- 运行单元测试 + 覆盖率 ---
echo ""
echo "=== 运行单元测试 (模式: $UNIT_TEST_MODE) ==="

# 如果 numpy 不可用，跳过依赖 numpy 的测试文件
NUMPY_IGNORE=""
if [ "${SKIP_NUMPY_TESTS:-0}" = "1" ]; then
  echo "⚠️ SKIP_NUMPY_TESTS=1，将跳过依赖 numpy 的测试"
  NUMPY_IGNORE="--ignore=tests/unit/test_dedup_engine.py"
fi

if [ "$UNIT_TEST_MODE" = "incremental" ]; then
  echo "=== 增量测试模式 ==="
  PYTHONPATH="$PWD/apps/api:$PWD/apps/worker:$PWD/packages:$PWD" python3 -m coverage run \
    --source=apps/api/app,packages \
    --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
    --branch \
    -m pytest $SELECTED_TEST_FILES $NUMPY_IGNORE -q
  python3 -m coverage report --show-missing
  python3 -m coverage xml -o coverage.xml
  python3 -m coverage report --fail-under=10 > /dev/null || true
else
  PYTHONPATH="$PWD/apps/api:$PWD/apps/worker:$PWD/packages:$PWD" python3 -m coverage run \
    --source=apps/api/app,packages \
    --omit="*/migrations/*,*/tests/*,*/test_*.py,*/site-packages/*" \
    --branch \
    -m pytest tests/unit $NUMPY_IGNORE -q
  python3 -m coverage report --show-missing
  python3 -m coverage xml -o coverage.xml
  python3 -m coverage report --fail-under=65 > /dev/null
fi

# --- Diff 覆盖率检查（仅PR） ---
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [ -n "${GITHUB_TOKEN:-}" ]; then
  echo ""
  echo "=== Diff 覆盖率检查 ==="
  BASE_BRANCH="${GITHUB_BASE_REF:-develop}"
  echo "Base branch: $BASE_BRANCH"

  PR_CODE_DIR="/tmp/pr-code-$$"
  mkdir -p "$PR_CODE_DIR"
  # 备份PR代码（含coverage.xml，diff-cover需要用到）
  find . -maxdepth 1 -mindepth 1 ! -name 'diff_coverage.html' -exec cp -r {} "$PR_CODE_DIR/" \;
  rm -rf .git
  git init > /dev/null 2>&1
  git remote add origin https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas.git > /dev/null 2>&1
  git config user.email "ci@local"
  git config user.name "CI"
  git fetch origin "$BASE_BRANCH" --depth=200
  # 先清理工作目录，避免未跟踪文件导致checkout失败
  find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} +
  git checkout -b ci-pr-branch "origin/$BASE_BRANCH" > /dev/null 2>&1
  # 清除base分支源码，用PR代码覆盖
  find . -mindepth 1 -maxdepth 1 ! -name '.git' -exec rm -rf {} +
  cp -r "$PR_CODE_DIR"/. .
  rm -rf "$PR_CODE_DIR"
  git add -A > /dev/null 2>&1
  git commit -m "ci-tmp" > /dev/null 2>&1

  if [ "$UNIT_TEST_MODE" = "incremental" ]; then
    THRESHOLD=40
    echo "增量测试模式，增量覆盖率门槛: ${THRESHOLD}%"
  else
    THRESHOLD=60
    echo "全量测试模式，增量覆盖率门槛: ${THRESHOLD}%"
  fi

  set +e
  python3 -m diff_cover.diff_cover_tool coverage.xml \
    --compare-branch="origin/$BASE_BRANCH" \
    --fail-under=$THRESHOLD \
    --html-report diff_coverage.html \
    2>&1
  DIFF_EXIT=$?
  set -e
  if [ $DIFF_EXIT -ne 0 ]; then
    echo ""
    echo "❌ 增量覆盖率未达到门槛 (${THRESHOLD}%)"
    echo "   请为改动的代码添加单元测试后再提交"
    echo ""
    echo "=== 覆盖率报告 ==="
    python3 -m diff_cover.diff_cover_tool coverage.xml \
      --compare-branch="origin/$BASE_BRANCH" 2>&1 | tail -30
    exit 1
  fi
  echo "✅ 增量覆盖率达标"
fi

echo ""
echo "=== CI Unit Tests 全部通过 ✅ ==="