Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b4890da58c | |||
| a141cd56ff | |||
| 90f3c0694c | |||
| 10cb082560 | |||
| cd66aace5b | |||
| e9ad12e072 | |||
| e557da4d98 | |||
| cef402f88d | |||
| c29297a032 |
+210
-452
@@ -84,13 +84,104 @@ jobs:
|
||||
python3 -m pip --version
|
||||
echo "CI environment is ready"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-${{ hashFiles('requirements-base.txt', 'requirements.txt', 'requirements-dev.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-
|
||||
|
||||
- name: Secret detection (gitleaks)
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
echo "=== Installing gitleaks ==="
|
||||
GITLEAKS_VERSION="v8.18.4"
|
||||
GITLEAKS_ARCH="linux_x64"
|
||||
GITLEAKS_FILE="gitleaks_${GITLEAKS_VERSION#v}_${GITLEAKS_ARCH}.tar.gz"
|
||||
GITHUB_BASE="https://github.com/gitleaks/gitleaks/releases/download/${GITLEAKS_VERSION}/${GITLEAKS_FILE}"
|
||||
|
||||
# 国内镜像源(按大致稳定性排序)
|
||||
MIRRORS="
|
||||
https://gh-proxy.com/${GITHUB_BASE}
|
||||
https://ghproxy.net/${GITHUB_BASE}
|
||||
https://hub.gitmirror.com/${GITHUB_BASE}
|
||||
https://ghps.cc/${GITHUB_BASE}
|
||||
https://mirror.ghproxy.com/${GITHUB_BASE}
|
||||
${GITHUB_BASE}
|
||||
"
|
||||
|
||||
INSTALLED=false
|
||||
for url in $MIRRORS; do
|
||||
echo "Trying: $url"
|
||||
if curl -fsSL --connect-timeout 8 --max-time 90 --retry 2 --retry-delay 3 \
|
||||
-o /tmp/gitleaks.tar.gz "$url" 2>/dev/null; then
|
||||
echo "Download successful from: $url"
|
||||
if tar -xzf /tmp/gitleaks.tar.gz -C /tmp gitleaks 2>/dev/null; then
|
||||
chmod +x /tmp/gitleaks
|
||||
/tmp/gitleaks version
|
||||
INSTALLED=true
|
||||
break
|
||||
else
|
||||
echo "Download OK but tar extraction failed, trying next..."
|
||||
fi
|
||||
else
|
||||
echo "Download failed from: $url, trying next..."
|
||||
fi
|
||||
done
|
||||
|
||||
# Fallback: 尝试 go install 从源码编译
|
||||
if [ "$INSTALLED" = "false" ] && command -v go >/dev/null 2>&1; then
|
||||
echo "All binary mirrors failed, trying go install..."
|
||||
if go install github.com/gitleaks/gitleaks/v8@${GITLEAKS_VERSION} 2>/dev/null; then
|
||||
GOPATH_BIN="$(go env GOPATH)/bin"
|
||||
if [ -x "$GOPATH_BIN/gitleaks" ]; then
|
||||
cp "$GOPATH_BIN/gitleaks" /tmp/gitleaks
|
||||
chmod +x /tmp/gitleaks
|
||||
/tmp/gitleaks version
|
||||
INSTALLED=true
|
||||
echo "Installed via go install"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$INSTALLED" = "false" ]; then
|
||||
echo "WARNING: Failed to install gitleaks from all sources"
|
||||
echo "gitleaks 安装失败,密钥检测跳过(告警模式,不阻断流水线)"
|
||||
echo "请检查Runner网络或手动安装gitleaks到Runner"
|
||||
exit 0
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Running gitleaks scan ==="
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
echo "PR mode: scanning changed files (origin/${{ github.base_ref }}..HEAD)"
|
||||
set +e
|
||||
/tmp/gitleaks detect \
|
||||
--source . \
|
||||
--config .gitleaks.toml \
|
||||
--verbose \
|
||||
--exit-code 1 \
|
||||
--log-opts="origin/${{ github.base_ref }}..HEAD"
|
||||
GITLEAKS_EXIT=$?
|
||||
set -e
|
||||
else
|
||||
echo "Push mode: full repository scan"
|
||||
set +e
|
||||
/tmp/gitleaks detect \
|
||||
--source . \
|
||||
--config .gitleaks.toml \
|
||||
--verbose \
|
||||
--exit-code 1
|
||||
GITLEAKS_EXIT=$?
|
||||
set -e
|
||||
fi
|
||||
if [ "$GITLEAKS_EXIT" = "1" ]; then
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "ERROR: Secrets detected!"
|
||||
echo "=========================================="
|
||||
echo "If these are false positives, add them to .gitleaks.toml allowlist."
|
||||
exit 1
|
||||
elif [ "$GITLEAKS_EXIT" != "0" ]; then
|
||||
echo "WARNING: gitleaks exited with code $GITLEAKS_EXIT (non-zero but not detection failure)"
|
||||
echo "This may indicate a configuration issue. Continuing for now..."
|
||||
else
|
||||
echo "gitleaks scan completed - no secrets detected"
|
||||
fi
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -104,6 +195,38 @@ jobs:
|
||||
bandit --version
|
||||
pytest --version
|
||||
|
||||
|
||||
- name: Python dependency vulnerability scan (pip-audit)
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
echo "=== Installing pip-audit ==="
|
||||
python3 -m pip install -q pip-audit
|
||||
pip-audit --version
|
||||
echo ""
|
||||
echo "=== Scanning Python dependencies ==="
|
||||
set +e
|
||||
HAS_VULN=0
|
||||
for req_file in requirements.txt requirements-base.txt requirements-dev.txt requirements-worker.txt; do
|
||||
if [ -f "$req_file" ]; then
|
||||
echo "--- Scanning $req_file ---"
|
||||
pip-audit -r "$req_file" --desc on 2>&1 | head -30
|
||||
EXIT_CODE=${PIPESTATUS[0]:-0}
|
||||
if [ "$EXIT_CODE" = "1" ]; then
|
||||
HAS_VULN=1
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
echo "=== Summary ==="
|
||||
if [ "$HAS_VULN" = "1" ]; then
|
||||
echo "WARNING: Vulnerabilities found in dependencies."
|
||||
else
|
||||
echo "No known vulnerabilities found in scanned requirements."
|
||||
fi
|
||||
echo "Mode: advisory only (not blocking CI)"
|
||||
exit 0
|
||||
- name: Run code quality checks
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -220,13 +343,6 @@ jobs:
|
||||
tar.extract(member, '.')
|
||||
PY
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-${{ hashFiles('requirements-base.txt', 'requirements.txt', 'requirements-dev.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -326,13 +442,6 @@ jobs:
|
||||
python3 -m pip --version
|
||||
echo "CI environment is ready"
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-${{ hashFiles('requirements-base.txt', 'requirements.txt', 'requirements-dev.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-
|
||||
- name: Install dependencies
|
||||
shell: sh
|
||||
run: |
|
||||
@@ -608,252 +717,70 @@ jobs:
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Frontend Lint" python3 scripts/ci_notify_failure.py
|
||||
|
||||
build-staging-api:
|
||||
name: Build Staging API Image
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: api
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
|
||||
- name: Build and push api image
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST: "true"
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST=true \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_SHA}" staging
|
||||
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Staging API Image" python3 scripts/ci_notify_failure.py
|
||||
|
||||
build-staging-worker:
|
||||
name: Build Staging Worker Image
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: worker
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
|
||||
- name: Build and push worker image
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST: "true"
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST=true \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_SHA}" staging
|
||||
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Staging Worker Image" python3 scripts/ci_notify_failure.py
|
||||
|
||||
build-staging-web:
|
||||
name: Build Staging Web Image
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: web
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
|
||||
- name: Build and push web image
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST: "true"
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST=true \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_SHA}" staging
|
||||
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Staging Web Image" python3 scripts/ci_notify_failure.py
|
||||
|
||||
deploy-staging:
|
||||
name: Deploy Staging (Watchtower auto-deploy)
|
||||
name: Build & Push Staging (Watchtower auto-deploy)
|
||||
runs-on: saas
|
||||
timeout-minutes: 20
|
||||
needs: [build-staging-api, build-staging-worker, build-staging-web]
|
||||
timeout-minutes: 30
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'develop')
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
- name: Build and push all images to Gitea Registry
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
ALLOW_SHARED_PRODUCTION_BUILD_HOST=true REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_SHA}" staging
|
||||
|
||||
- name: Tag and push :staging images (Watchtower auto-update)
|
||||
shell: sh
|
||||
env:
|
||||
@@ -865,7 +792,6 @@ jobs:
|
||||
printf '%s' "${REGISTRY_TOKEN}" | docker login git.xiaoxiajianji.com -u xiaoxia --password-stdin 2>/dev/null
|
||||
fi
|
||||
for svc in api worker web; do
|
||||
docker pull "${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}"
|
||||
docker tag "${REGISTRY}/xiaoxia-saas-${svc}:${GITHUB_SHA}" "${REGISTRY}/xiaoxia-saas-${svc}:staging"
|
||||
docker push "${REGISTRY}/xiaoxia-saas-${svc}:staging"
|
||||
done
|
||||
@@ -936,7 +862,9 @@ jobs:
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Deploy Staging" python3 scripts/ci_notify_failure.py
|
||||
FAILED_JOB="Build & Push Staging (Watchtower auto-deploy)" python3 scripts/ci_notify_failure.py
|
||||
|
||||
|
||||
|
||||
staging-e2e:
|
||||
name: Staging E2E Tests
|
||||
@@ -1089,17 +1017,14 @@ jobs:
|
||||
|
||||
|
||||
|
||||
build-prod-api:
|
||||
name: Build Production API Image
|
||||
build-production-runtime-images:
|
||||
name: Build Production Runtime Images
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 30
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: api
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
@@ -1107,7 +1032,7 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
python3 - <<'PY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
@@ -1132,92 +1057,7 @@ jobs:
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
|
||||
- name: Build and push production api image
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_REF_NAME}"
|
||||
|
||||
|
||||
- name: Cleanup old Docker images
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
docker image prune -f 2>/dev/null || true
|
||||
echo "Disk usage after cleanup:"
|
||||
df -h / | tail -1
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Production API Image" python3 scripts/ci_notify_failure.py
|
||||
|
||||
build-prod-worker:
|
||||
name: Build Production Worker Image
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: worker
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
@@ -1230,129 +1070,47 @@ jobs:
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
PY
|
||||
|
||||
|
||||
- name: Build and push production worker image
|
||||
- name: Build and push all images (api + worker + web, with buildx cache)
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_REF_NAME}"
|
||||
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" scripts/build_release_images.sh "${GITHUB_REF_NAME}"
|
||||
|
||||
- name: Cleanup old Docker images
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
docker image prune -f 2>/dev/null || true
|
||||
set -eu
|
||||
if [ -f scripts/cleanup_old_images.sh ]; then
|
||||
chmod +x scripts/cleanup_old_images.sh
|
||||
scripts/cleanup_old_images.sh
|
||||
else
|
||||
echo "Cleanup script not found, doing basic prune..."
|
||||
docker image prune -f 2>/dev/null || true
|
||||
fi
|
||||
echo "Disk usage after cleanup:"
|
||||
df -h / | tail -1
|
||||
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Production Worker Image" python3 scripts/ci_notify_failure.py
|
||||
FAILED_JOB="Build Production Runtime Images" python3 scripts/ci_notify_failure.py
|
||||
|
||||
build-prod-web:
|
||||
name: Build Production Web Image
|
||||
runs-on: saas
|
||||
timeout-minutes: 25
|
||||
needs: [validate, frontend-lint]
|
||||
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
|
||||
env:
|
||||
BUILD_ONLY: web
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: sh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -eu
|
||||
python3 - <<'INNERPY'
|
||||
import io, os, tarfile, time, urllib.request, urllib.error
|
||||
url = f"{os.environ['GITHUB_API_URL']}/repos/{os.environ['GITHUB_REPOSITORY']}/archive/{os.environ['GITHUB_SHA']}.tar.gz"
|
||||
request = urllib.request.Request(url, headers={"Authorization": f"token {os.environ['GITHUB_TOKEN']}"})
|
||||
last_err = None
|
||||
for attempt in range(5):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=120) as response:
|
||||
archive = response.read()
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
if e.code >= 500 and attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout HTTP {e.code}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
if attempt < 4:
|
||||
wait = 2 ** attempt
|
||||
print(f"Checkout error: {e}, retrying in {wait}s (attempt {attempt+1}/5)...")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
raise last_err
|
||||
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
||||
root_prefix = tar.getmembers()[0].name.split('/', 1)[0] + '/'
|
||||
for member in tar.getmembers():
|
||||
name = member.name
|
||||
if name == root_prefix[:-1]:
|
||||
continue
|
||||
if name.startswith(root_prefix):
|
||||
member.name = name[len(root_prefix):]
|
||||
if member.name:
|
||||
tar.extract(member, '.')
|
||||
INNERPY
|
||||
|
||||
|
||||
- name: Build and push production web image
|
||||
shell: sh
|
||||
env:
|
||||
REGISTRY_TOKEN: ${ secrets.REGISTRY_TOKEN }
|
||||
run: |
|
||||
set -eu
|
||||
chmod +x scripts/build_release_images.sh
|
||||
BUILD_ONLY="${BUILD_ONLY}" \
|
||||
REGISTRY_TOKEN="${REGISTRY_TOKEN}" \
|
||||
scripts/build_release_images.sh "${GITHUB_REF_NAME}"
|
||||
|
||||
|
||||
- name: Cleanup old Docker images
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
docker image prune -f 2>/dev/null || true
|
||||
echo "Disk usage after cleanup:"
|
||||
df -h / | tail -1
|
||||
- name: Notify CI failure
|
||||
if: failure()
|
||||
shell: sh
|
||||
run: |
|
||||
set +e
|
||||
echo "=== CI 失败通知 ==="
|
||||
FAILED_JOB="Build Production Web Image" python3 scripts/ci_notify_failure.py
|
||||
|
||||
deploy-production:
|
||||
name: Deploy Production
|
||||
runs-on: saas
|
||||
timeout-minutes: 20
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
needs: [build-prod-api, build-prod-worker, build-prod-web]
|
||||
needs: build-production-runtime-images
|
||||
|
||||
steps:
|
||||
- name: Install SSH client
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# .gitleaks.toml - gitleaks 白名单配置
|
||||
# 仓库: xiaoxia/xiaoxia-saas
|
||||
# 用途: 排除已知的测试密钥、示例配置等误报
|
||||
# 注意: paths 使用正则表达式(Regex),不是 glob 语法
|
||||
|
||||
# 允许路径/文件排除
|
||||
[allowlist]
|
||||
description = "全局白名单 - 排除示例配置和测试文件"
|
||||
paths = [
|
||||
# 环境配置示例(无真实密钥)
|
||||
'.env.example',
|
||||
'.env.sample',
|
||||
'\.env\.example$',
|
||||
'\.env\.sample$',
|
||||
# 测试文件
|
||||
'tests/',
|
||||
'test/',
|
||||
# 文档
|
||||
'docs/',
|
||||
'\.md$',
|
||||
'\.rst$',
|
||||
# 前端依赖
|
||||
'node_modules/',
|
||||
# Python包
|
||||
'site-packages/',
|
||||
# 锁定文件(自动生成)
|
||||
'poetry.lock',
|
||||
'Pipfile.lock',
|
||||
'requirements.*\.txt\.lock',
|
||||
# CI配置本身
|
||||
'.gitea/',
|
||||
# Docker相关
|
||||
'docker-compose.*\.yml',
|
||||
# gitleaks配置自身
|
||||
'.gitleaks.toml',
|
||||
# vulture配置
|
||||
'vulture.conf',
|
||||
'vulture_whitelist.py',
|
||||
# 前端构建产物
|
||||
'dist/',
|
||||
'build/',
|
||||
# 图片/二进制文件
|
||||
'\.png$',
|
||||
'\.jpg$',
|
||||
'\.jpeg$',
|
||||
'\.gif$',
|
||||
'\.ico$',
|
||||
'\.svg$',
|
||||
'\.woff$',
|
||||
'\.woff2$',
|
||||
'\.ttf$',
|
||||
# 项目文档
|
||||
'README',
|
||||
'CHANGELOG',
|
||||
'LICENSE',
|
||||
]
|
||||
|
||||
# 允许的密钥值/占位符正则
|
||||
# 使用三引号字符串避免引号转义问题
|
||||
regexes = [
|
||||
'''(?i)(your[_-]?password|your[_-]?secret|your[_-]?key|your[_-]?token|changeme|change[_-]?me|placeholder|example[_-]?key|test[_-]?key|dummy|fake|mock|xxx|none|not[_-]?set|TODO|FIXME)''',
|
||||
'''postgresql://[^:]+:changeme@''',
|
||||
'''postgresql://[^:]+:your-password@''',
|
||||
'''postgresql://[^:]+:password@localhost''',
|
||||
'''redis://:changeme@''',
|
||||
'''redis://:your-redis-password@''',
|
||||
'''(?i)jwt[_-]?secret\s*[:=]\s*["']?(your[_-]?jwt|change|placeholder|secret|example)''',
|
||||
'''postgresql://postgres:postgres@''',
|
||||
'''redis://localhost''',
|
||||
'''mysql://root:root@''',
|
||||
]
|
||||
@@ -0,0 +1,137 @@
|
||||
# 首次安全扫描简报
|
||||
|
||||
> 仓库: xiaoxia/xiaoxia-saas
|
||||
> 扫描时间: 2026-07-13
|
||||
> 负责人: 代码审计 Agent
|
||||
> 状态: CI 排队中(Runner 环境修复中),gitleaks 下载链路二次优化已提交,待 CI 跑通后获取真实扫描数据
|
||||
|
||||
## 一、概览
|
||||
|
||||
| 工具 | 优先级 | PR | 接入状态 | CI 验证 | 扫描结果 |
|
||||
|---|---|---|---|---|---|
|
||||
| gitleaks(密钥检测) | P0 | [#256](https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/pulls/256) | ✅ 二次优化提交 | 🕐 CI 排队中 | 待验证 |
|
||||
| pip-audit(Python 依赖漏洞) | P1 | [#256](https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/pulls/256) | ✅ 代码已提交 | 🕐 CI 排队中 | 待验证 |
|
||||
| vulture(死代码检测) | P2 | [#259](https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/pulls/259) | ✅ 代码已提交 | 🕐 等待 black 修复 + CI 排队 | 待验证 |
|
||||
|
||||
## 二、各工具详情
|
||||
|
||||
### 1. gitleaks 密钥检测(P0)
|
||||
|
||||
**接入配置**
|
||||
- 位置:validate Job 第 3 步(Verify CI environment 之后)
|
||||
- PR 模式:增量扫描(`--log-opts="origin/base..HEAD"`),只扫描改动文件
|
||||
- Push 模式:全量扫描
|
||||
- 阻断策略:发现密钥直接阻断合并(exit code 1)
|
||||
- 白名单:`.gitleaks.toml`,排除以下路径/内容:
|
||||
- `.env.example`、示例配置文件
|
||||
- `tests/`、`docs/`、`node_modules/`、`site-packages/`
|
||||
- 锁定文件(poetry.lock 等)
|
||||
- 占位符字符串(`your-password`、`changeme`、`placeholder` 等)
|
||||
|
||||
**下载问题(第二轮修复已提交)**
|
||||
- 第一轮修复(ghproxy + 99988866 + GitHub 直连):全部失败
|
||||
- `mirror.ghproxy.com`: 连接超时(7.7s)
|
||||
- `gh.api.99988866.xyz`: SSL 握手失败
|
||||
- GitHub 直连: 120s 超时,仅下载 1.6MB/2.9MB
|
||||
- 第二轮修复(6 个镜像 + go install 降级):
|
||||
- 新增 4 个国内镜像:`gh-proxy.com`、`ghproxy.net`、`hub.gitmirror.com`、`ghps.cc`
|
||||
- 每个镜像重试 2 次,connect-timeout 8s,max-time 90s
|
||||
- 增加 `go install` 源码编译降级方案
|
||||
- 全部失败时告警跳过(不阻断 CI),避免阻塞开发流程
|
||||
- 修复 commit:`fix(ci): add more Chinese mirrors for gitleaks download + graceful degradation`
|
||||
|
||||
**长期建议**:在 Runner 镜像中预装 gitleaks 二进制,彻底避免下载问题
|
||||
|
||||
**CI 状态**
|
||||
- 当前状态:大量 CI 任务排队中(9 个 Runner 在线但任务堆积)
|
||||
- 历史失败 Run #4071(旧 Runner):gitleaks 下载失败(见上)
|
||||
- 历史失败 Run #4075(新 Runner):Job in_progress 但步骤全 queued(新 Runner 执行环境问题)
|
||||
- 最新 Run #4096:queued 状态,等待执行
|
||||
- 构建运维 Agent 正在修复:Runner 标签匹配 + 新 Runner 执行环境 + 并发优化
|
||||
|
||||
### 2. pip-audit Python 依赖漏洞扫描(P1)
|
||||
|
||||
**接入配置**
|
||||
- 位置:validate Job 第 5 步(Install dependencies 之后)
|
||||
- 扫描范围:`requirements.txt`、`requirements-base.txt`、`requirements-dev.txt`、`requirements-worker.txt`
|
||||
- 数据源:OSV(PyPA 官方推荐)
|
||||
- 阻断策略:告警模式,不阻断 CI
|
||||
- 计划:运行 1-2 周摸清漏洞存量后,按严重等级设置阻断阈值
|
||||
|
||||
**CI 状态**
|
||||
- 同 PR #256,因 Runner 卡住暂未执行
|
||||
|
||||
### 3. vulture 死代码检测(P2)
|
||||
|
||||
**接入配置**
|
||||
- 位置:validate Job,Run security scan (bandit) 之后
|
||||
- 置信度阈值:80%
|
||||
- 扫描范围:`alembic/`、`apps/`、`packages/`、`scripts/`
|
||||
- 排除:测试文件、迁移文件、文档、node_modules、site-packages
|
||||
- 白名单:框架自动调用代码
|
||||
- FastAPI routes / dependencies / middleware
|
||||
- SQLAlchemy models / Pydantic schemas
|
||||
- Celery tasks
|
||||
- Alembic migration functions
|
||||
- CLI scripts / 工具函数
|
||||
- 阻断策略:告警模式,不阻断 CI
|
||||
|
||||
**CI 状态**
|
||||
- Workflow Run #4054
|
||||
- 失败原因:`scripts/check_migration_safety.py` 不符合 black 格式
|
||||
```
|
||||
would reformat scripts/check_migration_safety.py
|
||||
1 file would be reformatted, 376 files would be left unchanged.
|
||||
```
|
||||
- 说明:非 vulture 引入的问题(vulture 步骤还没执行到),是其他 Agent 修改了迁移安全检查脚本但没跑 black 格式化
|
||||
- 建议:后端开发 Agent 在迁移安全 PR 中同步修复 black 格式问题
|
||||
|
||||
## 三、PR #264(前端清理)安全审查
|
||||
|
||||
**审查结论:✅ 无安全风险,可合并**
|
||||
|
||||
**审查范围**:PR #264 `cleanup/phase3-frontend` → `develop`,6 个文件变更(+261/-1359 行)
|
||||
|
||||
**检查项**:
|
||||
| 检查项 | 结果 | 说明 |
|
||||
|---|---|---|
|
||||
| 危险 DOM 操作(innerHTML/eval 等) | ✅ 通过 | 未发现 dangerouslySetInnerHTML、eval、document.write 等 |
|
||||
| 硬编码密钥/Token | ✅ 通过 | 未发现 API Key、Secret、Password 等硬编码 |
|
||||
| 本地存储操作(localStorage 等) | ✅ 通过 | 未新增本地存储操作 |
|
||||
| 开放重定向漏洞 | ✅ 通过 | 未新增 window.location / redirect 操作 |
|
||||
| 新增第三方依赖 | ✅ 通过 | 仅新增 `@ant-design/icons` 的 DatabaseOutlined 图标 |
|
||||
| 删除文件安全性 | ✅ 通过 | 删除 `apps/web/src/api/accounts.ts`(Mock API 文件),无安全影响 |
|
||||
|
||||
**核心变更性质**:
|
||||
- 删除 Mock 数据(accounts.ts),替换为真实 API 调用或占位
|
||||
- 清理 Admin.css 冗余样式(-460 行)
|
||||
- Dashboard / GeneratePage / TitleLibrary 页面 Mock 替换为真实数据调用
|
||||
- 整体净删除 1098 行,代码量减少,攻击面缩小
|
||||
|
||||
## 四、发现的其他 CI 问题
|
||||
|
||||
### Runner 环境问题
|
||||
1. **新服务器 Runner 卡住**:`xiaoxia-ci-runner-new-2` 上的 Job 一直停留在 queued 状态,无法执行步骤
|
||||
2. **Unit Tests 快速失败**:Unit Tests Job 18 秒就失败了,可能是环境/依赖问题,非代码问题
|
||||
3. **Integration Tests 快速失败**:24 秒失败,同样可能是环境问题
|
||||
|
||||
### 代码质量预存问题
|
||||
1. `scripts/check_migration_safety.py` 不符合 black 格式(可能是后端开发刚改动过)
|
||||
|
||||
## 五、下一步计划
|
||||
|
||||
1. **等待 CI 环境修复**:构建运维修复 Runner 标签匹配 + 执行环境 + 并发配置后,CI 才能正常运行
|
||||
2. **收集首次扫描数据**:CI 跑通后,第一时间整理 gitleaks / pip-audit / vulture 的首次真实扫描结果
|
||||
3. **根据结果调优白名单**:如有误报,及时更新 `.gitleaks.toml` 和 `vulture_whitelist.py`
|
||||
4. **跟进 PR #259 black 格式问题**:等待后端开发修复 `check_migration_safety.py` 后 rebase 验证 vulture
|
||||
5. **推动 gitleaks 预装**:建议在 Runner 镜像中预装 gitleaks,彻底规避下载链路问题
|
||||
6. **推进 npm audit**:前端开发 PR #260 已提交 npm audit,等待 CI 验证
|
||||
7. **PR #264 合并**:安全审查通过,CI 跑通后可合并
|
||||
|
||||
## 六、相关文档
|
||||
|
||||
- [安全工具接入方案](docs/ci/代码安全扫描CI集成方案_report.md)
|
||||
- [安全工具路线图](docs/ci/security-scanning-roadmap.md)
|
||||
- [gitleaks 白名单配置](../.gitleaks.toml)
|
||||
- [vulture 配置](../vulture.conf)
|
||||
- [vulture 白名单](../vulture_whitelist.py)
|
||||
@@ -0,0 +1,105 @@
|
||||
# 安全与质量扫描工具接入路线图
|
||||
|
||||
> 仓库: xiaoxia/xiaoxia-saas
|
||||
> 更新时间: 2026-07-13
|
||||
> 负责人: 代码审计 Agent
|
||||
|
||||
## 整体概览
|
||||
|
||||
```
|
||||
Phase 0 (已完成) Phase 1 (进行中) Phase 2 (规划中) Phase 3 (远期)
|
||||
───────────── ────────────── ───────────── ────────────
|
||||
bandit gitleaks (P0) npm audit (P1) mypy (类型检查)
|
||||
(已存在) pip-audit (P1) vulture (P2) semgrep (SAST)
|
||||
迁移安全检查 依赖自动更新
|
||||
License 合规
|
||||
```
|
||||
|
||||
## 各阶段详情
|
||||
|
||||
### Phase 0: 基线能力(已存在)
|
||||
|
||||
| 工具 | 类型 | 状态 | 说明 |
|
||||
|---|---|---|---|
|
||||
| bandit | Python 代码安全审计 | ✅ 已接入 | validate Job 中运行,`-ll` 级别,阻断模式 |
|
||||
| black | 代码格式化 | ✅ 已接入 | `--check --fast` 模式,阻断模式 |
|
||||
| isort | 导入排序 | ✅ 已接入 | `--check-only` 模式,阻断模式 |
|
||||
| flake8 | 代码风格 | ✅ 已接入 | 阻断模式 |
|
||||
| compileall | 语法检查 | ✅ 已接入 | Python 编译检查 |
|
||||
|
||||
### Phase 1: 安全基础(进行中 / P0-P1)
|
||||
|
||||
预计完成时间:2026-07-13 ~ 2026-07-14
|
||||
|
||||
#### P0: gitleaks 密钥检测
|
||||
- **状态**: ✅ PR 已提交 ([PR #256](https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/pulls/256)),CI 验证中
|
||||
- **接入位置**: validate Job(最前面,快速反馈)
|
||||
- **扫描模式**:
|
||||
- PR 触发:增量扫描(只扫改动文件)
|
||||
- 主分支 push:全量扫描
|
||||
- **阻断策略**: 发现密钥直接阻断合并
|
||||
- **配置文件**: `.gitleaks.toml`(白名单排除示例配置、测试文件等)
|
||||
- **预估收益**: 防止 API Key、密码、Token 等敏感信息泄露
|
||||
|
||||
#### P1: pip-audit Python 依赖漏洞扫描
|
||||
- **状态**: ✅ PR 已提交(同 PR #256),CI 验证中
|
||||
- **接入位置**: validate Job(依赖安装后)
|
||||
- **扫描范围**: 全部 4 个 requirements 文件
|
||||
- **数据源**: OSV(PyPA 官方推荐)
|
||||
- **阻断策略**: 告警模式(初期不阻断)
|
||||
- 运行 1-2 周摸清漏洞存量
|
||||
- 评估后根据严重程度设置阻断阈值(HIGH/CRITICAL 阻断)
|
||||
- **预估收益**: 及时发现 Python 依赖中的 CVE 漏洞
|
||||
|
||||
### Phase 2: 质量与前端安全(P1-P2)
|
||||
|
||||
预计启动时间:Phase 1 稳定后(约 2026-07-15 ~ 2026-07-18)
|
||||
|
||||
#### P1: npm audit 前端依赖漏洞扫描
|
||||
- **状态**: 📋 规划中(前端开发负责)
|
||||
- **接入位置**: frontend-lint Job
|
||||
- **扫描范围**: `package.json` / `package-lock.json`
|
||||
- **初始阈值**: `--audit-level=high`
|
||||
- **阻断策略**: 告警模式(初期不阻断)
|
||||
- **相关 PR**: PR#255(前端缓存优化)后续接入
|
||||
|
||||
#### P2: vulture 死代码检测
|
||||
- **状态**: ✅ PR 已提交 ([PR #259](https://git.xiaoxiajianji.com/xiaoxia/xiaoxia-saas/pull/259))
|
||||
- **接入位置**: validate Job(安全扫描之后)
|
||||
- **置信度阈值**: 80%(逐步调高)
|
||||
- **扫描范围**: `alembic` / `apps` / `packages` / `scripts`
|
||||
- **白名单**: 框架自动调用代码(FastAPI routes、SQLAlchemy models、Celery tasks 等)
|
||||
- **阻断策略**: 告警模式(不阻断)
|
||||
- 运行 1-2 周统计死代码存量
|
||||
- 分批清理确认的死代码
|
||||
- 稳定后逐步提高置信度并考虑阻断
|
||||
- **预估收益**: 减少维护负担,降低安全攻击面
|
||||
|
||||
### Phase 3: 深度质量(远期规划)
|
||||
|
||||
预计启动时间:Phase 2 完成后(约 2026-07 下旬)
|
||||
|
||||
| 工具 | 类型 | 优先级 | 说明 |
|
||||
|---|---|---|---|
|
||||
| mypy | 类型检查 | P1 | 先宽松模式,逐步收紧,提升代码质量 |
|
||||
| semgrep | SAST 静态分析 | P2 | 自定义规则集,检测特定安全模式 |
|
||||
| 依赖自动更新 | 依赖管理 | P2 | Dependabot / Renovate,自动更新依赖版本 |
|
||||
| License 合规 | 合规检查 | P3 | 检查第三方依赖的 License 是否合规 |
|
||||
|
||||
## 接入原则
|
||||
|
||||
1. **先告警后阻断**:新工具初期一律告警模式,摸清存量后再设阈值
|
||||
2. **增量优先**:PR 阶段增量扫描,主分支全量扫描,平衡速度与覆盖
|
||||
3. **白名单先行**:提前配置好合理的白名单,减少误报干扰
|
||||
4. **分层递进**:P0 → P1 → P2 → P3,按优先级逐步接入
|
||||
5. **文档同步**:每个工具接入都同步更新接入方案和最佳实践
|
||||
|
||||
## 进度跟踪
|
||||
|
||||
| 阶段 | 工具 | PR | 状态 | 备注 |
|
||||
|---|---|---|---|---|
|
||||
| Phase 1 | gitleaks | #256 | 🟡 CI 验证中 | 国内镜像下载修复已提交 |
|
||||
| Phase 1 | pip-audit | #256 | 🟡 CI 验证中 | 告警模式 |
|
||||
| Phase 2 | vulture | #259 | 🟡 等待 CI | 80% 置信度,告警模式 |
|
||||
| Phase 2 | npm audit | - | 📋 规划中 | 前端开发负责 |
|
||||
| Phase 3 | mypy | - | 📋 规划中 | 后端开发负责(同迁移安全 PR) |
|
||||
@@ -17,12 +17,6 @@ case "$BUILD_ENV" in
|
||||
esac
|
||||
echo "Build environment: $BUILD_ENV → nginx config: $NGINX_CONF_FILE"
|
||||
|
||||
# 可选:只构建指定镜像(api|worker|web),空则全部构建
|
||||
BUILD_ONLY="${BUILD_ONLY:-}"
|
||||
if [ -n "$BUILD_ONLY" ]; then
|
||||
echo "Build mode: only $BUILD_ONLY"
|
||||
fi
|
||||
|
||||
ROOT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
@@ -66,22 +60,26 @@ fi
|
||||
# 默认只读不写,防止 feature 分支污染主缓存
|
||||
# 只有 develop/main 分支才写回缓存
|
||||
BRANCH_NAME="${GITHUB_REF_NAME:-${CI_COMMIT_BRANCH:-unknown}}"
|
||||
case "$BRANCH_NAME" in
|
||||
develop|main) CACHE_WRITE=1 ; echo "Cache mode: read+write (branch: $BRANCH_NAME)" ;;
|
||||
*) CACHE_WRITE=0 ; echo "Cache mode: read-only (branch: $BRANCH_NAME)" ;;
|
||||
esac
|
||||
if [ "$USE_CACHE" -eq 1 ]; then
|
||||
docker buildx build \
|
||||
--build-arg APP_VERSION="$VERSION" \
|
||||
--cache-from "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_TAG},ignore-error=true" \
|
||||
--cache-to "type=registry,ref=${CACHE_REGISTRY}/api-cache:${CACHE_TAG},mode=max" \
|
||||
-f infra/docker/api.Dockerfile \
|
||||
-t "$API_IMAGE" -t "$API_LATEST" \
|
||||
--load \
|
||||
.
|
||||
else
|
||||
docker build --pull=false --build-arg APP_VERSION="$VERSION" -f infra/docker/api.Dockerfile -t "$API_IMAGE" -t "$API_LATEST" .
|
||||
fi
|
||||
|
||||
# ---- 构建函数:使用 buildx 直接 push 到 registry ----
|
||||
# 不再使用 --load → docker tag → docker push 的串行方式
|
||||
# buildx --push 直接从 buildkit 推送,省去序列化导入 daemon 的开销
|
||||
build_with_cache() {
|
||||
# usage: build_with_cache <image_name> <dockerfile> [extra_args...]
|
||||
# usage: build_with_cache <image_name> <dockerfile> <extra_args...>
|
||||
IMG_NAME="$1"
|
||||
DOCKERFILE="$2"
|
||||
shift 2
|
||||
EXTRA_ARGS="$*"
|
||||
|
||||
REGISTRY_IMG="${REGISTRY}/xiaoxia-saas-${IMG_NAME}:$VERSION"
|
||||
CACHE_FROM="type=registry,ref=${CACHE_REGISTRY}/${IMG_NAME}-cache:${CACHE_TAG_PRIMARY},ignore-error=true"
|
||||
|
||||
if [ "$CACHE_WRITE" -eq 1 ]; then
|
||||
@@ -92,100 +90,96 @@ build_with_cache() {
|
||||
echo " cache: read-only from ${CACHE_REGISTRY}/${IMG_NAME}-cache:${CACHE_TAG_PRIMARY}"
|
||||
fi
|
||||
|
||||
if [ "$USE_CACHE" -eq 1 ] && [ "$USE_PUSH" -eq 1 ]; then
|
||||
if [ "$USE_CACHE" -eq 1 ]; then
|
||||
if [ -n "$CACHE_TO" ]; then
|
||||
docker buildx build \
|
||||
$EXTRA_ARGS \
|
||||
--cache-from "$CACHE_FROM" \
|
||||
--cache-to "$CACHE_TO" \
|
||||
-f "$DOCKERFILE" \
|
||||
-t "$REGISTRY_IMG" \
|
||||
--push \
|
||||
-t "$IMG_NAME:$VERSION" \
|
||||
--load \
|
||||
.
|
||||
else
|
||||
docker buildx build \
|
||||
$EXTRA_ARGS \
|
||||
--cache-from "$CACHE_FROM" \
|
||||
-f "$DOCKERFILE" \
|
||||
-t "$REGISTRY_IMG" \
|
||||
--push \
|
||||
-t "$IMG_NAME:$VERSION" \
|
||||
--load \
|
||||
.
|
||||
fi
|
||||
echo " ✅ Pushed: $REGISTRY_IMG"
|
||||
else
|
||||
# 无push权限时,本地构建用于测试
|
||||
docker build --pull=false $EXTRA_ARGS -f "$DOCKERFILE" -t "xiaoxia-saas-${IMG_NAME}:$VERSION" .
|
||||
echo " ✅ Built locally: xiaoxia-saas-${IMG_NAME}:$VERSION"
|
||||
docker build --pull=false $EXTRA_ARGS -f "$DOCKERFILE" -t "$IMG_NAME:$VERSION" .
|
||||
fi
|
||||
}
|
||||
|
||||
should_build() {
|
||||
# usage: should_build <image_name>
|
||||
# 返回0表示需要构建,1表示跳过
|
||||
local img="$1"
|
||||
if [ -z "$BUILD_ONLY" ]; then
|
||||
return 0 # 全部构建
|
||||
fi
|
||||
if [ "$BUILD_ONLY" = "$img" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
echo "=== Building API image ==="
|
||||
build_with_cache "api" "infra/docker/api.Dockerfile" \
|
||||
"--build-arg APP_VERSION=$VERSION"
|
||||
docker tag "$API_IMAGE" "$API_LATEST"
|
||||
|
||||
if should_build "api"; then
|
||||
echo ""
|
||||
echo "=== Building API image ==="
|
||||
build_with_cache "api" "infra/docker/api.Dockerfile" \
|
||||
"--build-arg APP_VERSION=$VERSION"
|
||||
|
||||
# 本地也打一个 :dev 标签方便本地引用(如果有本地镜像的话)
|
||||
if [ "$USE_PUSH" -eq 0 ]; then
|
||||
docker tag "$API_IMAGE" "$API_LATEST" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if should_build "worker"; then
|
||||
echo ""
|
||||
echo "=== Building Worker image ==="
|
||||
build_with_cache "worker" "infra/docker/worker.Dockerfile" \
|
||||
"--build-arg APP_VERSION=$VERSION"
|
||||
|
||||
if [ "$USE_PUSH" -eq 0 ]; then
|
||||
docker tag "$WORKER_IMAGE" "$WORKER_LATEST" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if should_build "web"; then
|
||||
echo ""
|
||||
echo "=== Building Web image ==="
|
||||
# 先构建前端产物(使用持久化 npm 缓存卷)
|
||||
NPM_CACHE_VOLUME="xiaoxia-npm-cache"
|
||||
if ! docker volume inspect "$NPM_CACHE_VOLUME" >/dev/null 2>&1; then
|
||||
docker volume create "$NPM_CACHE_VOLUME" >/dev/null
|
||||
echo " Created npm cache volume: $NPM_CACHE_VOLUME"
|
||||
fi
|
||||
|
||||
docker run --rm \
|
||||
-v "$PWD:/workspace" \
|
||||
-v "$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules" \
|
||||
-w /workspace/apps/web \
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc "npm ci && npm run build"
|
||||
|
||||
test -f apps/web/dist/index.html
|
||||
|
||||
build_with_cache "web" "infra/docker/web-artifact.Dockerfile" \
|
||||
"--build-arg NGINX_CONF=$NGINX_CONF_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Build complete ==="
|
||||
if [ "$USE_PUSH" -eq 1 ]; then
|
||||
echo "Images pushed to $REGISTRY:"
|
||||
should_build "api" && echo " - $REGISTRY_API"
|
||||
should_build "worker" && echo " - $REGISTRY_WORKER"
|
||||
should_build "web" && echo " - $REGISTRY_WEB"
|
||||
echo "=== Building Worker image ==="
|
||||
if [ "$USE_CACHE" -eq 1 ]; then
|
||||
docker buildx build \
|
||||
--build-arg APP_VERSION="$VERSION" \
|
||||
--cache-from "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_TAG},ignore-error=true" \
|
||||
--cache-to "type=registry,ref=${CACHE_REGISTRY}/worker-cache:${CACHE_TAG},mode=max" \
|
||||
-f infra/docker/worker.Dockerfile \
|
||||
-t "$WORKER_IMAGE" -t "$WORKER_LATEST" \
|
||||
--load \
|
||||
.
|
||||
else
|
||||
echo "Images built locally (registry push skipped)"
|
||||
docker images | grep "xiaoxia-saas.*:$VERSION" || true
|
||||
docker build --pull=false --build-arg APP_VERSION="$VERSION" -f infra/docker/worker.Dockerfile -t "$WORKER_IMAGE" -t "$WORKER_LATEST" .
|
||||
fi
|
||||
|
||||
echo "=== Building Web image (with buildx cache) ==="
|
||||
# 先构建前端产物(使用持久化 npm 缓存卷)
|
||||
NPM_CACHE_VOLUME="xiaoxia-npm-cache"
|
||||
if ! docker volume inspect "$NPM_CACHE_VOLUME" >/dev/null 2>&1; then
|
||||
docker volume create "$NPM_CACHE_VOLUME" >/dev/null
|
||||
echo " Created npm cache volume: $NPM_CACHE_VOLUME"
|
||||
fi
|
||||
|
||||
docker run --rm \
|
||||
-v "$PWD:/workspace" \
|
||||
-v "$NPM_CACHE_VOLUME:/workspace/apps/web/node_modules" \
|
||||
-w /workspace/apps/web \
|
||||
docker.m.daocloud.io/library/node:20 \
|
||||
sh -lc "npm ci && npm run build"
|
||||
|
||||
test -f apps/web/dist/index.html
|
||||
|
||||
if [ "$USE_CACHE" -eq 1 ]; then
|
||||
docker buildx build \
|
||||
--cache-from "type=registry,ref=${CACHE_REGISTRY}/web-cache:${CACHE_TAG},ignore-error=true" \
|
||||
--cache-to "type=registry,ref=${CACHE_REGISTRY}/web-cache:${CACHE_TAG},mode=max" \
|
||||
-f infra/docker/web-artifact.Dockerfile \
|
||||
--build-arg "NGINX_CONF=$NGINX_CONF_FILE" \
|
||||
-t "$WEB_IMAGE" \
|
||||
--load \
|
||||
.
|
||||
else
|
||||
docker build --pull=false \
|
||||
-f infra/docker/web-artifact.Dockerfile \
|
||||
--build-arg "NGINX_CONF=$NGINX_CONF_FILE" \
|
||||
-t "$WEB_IMAGE" \
|
||||
.
|
||||
fi
|
||||
|
||||
# Push 到 Registry
|
||||
if [ "$USE_PUSH" -eq 1 ]; then
|
||||
echo "=== Pushing images to Registry ==="
|
||||
docker tag "$API_IMAGE" "$REGISTRY_API"
|
||||
docker tag "$WORKER_IMAGE" "$REGISTRY_WORKER"
|
||||
docker tag "$WEB_IMAGE" "$REGISTRY_WEB"
|
||||
docker push "$REGISTRY_API"
|
||||
docker push "$REGISTRY_WORKER"
|
||||
docker push "$REGISTRY_WEB"
|
||||
echo "All images pushed to $REGISTRY"
|
||||
else
|
||||
echo "Registry push skipped (no auth token available)"
|
||||
fi
|
||||
|
||||
echo "=== Build complete ==="
|
||||
docker images | grep "xiaoxia-saas.*:$VERSION"
|
||||
|
||||
Reference in New Issue
Block a user