Compare commits

...

4 Commits

Author SHA1 Message Date
xiaoxia 4b0887e9d8 test: update after rebase (simulate force push)
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 19s
CI/CD Pipeline / Frontend Lint (pull_request) Successful in 45s
AI Code Review / AI Code Review (pull_request) Successful in 1m47s
CI/CD Pipeline / Validate Code Quality And Tests (pull_request) Successful in 2m41s
CI/CD Pipeline / Unit Tests (pull_request) Successful in 2m46s
Auto Approve CI PRs / Auto Approve on CI Green (pull_request) Successful in 3m16s
CI/CD Pipeline / Integration Tests (pull_request) Successful in 1m31s
CI Build & Deploy Pipeline / Production Browser E2E (pull_request) Failing after 1468h40m7s
CI Build & Deploy Pipeline / Build Production Worker Image (pull_request) Failing after 1468h40m8s
CI Build & Deploy Pipeline / Build Production Web Image (pull_request) Failing after 1468h40m8s
CI Build & Deploy Pipeline / Staging API Integration Tests (pull_request) Failing after 1468h40m8s
CI Build & Deploy Pipeline / Build Production API Image (pull_request) Failing after 1468h40m9s
CI Build & Deploy Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 1468h40m9s
CI Build & Deploy Pipeline / Build Staging Worker Image (pull_request) Failing after 1468h40m11s
CI Build & Deploy Pipeline / Build Staging API Image (pull_request) Failing after 1468h40m12s
CI Build & Deploy Pipeline / Staging E2E Tests (pull_request) Failing after 1469h12m6s
CI Build & Deploy Pipeline / Build Staging Web Image (pull_request) Failing after 1469h12m10s
CI Build & Deploy Pipeline / Deploy Production (pull_request) Failing after 1469h12m5s
2026-07-17 10:49:32 +08:00
xiaoxia c36551bf30 test: add ci trigger verify file
CI/CD Pipeline / Check if frontend-only change (pull_request) Successful in 14s
CI/CD Pipeline / Validate Code Quality And Tests (pull_request) Has been cancelled
CI/CD Pipeline / Unit Tests (pull_request) Has been cancelled
CI/CD Pipeline / Integration Tests (pull_request) Has been cancelled
CI/CD Pipeline / Frontend Lint (pull_request) Has been cancelled
AI Code Review / AI Code Review (pull_request) Has been cancelled
Auto Approve CI PRs / Auto Approve on CI Green (pull_request) Successful in 52s
CI Build & Deploy Pipeline / Production Browser E2E (pull_request) Failing after 1468h40m59s
CI Build & Deploy Pipeline / Staging API Integration Tests (pull_request) Failing after 1468h41m0s
CI Build & Deploy Pipeline / Staging E2E Tests (pull_request) Failing after 1468h41m1s
CI Build & Deploy Pipeline / Build Production API Image (pull_request) Failing after 1468h41m2s
CI Build & Deploy Pipeline / Build Production Worker Image (pull_request) Failing after 1468h41m1s
CI Build & Deploy Pipeline / Deploy Staging (Watchtower auto-deploy) (pull_request) Failing after 1468h41m2s
CI Build & Deploy Pipeline / Build Staging Web Image (pull_request) Failing after 1468h41m4s
CI Build & Deploy Pipeline / Build Staging API Image (pull_request) Failing after 1468h41m5s
CI Build & Deploy Pipeline / Build Staging Worker Image (pull_request) Failing after 1469h13m2s
CI Build & Deploy Pipeline / Deploy Production (pull_request) Failing after 1469h12m58s
CI Build & Deploy Pipeline / Build Production Web Image (pull_request) Failing after 1469h12m59s
2026-07-17 10:48:36 +08:00
xiaoxia 3c9509d373 feat(ci): P1-5 CI触发可靠性监控 - 定时检测PR未触发CI并告警 (#435)
CI Build & Deploy Pipeline / Build Staging API Image (push) Has been cancelled
CI Build & Deploy Pipeline / Build Staging Web Image (push) Has been cancelled
CI Build & Deploy Pipeline / Build Staging Worker Image (push) Has been cancelled
CI Build & Deploy Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Has been cancelled
CI Build & Deploy Pipeline / Staging E2E Tests (push) Has been cancelled
CI Build & Deploy Pipeline / Staging API Integration Tests (push) Has been cancelled
CI Build & Deploy Pipeline / Build Production API Image (push) Has been cancelled
CI Build & Deploy Pipeline / Build Production Web Image (push) Has been cancelled
CI Build & Deploy Pipeline / Build Production Worker Image (push) Has been cancelled
CI Build & Deploy Pipeline / Deploy Production (push) Has been cancelled
CI Build & Deploy Pipeline / Production Browser E2E (push) Has been cancelled
CI/CD Pipeline / Validate Code Quality And Tests (push) Has been cancelled
CI/CD Pipeline / Unit Tests (push) Has been cancelled
CI/CD Pipeline / Integration Tests (push) Has been cancelled
CI/CD Pipeline / Check if frontend-only change (push) Failing after 1468h41m39s
CI/CD Pipeline / Frontend Lint (push) Has been skipped
2026-07-17 10:48:06 +08:00
xiaoxia fbb4f2f4c0 fix(ci): 修复auto-approve YAML语法问题 - 内联Python多行缩进导致workflow不触发 (#438)
CI/CD Pipeline / Validate Code Quality And Tests (push) Has been cancelled
CI/CD Pipeline / Unit Tests (push) Has been cancelled
CI/CD Pipeline / Integration Tests (push) Has been cancelled
CI/CD Pipeline / Frontend Lint (push) Has been cancelled
CI Build & Deploy Pipeline / Production Browser E2E (push) Failing after 1468h41m22s
CI Build & Deploy Pipeline / Build Production Worker Image (push) Failing after 1468h41m24s
CI Build & Deploy Pipeline / Build Production Web Image (push) Failing after 1468h41m24s
CI Build & Deploy Pipeline / Build Production API Image (push) Failing after 1468h41m25s
CI/CD Pipeline / Check if frontend-only change (push) Failing after 1468h41m53s
CI Build & Deploy Pipeline / Build Staging API Image (push) Has been skipped
CI Build & Deploy Pipeline / Build Staging Web Image (push) Has been skipped
CI Build & Deploy Pipeline / Build Staging Worker Image (push) Has been skipped
CI Build & Deploy Pipeline / Deploy Staging (Watchtower auto-deploy) (push) Has been skipped
CI Build & Deploy Pipeline / Staging E2E Tests (push) Has been skipped
CI Build & Deploy Pipeline / Staging API Integration Tests (push) Has been skipped
CI Build & Deploy Pipeline / Deploy Production (push) Failing after 1469h13m21s
2026-07-17 10:47:52 +08:00
5 changed files with 354 additions and 59 deletions
+34 -59
View File
@@ -11,39 +11,37 @@ jobs:
if: github.event_name == 'pull_request' && !github.event.pull_request.draft
timeout-minutes: 20
steps:
- name: Check if frontend-only change
id: frontend-only
- name: Checkout code
uses: actions/checkout@v3
- name: Auto approve when CI passes
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
set -eu
echo "PR #${PR_NUMBER} - 检查CI状态并自动审批"
# 检查是否纯前端改动
API_URL="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?limit=300"
FILES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "$API_URL" | python3 -c "import sys,json; [print(f['filename']) for f in json.load(sys.stdin)]")
FRONTEND_COUNT=$(echo "$FILES" | grep -c '^apps/web/' || true)
BACKEND_COUNT=$(echo "$FILES" | grep -cv '^apps/web/' || true)
TOTAL=$(echo "$FILES" | grep -cv '^$' || true)
echo "变更文件: ${TOTAL} 个 (前端: ${FRONTEND_COUNT}, 后端/公共: ${BACKEND_COUNT})"
if [ "$BACKEND_COUNT" = "0" ] && [ "$FRONTEND_COUNT" -gt "0" ]; then
echo "skip_backend=true" >> $GITHUB_OUTPUT
echo "✅ 纯前端改动,跳过后端CI检查"
SKIP_BACKEND=true
echo "✅ 纯前端改动,只检查Frontend Lint"
else
echo "skip_backend=false" >> $GITHUB_OUTPUT
SKIP_BACKEND=false
echo "🔧 包含后端/公共变更,检查全部CI"
fi
- name: Wait for CI and auto approve
shell: bash
env:
GITHUB_TOKEN: ${{ github.token }}
REVIEW_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
SKIP_BACKEND: ${{ steps.frontend-only.outputs.skip_backend }}
run: |
set -eu
# 定义需要检查的status context
# 定义需要检查的context
if [ "$SKIP_BACKEND" = "true" ]; then
CONTEXTS=("CI/CD Pipeline / Frontend Lint (pull_request)")
else
@@ -53,39 +51,25 @@ jobs:
"CI/CD Pipeline / Frontend Lint (pull_request)"
)
fi
echo "需要通过的CI检查: ${#CONTEXTS[@]} 项"
for ctx in "${CONTEXTS[@]}"; do
echo " - $ctx"
done
echo
# 轮询等待,最多20分钟(120次x10秒)
for attempt in $(seq 1 120); do
ALL_SUCCESS=true
ANY_FAILED=false
echo "--- 第${attempt}次检查 ($(date '+%H:%M:%S')) ---"
# 获取当前commit的所有status
STATUSES=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/statuses?per_page=100")
# 调用辅助脚本检查每个context状态
for ctx in "${CONTEXTS[@]}"; do
STATE=$(echo "$STATUSES" | python3 -c "
import sys, json
statuses = json.load(sys.stdin)
target = '$ctx'
# 找最新的对应context
latest = None
for s in statuses:
if s.get('context') == target:
latest = s
break # API返回的是倒序,第一个就是最新的
print(latest.get('state', 'pending') if latest else 'pending')
")
STATE=$(python3 scripts/check_ci_status.py "$GITHUB_TOKEN" "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$ctx")
echo " $ctx: $STATE"
if [ "$STATE" != "success" ]; then
ALL_SUCCESS=false
fi
@@ -93,30 +77,21 @@ print(latest.get('state', 'pending') if latest else 'pending')
ANY_FAILED=true
fi
done
if [ "$ALL_SUCCESS" = "true" ]; then
echo
echo "✅ 所有CI检查通过,自动审批 PR #${PR_NUMBER}"
# 先检查是否已有审批,避免重复审批
# 检查是否已有审批
EXISTING=$(curl -s -H "Authorization: token ${REVIEW_TOKEN}" \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" \
| python3 -c "
import sys, json
reviews = json.load(sys.stdin)
for r in reviews:
if r.get('state') == 'APPROVED' and r.get('user',{}).get('login') == 'xiaoxia':
print('yes')
break
else:
print('no')
")
| python3 -c "import sys,json; reviews=json.load(sys.stdin); print('yes' if any(r.get('state')=='APPROVED' and r.get('user',{}).get('login')=='xiaoxia' for r in reviews) else 'no')")
if [ "$EXISTING" = "yes" ]; then
echo "ℹ️ PR #${PR_NUMBER} 已有审批,跳过"
exit 0
fi
# 提交审批
HTTP_CODE=$(curl -s -o /tmp/approve_resp.json -w "%{http_code}" \
-X POST \
@@ -124,11 +99,11 @@ else:
-H "Content-Type: application/json" \
-d '{"event": "APPROVE", "body": "CI全绿,自动审批通过。"}' \
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews")
echo "审批API HTTP状态: $HTTP_CODE"
cat /tmp/approve_resp.json 2>/dev/null || true
echo
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "201" ]; then
echo "✅ 自动审批成功"
exit 0
@@ -137,16 +112,16 @@ else:
exit 1
fi
fi
if [ "$ANY_FAILED" = "true" ]; then
echo
echo "❌ CI检查有失败项,不自动审批"
exit 0 # 正常退出,不算workflow失败
exit 0
fi
sleep 10
done
echo
echo "⏰ 等待超时(20分钟),CI尚未全部完成"
exit 0 # 超时也正常退出,不产生告警
exit 0
+36
View File
@@ -0,0 +1,36 @@
name: CI Trigger Monitor
on:
schedule:
- cron: '*/5 * * * *' # 每5分钟检查一次
workflow_dispatch:
inputs:
stale_threshold:
description: 'CI未触发告警阈值(分钟)'
required: false
default: '5'
permissions:
contents: read
jobs:
monitor:
name: Monitor CI Trigger Reliability
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Check CI trigger status for all open PRs
env:
GITEA_API_TOKEN: ${{ secrets.REVIEW_GITEA_TOKEN }}
GITEA_URL: https://git.xiaoxiajianji.com
GITEA_REPO: xiaoxia/xiaoxia-saas
CI_NOTIFY_WEBHOOK: ${{ secrets.CI_NOTIFY_WEBHOOK }}
STALE_THRESHOLD_MIN: ${{ inputs.stale_threshold || 5 }}
run: |
set +e
python3 scripts/ci_trigger_monitor.py
# 监控脚本永远不fail,避免告警风暴
exit 0
+1
View File
@@ -0,0 +1 @@
# CI Trigger Rebase Verify Test - Updated
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
"""检查指定commit的CI status状态。
用法: python3 check_ci_status.py <token> <repo> <sha> <context>
返回: 打印状态 (success/failure/pending/error)
"""
import json
import sys
import urllib.error
import urllib.request
def main():
if len(sys.argv) != 5:
print("pending")
return
token = sys.argv[1]
repo = sys.argv[2]
sha = sys.argv[3]
target_context = sys.argv[4]
api_url = f"https://git.xiaoxiajianji.com/api/v1/repos/{repo}/commits/{sha}/statuses?per_page=100"
req = urllib.request.Request(api_url, headers={"Authorization": f"token {token}"})
try:
with urllib.request.urlopen(req, timeout=30) as resp:
statuses = json.loads(resp.read().decode())
except Exception:
print("pending")
return
# API返回按时间倒序,第一个就是最新的
for s in statuses:
if s.get("context") == target_context:
print(s.get("status", "pending"))
return
print("pending")
if __name__ == "__main__":
main()
+239
View File
@@ -0,0 +1,239 @@
#!/usr/bin/env python3
"""
CI触发可靠性监控 - 定时检查PR的CI触发状态
- 监控open PR的最新commit是否在5分钟内触发了CI
- 异常时通过飞书webhook告警
环境变量:
GITEA_API_TOKEN - Gitea API Token (必填)
GITEA_REPO - 仓库路径,如 xiaoxia/xiaoxia-saas
GITEA_URL - Gitea地址,如 https://git.xiaoxiajianji.com
CI_NOTIFY_WEBHOOK - 飞书告警webhook (必填)
CHECK_INTERVAL_MIN - 检查间隔(分钟),默认5
STALE_THRESHOLD_MIN - CI未触发告警阈值(分钟),默认5
"""
import json
import os
import sys
import time
import urllib.error
import urllib.request
def get_env(name, default=""):
return os.environ.get(name, default)
def api_get(path):
"""调用Gitea API"""
token = get_env("GITEA_API_TOKEN")
base_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com")
repo = get_env("GITEA_REPO", "xiaoxia/xiaoxia-saas")
url = f"{base_url}/api/v1/repos/{repo}{path}"
req = urllib.request.Request(url)
req.add_header("Authorization", f"token {token}")
for attempt in range(3):
try:
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read())
except urllib.error.HTTPError as e:
if e.code >= 500 and attempt < 2:
time.sleep(2**attempt)
continue
raise
except Exception as e:
if attempt < 2:
time.sleep(2**attempt)
continue
raise
def get_open_prs():
"""获取所有open PR"""
prs = []
page = 1
while True:
batch = api_get(f"/pulls?state=open&sort=updated&direction=desc&limit=50&page={page}")
if not batch:
break
prs.extend(batch)
if len(batch) < 50:
break
page += 1
return prs
def get_commit_status(sha):
"""获取commit的CI状态"""
try:
return api_get(f"/commits/{sha}/status")
except Exception as e:
print(f" ⚠️ 获取commit状态失败: {e}")
return {"state": "error", "statuses": []}
def has_ci_started(statuses):
"""判断是否有CI job已经启动(pending/running/success/failure都算启动了)"""
pr_statuses = [s for s in statuses if "pull_request" in s.get("context", "")]
if not pr_statuses:
return False
# 只要有非pending且非空的状态,就算启动了
for s in pr_statuses:
if s.get("status") in ["success", "failure", "running"]:
return True
if s.get("status") == "pending" and "Has started running" in s.get("description", ""):
return True
# 全是"Blocked by required conditions"的pending也算(说明CI系统收到了事件)
for s in pr_statuses:
if "Blocked" in s.get("description", ""):
return True
return False
def send_alert(pr_num, pr_title, pr_url, head_sha, commit_age_min):
"""发送飞书告警"""
webhook = get_env("CI_NOTIFY_WEBHOOK")
if not webhook:
print(" ⚠️ 未配置CI_NOTIFY_WEBHOOK,跳过告警")
return
gitea_url = get_env("GITEA_URL", "https://git.xiaoxiajianji.com")
content = {
"msg_type": "interactive",
"card": {
"header": {
"title": {"tag": "plain_text", "content": f"⚠️ CI告警 - PR#{pr_num} CI未触发"},
"template": "red",
},
"elements": [
{
"tag": "div",
"text": {
"tag": "lark_md",
"content": f"**PR**: [{pr_title}]({pr_url})\n**最新commit**: `{head_sha[:12]}`\n**已等待**: {commit_age_min:.0f} 分钟仍无CI启动\n**可能原因**: Gitea Actions事件丢失 / Webhook失败 / Runner资源不足",
},
},
{
"tag": "action",
"actions": [
{
"tag": "button",
"text": {"tag": "plain_text", "content": "查看PR"},
"url": pr_url,
"type": "primary",
},
{
"tag": "button",
"text": {"tag": "plain_text", "content": "查看Actions"},
"url": f"{pr_url}/files",
"type": "default",
},
],
},
{
"tag": "note",
"elements": [
{"tag": "plain_text", "content": f"CI触发监控 | 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}"}
],
},
],
},
}
try:
data = json.dumps(content).encode()
req = urllib.request.Request(webhook, data=data, method="POST")
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=10) as resp:
resp.read()
print(f" 📢 告警已发送: PR#{pr_num}")
except Exception as e:
print(f" ⚠️ 告警发送失败: {e}")
def main():
stale_threshold = int(get_env("STALE_THRESHOLD_MIN", "5"))
print("=" * 60)
print(f"CI触发监控 - 检测时间: {time.strftime('%Y-%m-%d %H:%M:%S')}")
print(f"告警阈值: {stale_threshold}分钟无CI启动")
print("=" * 60)
# 获取open PR列表
try:
prs = get_open_prs()
except Exception as e:
print(f"❌ 获取PR列表失败: {e}")
sys.exit(0) # 告警脚本不阻断CI
print(f"\n共 {len(prs)} 个open PR\n")
stale_prs = []
now = time.time()
for pr in prs:
pr_num = pr["number"]
pr_title = pr["title"]
pr_url = pr["html_url"]
head_sha = pr["head"]["sha"]
updated_at = pr["updated_at"]
# 解析updated_at(ISO格式)
try:
# 2026-07-17T09:22:43+08:00
from datetime import datetime, timedelta, timezone
# 简化处理:直接用字符串解析
ts_str = updated_at.replace("Z", "+00:00")
# 手动解析
dt = datetime.fromisoformat(ts_str)
commit_time = dt.timestamp()
except Exception as e:
print(f" ⚠️ PR#{pr_num} 时间解析失败: {e}")
continue
age_min = (now - commit_time) / 60
print(f"PR#{pr_num:3d} | {pr_title[:45]:45s} | 更新于 {age_min:.0f}min前")
# 少于2分钟的跳过,给CI一点启动时间
if age_min < 2:
print(f" ⏳ 刚更新,等待CI启动...")
continue
# 获取commit状态
status = get_commit_status(head_sha)
statuses = status.get("statuses", [])
if has_ci_started(statuses):
print(f" ✅ CI已启动 (state={status.get('state')})")
continue
# CI未启动,判断是否超过阈值
if age_min >= stale_threshold:
print(f" 🚨 CI未触发!已等待 {age_min:.0f} 分钟")
stale_prs.append({"num": pr_num, "title": pr_title, "url": pr_url, "sha": head_sha, "age_min": age_min})
else:
print(f" ⏳ CI尚未启动 ({age_min:.0f}min < {stale_threshold}min阈值)")
# 发送告警
print(f"\n{'=' * 60}")
print(f"检测结果: {len(stale_prs)} 个PR CI未触发超过阈值")
if stale_prs:
print("\n告警列表:")
for pr in stale_prs:
print(f" - PR#{pr['num']}: {pr['title'][:40]} ({pr['age_min']:.0f}min)")
send_alert(pr["num"], pr["title"], pr["url"], pr["sha"], pr["age_min"])
else:
print("✅ 所有PR CI触发正常")
print("=" * 60)
if __name__ == "__main__":
main()