Repository navigation
Health Checks #984
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Health Checks | |
| on: | |
| schedule: | |
| # 3시간마다 quick 체크 실행 | |
| - cron: '10 0,3,6,9,12,18,21 * * *' | |
| # 한국시간 00:10 = UTC 15:10, 하루 1회 full fresh 체크 실행 | |
| - cron: '10 15 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: health-checks-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| health-checks: | |
| runs-on: ubuntu-latest | |
| name: Service health checks | |
| steps: | |
| - name: Run health checks | |
| env: | |
| HEALTH_CHECK_SECRET: ${{ secrets.HEALTH_CHECK_SECRET }} | |
| HEALTH_CHECK_URL: ${{ vars.HEALTH_CHECK_URL || 'https://mcp.aka.page' }} | |
| HEALTH_CHECK_SCHEDULE: ${{ github.event.schedule }} | |
| run: | | |
| if [ -z "${HEALTH_CHECK_SECRET}" ]; then | |
| echo "HEALTH_CHECK_SECRET is not set" | |
| exit 1 | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] || [ "${HEALTH_CHECK_SCHEDULE:-}" = "10 15 * * *" ]; then | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=full&fresh=true&includeSamples=true&timeoutMs=20000&slowThresholdMs=9000" | |
| HEALTH_CHECK_FORCE_FRESH="true" | |
| else | |
| HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_URL}/api/health/checks?mode=quick&timeoutMs=5000&slowThresholdMs=3000" | |
| HEALTH_CHECK_FORCE_FRESH="false" | |
| fi | |
| run_health_check() { | |
| curl -fsS \ | |
| -H "Authorization: Bearer ${HEALTH_CHECK_SECRET}" \ | |
| -H "x-health-check-force-fresh: ${HEALTH_CHECK_FORCE_FRESH}" \ | |
| "${HEALTH_CHECK_ENDPOINT}" \ | |
| -o health-checks.json | |
| node <<'NODE' | |
| const fs = require('node:fs'); | |
| const payload = JSON.parse(fs.readFileSync('health-checks.json', 'utf8')); | |
| const failedChecks = (payload.checks || []).filter((check) => check.status === 'fail'); | |
| const degradedChecks = (payload.checks || []).filter((check) => check.status === 'degraded'); | |
| const lines = [...failedChecks, ...degradedChecks].map((check) => { | |
| const sample = check.sample?.first ? ` sample=${check.sample.first}` : ''; | |
| return `${check.id}:${check.status}:${check.message}${sample}`; | |
| }); | |
| const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const summary = [ | |
| `status=${payload.status || 'unknown'}`, | |
| `failed=${failedChecks.length}`, | |
| `degraded=${degradedChecks.length}`, | |
| `failedIds=${failedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `degradedIds=${degradedChecks.map((check) => check.id).join(',') || 'none'}`, | |
| `run=${runUrl}`, | |
| lines.join(' | '), | |
| ].filter(Boolean).join(' '); | |
| fs.writeFileSync('health-check-summary.txt', summary); | |
| fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `## Service health checks\n\n${summary}\n\n`); | |
| console.log(summary); | |
| if (failedChecks.length > 0 || payload.status === 'fail') { | |
| process.exitCode = 1; | |
| } | |
| NODE | |
| } | |
| if ! run_health_check; then | |
| echo "Health check failed on attempt 1; retrying once" | |
| sleep 10 | |
| run_health_check | |
| fi | |
| - name: Notify health failure | |
| if: failure() && github.event_name == 'schedule' && github.event.schedule == '10 15 * * *' | |
| env: | |
| MOSHI_WEBHOOK_TOKEN: ${{ secrets.MOSHI_WEBHOOK_TOKEN }} | |
| run: | | |
| if [ -z "${MOSHI_WEBHOOK_TOKEN}" ]; then | |
| echo "MOSHI_WEBHOOK_TOKEN is not set; skipping failure notification" | |
| exit 0 | |
| fi | |
| SUMMARY="$(cat health-check-summary.txt 2>/dev/null || echo 'health check command failed')" node <<'NODE' | |
| const fs = require('node:fs'); | |
| fs.writeFileSync( | |
| 'moshi-payload.json', | |
| JSON.stringify({ | |
| token: process.env.MOSHI_WEBHOOK_TOKEN, | |
| title: 'Health Checks Failed', | |
| message: `${process.env.GITHUB_REPOSITORY}: ${process.env.SUMMARY || ''}`.slice(0, 3500), | |
| }), | |
| ); | |
| NODE | |
| curl --fail-with-body --silent --show-error \ | |
| -X POST https://api.getmoshi.app/api/webhook \ | |
| -H "Content-Type: application/json" \ | |
| --data-binary @moshi-payload.json |