From ea4ae44eb2d09e6b89e43aaae931c00762504fcb Mon Sep 17 00:00:00 2001 From: hmmhmmhm Date: Thu, 8 Oct 2026 09:54:55 +0900 Subject: [PATCH] =?UTF-8?q?ci:=20=EB=B6=84=EB=8B=B9=20=EC=A0=9C=ED=95=9C?= =?UTF-8?q?=20=EA=B2=80=EC=82=AC=EB=A7=8C=20=EB=A7=8C=EB=A3=8C=20=ED=9B=84?= =?UTF-8?q?=20=EC=9E=AC=ED=99=95=EC=9D=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/health-checks.yml | 39 +++-- docs/health-check-stock-and-quota.md | 2 + scripts/ops/health-check-retry.ts | 96 +++++++++++ tests/workflows/health-check-retry.test.ts | 182 +++++++++++++++++++++ tests/workflows/health-runner.test.ts | 39 ++++- tests/workflows/health-summary.test.ts | 6 +- vitest.config.ts | 2 +- 7 files changed, 351 insertions(+), 15 deletions(-) create mode 100644 scripts/ops/health-check-retry.ts create mode 100644 tests/workflows/health-check-retry.test.ts diff --git a/.github/workflows/health-checks.yml b/.github/workflows/health-checks.yml index 59db43de..2e0d8386 100644 --- a/.github/workflows/health-checks.yml +++ b/.github/workflows/health-checks.yml @@ -21,6 +21,10 @@ jobs: timeout-minutes: 10 name: Service health checks steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '24' - name: Run health checks env: HEALTH_CHECK_SECRET: ${{ secrets.HEALTH_CHECK_SECRET }} @@ -55,18 +59,13 @@ jobs: fi cp health-checks.json "health-checks-attempt-${HEALTH_CHECK_ATTEMPT}.json" || return 1 - node <<'NODE' + HEALTH_CHECK_ENDPOINT="${HEALTH_CHECK_ENDPOINT}" node <<'NODE' const fs = require('node:fs'); + (async () => { + const { isValidHealthPayload, retryMinuteChecks } = await import('./scripts/ops/health-check-retry.ts'); let payload; try { payload = JSON.parse(fs.readFileSync('health-checks.json', 'utf8')); } catch {} - const statuses = ['ok', 'degraded', 'fail']; - const valid = payload && statuses.includes(payload.status) && - Array.isArray(payload.checks) && payload.checks.length > 0 && - payload.checks.every(check => check && typeof check.id === 'string' && check.id.length > 0 && - [...statuses, 'skipped'].includes(check.status) && typeof check.message === 'string') && - new Set(payload.checks.map(check => check.id)).size === payload.checks.length && - payload.status === (payload.checks.some(check => check.status === 'fail') ? 'fail' : - payload.checks.some(check => check.status === 'degraded') ? 'degraded' : 'ok'); + const valid = isValidHealthPayload(payload); if (!valid) { const summary = 'Health check returned an invalid or inconsistent payload'; fs.writeFileSync('health-check-summary.txt', summary); @@ -74,6 +73,14 @@ jobs: console.log(summary); process.exitCode = 1; } else { + const firstFailures = payload.checks.filter(check => check.status === 'fail'); + fs.writeFileSync('health-check-summary-first.txt', JSON.stringify({ status: payload.status, failedChecks: firstFailures }, null, 2)); + const recovery = await retryMinuteChecks(payload, { endpoint: process.env.HEALTH_CHECK_ENDPOINT, secret: process.env.HEALTH_CHECK_SECRET }); + for (const attempt of recovery.attempts) { + fs.writeFileSync(`health-checks-recheck-${attempt.id}.json`, JSON.stringify(attempt, null, 2)); + } + payload = recovery.payload; + fs.writeFileSync('health-checks.json', JSON.stringify(payload, null, 2)); const failedChecks = (payload.checks || []).filter((check) => check.status === 'fail'); const degradedChecks = (payload.checks || []).filter((check) => check.status === 'degraded'); const lines = [...failedChecks, ...degradedChecks].map((check) => { @@ -89,6 +96,9 @@ jobs: `degraded=${degradedChecks.length}`, `failedIds=${failedChecks.map((check) => check.id).join(',') || 'none'}`, `degradedIds=${degradedChecks.map((check) => check.id).join(',') || 'none'}`, + `firstFailedIds=${firstFailures.map(check => check.id).join(',') || 'none'}`, + `retryCount=${recovery.attempts.length}`, + `recheckedIds=${recovery.attempts.map(attempt => attempt.id).join(',') || 'none'}`, `run=${runUrl}`, lines.join(' | '), ].filter(Boolean).join(' '); @@ -99,11 +109,17 @@ jobs: process.exitCode = 1; } } + })().catch(() => { + const summary = 'Health check retry runner failed; original attempts remain available'; + fs.writeFileSync('health-check-summary.txt', summary); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary + '\n'); + process.exitCode = 1; + }); NODE } # 실패 직후 전체 검사를 반복하면 정상 경로까지 중계 한도를 다시 소비합니다. - # 원본 실패와 Retry-After는 첫 검사 기록으로 판단하고 다음 점검에서 재확인합니다. + # 검증된 분당 제한만 만료 후 해당 ID를 한 번 재확인하고 최초 실패를 보존합니다. run_health_check - name: Preserve health check attempts @@ -115,6 +131,9 @@ jobs: health-checks-attempt-*.json health-checks-attempt-*.txt health-check-summary.txt + health-check-summary-first.txt + health-checks-recheck-*.json + health-checks.json retention-days: 7 if-no-files-found: warn diff --git a/docs/health-check-stock-and-quota.md b/docs/health-check-stock-and-quota.md index c15dd64d..4091bb71 100644 --- a/docs/health-check-stock-and-quota.md +++ b/docs/health-check-stock-and-quota.md @@ -20,3 +20,5 @@ CU·GS25·세븐일레븐 Mac 중계는 같은 원장을 공유하며 분당 69 CGV·디트릭스 중계는 기존 디트릭스 원장을 공유합니다. 편의점 원장과 분리되어 있으며 이번 복구로 한도를 늘리거나 원장을 초기화하지 않습니다. 편의점 중계가 자체 분·일 한도로 거부하면 HTTP 429에 `x-relay-quota-reason: minute|daily`와 `Retry-After`를 반환합니다. Worker는 이 값을 공개 진단과 재시도 대기 시간에 반영합니다. 외부 서비스가 반환한 429에는 자체 한도 사유를 붙이지 않습니다. + +CI는 검증된 자체 분당 제한(429·minute·1~60초 Retry-After)에 걸린 알려진 검사만 만료 후 한 번 재확인합니다. 전체 스위트를 다시 실행하지 않으며, 일일 제한·외부 오류·사유가 없는 오류는 재시도하지 않습니다. 항목당 20초·재확인 전체 120초 상한을 적용하고, 같은 검사 ID의 캐시되지 않은 정상 응답만 실패를 대체합니다. 선택 항목인 세븐일레븐 인기 검색어는 검증된 HTTP 200 빈 응답도 기존 `skipped` 기준을 유지합니다. 첫 실패·재확인·최종 결과와 재시도 항목을 아티팩트에 보존합니다. 재확인이 실패하면 CI도 계속 실패합니다. diff --git a/scripts/ops/health-check-retry.ts b/scripts/ops/health-check-retry.ts new file mode 100644 index 00000000..47fa0670 --- /dev/null +++ b/scripts/ops/health-check-retry.ts @@ -0,0 +1,96 @@ +/** 분당 중계 한도로 실패한 알려진 검사만 한 번 다시 확인합니다. */ +import type { HealthCheckSummary, HealthCheckResult } from '../../src/api/healthCheckTypes.js'; +export const MINUTE_RECHECK_IDS = [ + 'cu.stores', 'cu.inventory', 'gs25.products', 'gs25.stores', 'gs25.inventory', + 'seveneleven.products', 'seveneleven.stores', 'seveneleven.popwords', 'seveneleven.inventory', + 'cli.contract', 'oliveyoung.products', 'oliveyoung.inventory', 'dtryx.movies', 'cgv.theaters', +]; +function relayErrorCode(id: string): string { + if (id.startsWith('oliveyoung.')) return 'OLIVEYOUNG_RELAY_HTTP_ERROR'; + if (id.startsWith('dtryx.')) return 'DTRYX_RELAY_FAILED'; + if (id.startsWith('cgv.')) return 'CGV_RELAY_FAILED'; + return 'CONVENIENCE_RELAY_FAILED'; +} +function aggregate(checks: HealthCheckResult[]): 'ok' | 'degraded' | 'fail' { + return checks.some(check => check.status === 'fail') ? 'fail' + : checks.some(check => check.status === 'degraded') ? 'degraded' : 'ok'; +} +export function isValidHealthPayload(value: unknown): value is HealthCheckSummary { + if (!value || typeof value !== 'object') return false; + const payload = value as HealthCheckSummary; + return ['ok', 'degraded', 'fail'].includes(payload.status) && + Array.isArray(payload.checks) && payload.checks.length > 0 && + payload.checks.every(check => check && typeof check.id === 'string' && check.id.length > 0 && + ['ok', 'degraded', 'fail', 'skipped'].includes(check.status) && typeof check.message === 'string') && + new Set(payload.checks.map(check => check.id)).size === payload.checks.length && + payload.status === aggregate(payload.checks); +} +interface RetryOptions { + endpoint: string; + secret: string; + fetchImpl?: typeof fetch; + sleep?: (ms: number) => Promise; + now?: () => number; +} +export async function retryMinuteChecks(payload: HealthCheckSummary, options: RetryOptions) { + const attempts: Array<{ id: string; response?: unknown; error?: string }> = []; + if (!isValidHealthPayload(payload)) throw new Error('Invalid health payload'); + const candidates = payload.checks.filter(check => check.status === 'fail' && + MINUTE_RECHECK_IDS.includes(check.id) && check.httpStatus === 429 && check.upstreamStatus === 429 && + (check.id === 'cli.contract' + ? ['CONVENIENCE_RELAY_FAILED', 'OLIVEYOUNG_RELAY_HTTP_ERROR', 'DTRYX_RELAY_FAILED', 'CGV_RELAY_FAILED'].includes(check.errorCode ?? '') + : check.errorCode === relayErrorCode(check.id)) && check.quotaReason === 'minute' && + typeof check.retryAfter === 'number' && Number.isInteger(check.retryAfter) && + check.retryAfter >= 1 && check.retryAfter <= 60); + if (!candidates.length) return { payload, attempts }; + const now = options.now ?? Date.now; + const deadline = now() + 120000; + const sleep = options.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); + await sleep((Math.max(...candidates.map(check => check.retryAfter!)) + 1) * 1000); + const checks = [...payload.checks]; + for (const check of candidates) { + const remaining = deadline - now(); + if (remaining <= 0) break; + const url = new URL(options.endpoint); + url.searchParams.delete('service'); + url.searchParams.set('check', check.id); + url.searchParams.set('mode', 'full'); + url.searchParams.set('fresh', 'true'); + url.searchParams.set('timeoutMs', String(Math.min(15000, remaining))); + const controller = new AbortController(); + let timer!: ReturnType; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(); + reject(new Error('Health recheck timed out')); + }, Math.min(20000, remaining)); + }); + try { + const response: unknown = await Promise.race([(async () => { + const response = await (options.fetchImpl ?? fetch)(url, { + headers: { Authorization: `Bearer ${options.secret}`, 'x-health-check-force-fresh': 'true' }, + redirect: 'error', signal: controller.signal, + }); + if (!response.ok) throw new Error(`Health recheck HTTP ${response.status}`); + return response.json(); + })(), timeout]); + attempts.push({ id: check.id, response }); + if (isValidHealthPayload(response) && response.checks.length === 1 && + response.checks[0].id === check.id && response.cached === false && + (response.checks[0].status === 'ok' || + (check.id === 'seveneleven.popwords' && response.checks[0].status === 'skipped' && + response.checks[0].httpStatus === 200 && response.checks[0].errorCode === undefined && + response.checks[0].upstreamStatus === undefined && response.checks[0].quotaReason === undefined && + response.checks[0].retryAfter === undefined))) { + checks[checks.findIndex(original => original.id === check.id)] = response.checks[0]; + } + } catch { + // 원본 본문과 인증 정보는 오류 아티팩트에 기록하지 않습니다. + attempts.push({ id: check.id, error: 'Health recheck failed or timed out' }); + } finally { + clearTimeout(timer); + controller.abort(); + } + } + return { payload: { ...payload, checks, status: aggregate(checks) }, attempts }; +} diff --git a/tests/workflows/health-check-retry.test.ts b/tests/workflows/health-check-retry.test.ts new file mode 100644 index 00000000..8fb9502b --- /dev/null +++ b/tests/workflows/health-check-retry.test.ts @@ -0,0 +1,182 @@ +/** 최초 실패를 유지하면서 분당 제한 검사만 한 번 재검증합니다. */ +import { afterEach, expect, it, vi } from 'vitest'; +import { HEALTH_CHECKS } from '../../src/api/healthCheckDefinitions.js'; +import type { HealthCheckResult, HealthCheckSummary } from '../../src/api/healthCheckTypes.js'; +import { isValidHealthPayload, retryMinuteChecks, MINUTE_RECHECK_IDS } from '../../scripts/ops/health-check-retry.js'; +function check(id: string, overrides: Partial = {}): HealthCheckResult { + return { id, service: id.split('.')[0], target: '/api', durationMs: 1, status: 'fail', message: 'limited', httpStatus: 429, upstreamStatus: 429, errorCode: 'CONVENIENCE_RELAY_FAILED', quotaReason: 'minute', retryAfter: 14, ...overrides }; +} +function payload(checks: HealthCheckResult[]): HealthCheckSummary { + return { status: checks.some(c => c.status === 'fail') ? 'fail' : checks.some(c => c.status === 'degraded') ? 'degraded' : 'ok', checks, checkedAt: '2026-10-08T00:00:00Z', durationMs: 1, cached: false, filters: { service: null, check: null, mode: 'full' } }; +} +const endpoint = 'https://health.example/api/health/checks?mode=full&fresh=true&timeoutMs=20000'; +afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); }); +it('전체 재실행 없이 최대 Retry-After 이후 알려진 실패 ID만 한 번 확인한다', async () => { + const initial = payload([check('cu.stores'), check('gs25.products', { retryAfter: 21 }), check('cgv.theaters', { status: 'ok' }), check('seveneleven.inventory', { quotaReason: 'daily' })]); + const sleep = vi.fn(async () => undefined); + const fetchImpl = vi.fn(async (url: string | URL | Request) => { + const id = new URL(String(url)).searchParams.get('check')!; + expect(HEALTH_CHECKS.some(c => c.id === id)).toBe(true); + return Response.json(payload([check(id, { status: 'ok', message: 'verified' })])); + }); + const result = await retryMinuteChecks(initial, { endpoint, secret: 'test-secret', sleep, fetchImpl }); + expect(sleep).toHaveBeenCalledExactlyOnceWith(22000); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(fetchImpl.mock.calls.map(c => new URL(String(c[0])).searchParams.get('check'))).toEqual(['cu.stores', 'gs25.products']); + expect(result.payload.status).toBe('fail'); + expect(result.payload.checks.map(c => c.status)).toEqual(['ok', 'ok', 'ok', 'fail']); + expect(initial.checks[0].status).toBe('fail'); + expect(result.attempts).toHaveLength(2); +}); +it.each([0, -1, 1.5, 61, NaN, Infinity, '14', undefined])('무효 재시도 값 %s는 호출하거나 대기하지 않는다', async retryAfter => { + const initial = payload([check('cu.stores', { retryAfter: retryAfter as number })]); + const fetchImpl = vi.fn(); const sleep = vi.fn(); + expect((await retryMinuteChecks(initial, { endpoint, secret: 'secret', fetchImpl, sleep })).payload).toBe(initial); + expect(fetchImpl).not.toHaveBeenCalled(); expect(sleep).not.toHaveBeenCalled(); +}); +it.each(['daily', 'consumer', 'consumer-busy', undefined])('분당 한도가 아닌 %s 실패는 반복하지 않는다', async quotaReason => { + const fetchImpl = vi.fn(); + await retryMinuteChecks(payload([check('cu.stores', { quotaReason: quotaReason as 'daily' })]), { endpoint, secret: 'secret', fetchImpl }); + expect(fetchImpl).not.toHaveBeenCalled(); +}); +it('알 수 없는 ID와 정상·degraded 검사는 다시 호출하지 않는다', async () => { + const fetchImpl = vi.fn(); + await retryMinuteChecks(payload([check('unknown'), check('cu.stores', { status: 'ok' }), check('gs25.products', { status: 'degraded' })]), { endpoint, secret: 'secret', fetchImpl }); + expect(fetchImpl).not.toHaveBeenCalled(); +}); +it.each(['fail', 'degraded', 'skipped', 'other-id', 'invalid', 'http', 'network'])('재확인이 %s이면 최초 실패를 숨기지 않는다', async outcome => { + const initial = payload([check('cu.stores')]); + const fetchImpl = vi.fn(async () => { + if (outcome === 'network') throw new Error('private'); + if (outcome === 'http') return new Response('private', { status: 503 }); + if (outcome === 'invalid') return new Response('broken'); + return Response.json(payload([check(outcome === 'other-id' ? 'gs25.products' : 'cu.stores', { status: (outcome === 'other-id' ? 'ok' : outcome) as HealthCheckResult['status'] })])); + }); + const result = await retryMinuteChecks(initial, { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + expect(result.payload.checks[0]).toBe(initial.checks[0]); expect(result.payload.status).toBe('fail'); + expect(result.attempts).toHaveLength(1); expect(JSON.stringify(result)).not.toContain('private'); +}); +it('기한을 무시하는 fetch도 20초에 종료하고 원래 실패를 유지한다', async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn(() => new Promise(() => undefined)); + const task = retryMinuteChecks(payload([check('cu.stores')]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + await vi.advanceTimersByTimeAsync(20000); + expect((await task).payload.status).toBe('fail'); expect(fetchImpl).toHaveBeenCalledTimes(1); +}); +it('재확인 전체 예산 120초를 소진하면 남은 실패를 그대로 둔다', async () => { + let now = 0; + const fetchImpl = vi.fn(async () => { now += 120000; return Response.json(payload([check('cu.stores', { status: 'ok' })])); }); + const result = await retryMinuteChecks(payload([check('cu.stores'), check('gs25.products')]), { endpoint, secret: 'secret', fetchImpl, now: () => now, sleep: async () => undefined }); + expect(fetchImpl).toHaveBeenCalledTimes(1); expect(result.payload.status).toBe('fail'); expect(result.payload.checks[1].status).toBe('fail'); +}); +it('정확한 재확인이 모두 성공하면 aggregate를 다시 계산한다', async () => { + const result = await retryMinuteChecks(payload([check('cu.stores')]), { endpoint, secret: 'secret', fetchImpl: async () => Response.json(payload([check('cu.stores', { status: 'ok' })])), sleep: async () => undefined }); + expect(result.payload.status).toBe('ok'); +}); +it.each([null, {}, { status: 'ok', checks: [] }, { status: 'ok', checks: [check('cu.stores')] }])('잘못된 최초 응답을 인정하지 않는다', value => expect(isValidHealthPayload(value)).toBe(false)); + +it.each([{ httpStatus: 502 }, { upstreamStatus: 403 }, { errorCode: 'UPSTREAM_FAILED' }, { errorCode: undefined }])('분당 문구가 있어도 중계 429 계약이 아니면 재시도하지 않는다: %j', async fields => { + const fetchImpl = vi.fn(); + await retryMinuteChecks(payload([check('cu.stores', fields)]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + expect(fetchImpl).not.toHaveBeenCalled(); +}); +it('허용 ID 목록은 실제 헬스 정의와 일치한다', () => { + expect(MINUTE_RECHECK_IDS.every(id => HEALTH_CHECKS.some(check => check.id === id))).toBe(true); +}); +it.each([ + ['oliveyoung.products', 'OLIVEYOUNG_RELAY_HTTP_ERROR'], + ['dtryx.movies', 'DTRYX_RELAY_FAILED'], ['cgv.theaters', 'CGV_RELAY_FAILED'], +])('서비스 %s의 자체 중계 코드만 다시 확인한다', async (id, errorCode) => { + const fetchImpl = vi.fn(async () => Response.json(payload([check(id, { status: 'ok' })]))); + expect((await retryMinuteChecks(payload([check(id, { errorCode })]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined })).payload.status).toBe('ok'); +}); +it('기본 대기와 fetch도 가짜 시계로 검증하며 인증·fresh 단일 검사만 보낸다', async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn(async (url: URL, init: RequestInit) => { + expect(url.searchParams.get('check')).toBe('cu.stores'); + expect(url.searchParams.get('service')).toBeNull(); + expect(init.headers).toEqual({ Authorization: 'Bearer secret', 'x-health-check-force-fresh': 'true' }); + return Response.json(payload([check('cu.stores', { status: 'ok' })])); + }); + vi.stubGlobal('fetch', fetchImpl); + const task = retryMinuteChecks(payload([check('cu.stores', { retryAfter: 60 })]), { endpoint: endpoint + '&service=gs25', secret: 'secret' }); + await vi.advanceTimersByTimeAsync(61000); + expect((await task).payload.status).toBe('ok'); +}); +it('기한을 무시하는 JSON 본문도 요청 기한 안에 반환한다', async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn(async () => ({ ok: true, json: () => new Promise(() => undefined) }) as Response); + const task = retryMinuteChecks(payload([check('cu.stores')]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + await vi.advanceTimersByTimeAsync(20000); + expect((await task).payload.status).toBe('fail'); +}); +it('유효하지 않은 최초 payload는 재검증 전에 실패한다', async () => { + await expect(retryMinuteChecks({} as HealthCheckSummary, { endpoint, secret: 'secret' })).rejects.toThrow('Invalid'); +}); +it('남은 degraded 상태는 성공 재검증 뒤에도 집계에 유지한다', async () => { + const result = await retryMinuteChecks(payload([check('cu.stores'), check('gs25.products', { status: 'degraded' })]), { endpoint, secret: 'secret', fetchImpl: async () => Response.json(payload([check('cu.stores', { status: 'ok' })])), sleep: async () => undefined }); + expect(result.payload.status).toBe('degraded'); +}); +it.each([ + { status: 'unknown', checks: [check('cu.stores')] }, + { status: 'fail', checks: [null] }, + { status: 'fail', checks: [check('')] }, + { status: 'fail', checks: [check('cu.stores', { message: 1 as unknown as string })] }, + { status: 'fail', checks: [check('cu.stores'), check('cu.stores')] }, +])('구조·중복·집계가 잘못된 payload는 거부한다: %j', value => expect(isValidHealthPayload(value)).toBe(false)); + +it.each(['CGV_RELAY_FAILED', 'DTRYX_RELAY_FAILED', 'CONVENIENCE_RELAY_FAILED', 'OLIVEYOUNG_RELAY_HTTP_ERROR'])('CLI 계약의 자체 중계 %s 분당 제한도 단일 ID로 재확인한다', async errorCode => { + const fetchImpl = vi.fn(async (url: URL | RequestInfo) => { + expect(new URL(String(url)).searchParams.get('check')).toBe('cli.contract'); + return Response.json(payload([check('cli.contract', { status: 'ok' })])); + }); + const result = await retryMinuteChecks(payload([check('cli.contract', { errorCode })]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + expect(result.payload.status).toBe('ok'); expect(fetchImpl).toHaveBeenCalledTimes(1); +}); +it('CLI 계약의 외부 비분류 오류는 분당 제한 문구가 있어도 반복하지 않는다', async () => { + const fetchImpl = vi.fn(); + await retryMinuteChecks(payload([check('cli.contract', { errorCode: 'UPSTREAM_FAILED' })]), { endpoint, secret: 'secret', fetchImpl, sleep: async () => undefined }); + expect(fetchImpl).not.toHaveBeenCalled(); +}); +it('여러 ID 또는 캐시 결과는 최초 실패를 대체하지 않는다', async () => { + for (const response of [payload([check('cu.stores', { status: 'ok' }), check('gs25.products', { status: 'ok' })]), { ...payload([check('cu.stores', { status: 'ok' })]), cached: true }]) { + const result = await retryMinuteChecks(payload([check('cu.stores')]), { endpoint, secret: 'secret', fetchImpl: async () => Response.json(response), sleep: async () => undefined }); + expect(result.payload.status).toBe('fail'); + } +}); +it('선택적 인기 검색어는 공식 200 빈 결과 skipped를 성공으로 인정한다', async () => { + const response = payload([check('seveneleven.popwords', { + status: 'skipped', message: 'optional data unavailable', httpStatus: 200, + errorCode: undefined, upstreamStatus: undefined, quotaReason: undefined, retryAfter: undefined, + })]); + const result = await retryMinuteChecks(payload([check('seveneleven.popwords')]), { + endpoint, secret: 'secret', sleep: async () => undefined, fetchImpl: async () => Response.json(response), + }); + expect(result.payload.status).toBe('ok'); + expect(result.payload.checks[0].status).toBe('skipped'); +}); +it.each([ + { httpStatus: 429 }, { errorCode: 'CONVENIENCE_RELAY_FAILED' }, { upstreamStatus: 429 }, + { quotaReason: 'minute' as const }, { retryAfter: 1 }, +])('선택적 skipped라도 오류 진단이 남으면 최초 실패를 유지한다: %j', async fields => { + const response = payload([check('seveneleven.popwords', { + status: 'skipped', message: 'optional data unavailable', httpStatus: 200, + errorCode: undefined, upstreamStatus: undefined, quotaReason: undefined, retryAfter: undefined, + ...fields, + })]); + const result = await retryMinuteChecks(payload([check('seveneleven.popwords')]), { + endpoint, secret: 'secret', sleep: async () => undefined, fetchImpl: async () => Response.json(response), + }); + expect(result.payload.status).toBe('fail'); +}); +it('전체 120초 예산에는 최초 61초 대기도 포함한다', async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn(() => new Promise(() => undefined)); + const initial = payload(['cu.stores', 'gs25.products', 'seveneleven.popwords', 'seveneleven.inventory'].map(id => check(id, { retryAfter: 60 }))); + let complete = false; + const task = retryMinuteChecks(initial, { endpoint, secret: 'secret', fetchImpl }).then(result => { complete = true; return result; }); + await vi.advanceTimersByTimeAsync(120000); + expect(complete).toBe(true); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect((await task).payload.status).toBe('fail'); +}); diff --git a/tests/workflows/health-runner.test.ts b/tests/workflows/health-runner.test.ts index 454e9bee..a14a91d2 100644 --- a/tests/workflows/health-runner.test.ts +++ b/tests/workflows/health-runner.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readFileSync, existsSync, copyFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { spawnSync } from 'node:child_process'; @@ -16,18 +16,28 @@ const payload = JSON.stringify({ cached: false, checks: [{ id: 'test.products', status: 'ok', message: '1 item' }], }); -function execute(curl: string, stale = false) { +function execute(curl: string, stale = false, retryPayload?: string) { const dir = mkdtempSync(join(tmpdir(), 'health-runner-')); dirs.push(dir); mkdirSync(join(dir, 'bin')); + mkdirSync(join(dir, 'scripts/ops'), { recursive: true }); + copyFileSync('scripts/ops/health-check-retry.ts', join(dir, 'scripts/ops/health-check-retry.ts')); writeFileSync(join(dir, 'bin/curl'), '#!/bin/bash\n' + curl, { mode: 0o700 }); writeFileSync(join(dir, 'bin/sleep'), '#!/bin/bash\nexit 0\n', { mode: 0o700 }); + if (retryPayload) { + writeFileSync(join(dir, 'retry-response.json'), retryPayload); + writeFileSync(join(dir, 'retry-fetch.mjs'), `import fs from 'node:fs'; + const timer = globalThis.setTimeout; + globalThis.setTimeout = (fn, ms, ...args) => timer(fn, ms === 2000 ? 0 : ms, ...args); + globalThis.fetch = async (url) => { fs.appendFileSync('retry-calls.txt', String(url) + '\\n'); return Response.json(JSON.parse(fs.readFileSync('retry-response.json', 'utf8'))); };`); + } if (stale) writeFileSync(join(dir, 'health-checks.json'), payload); const result = spawnSync('/bin/bash', ['-e', '-c', script], { cwd: dir, encoding: 'utf8', env: { ...process.env, + ...(retryPayload ? { NODE_OPTIONS: '--import=' + join(dir, 'retry-fetch.mjs') } : {}), PATH: join(dir, 'bin') + ':' + process.env.PATH, HEALTH_CHECK_SECRET: 'test-only', HEALTH_CHECK_URL: 'https://test.invalid', @@ -86,3 +96,28 @@ describe('실제 Health workflow shell', () => { expect(readFileSync(join(dir, 'calls.txt'), 'utf8')).not.toContain('cacheBust=true'); }); }); + +it.each([false, true])('워크플로가 최초 실패를 보존하고 정확한 분당 재검사 결과를 집계한다: unrelatedFail=%s', unrelatedFail => { + const limited = { id: 'cu.stores', status: 'fail', message: 'minute limited', httpStatus: 429, upstreamStatus: 429, errorCode: 'CONVENIENCE_RELAY_FAILED', quotaReason: 'minute', retryAfter: 1 }; + const original = JSON.stringify({ status: 'fail', checks: [limited, { id: 'other', status: unrelatedFail ? 'fail' : 'ok', message: 'original result' }] }); + const retry = JSON.stringify({ status: 'ok', cached: false, checks: [{ id: 'cu.stores', status: 'ok', message: 'verified' }] }); + const { dir, result } = execute(`printf '%s' '${original}' > health-checks.json`, false, retry); + expect(result.status, result.stderr).toBe(unrelatedFail ? 1 : 0); + expect(JSON.parse(readFileSync(join(dir, 'health-checks-attempt-1.json'), 'utf8')).checks[0].status).toBe('fail'); + expect(readFileSync(join(dir, 'health-check-summary-first.txt'), 'utf8')).toContain('minute limited'); + expect(readFileSync(join(dir, 'health-check-summary.txt'), 'utf8')).toContain('retryCount=1'); + expect(readFileSync(join(dir, 'health-check-summary.txt'), 'utf8')).toContain('firstFailedIds=cu.stores'); + expect(existsSync(join(dir, 'health-checks-recheck-cu.stores.json'))).toBe(true); + expect(readFileSync(join(dir, 'retry-calls.txt'), 'utf8')).toContain('check=cu.stores'); + expect(JSON.parse(readFileSync(join(dir, 'health-checks.json'), 'utf8')).checks[1].message).toBe('original result'); +}); + +it('선택적 인기 검색어의 공식 200 빈 응답은 최초 제한 기록을 남기고 통과한다', () => { + const original = JSON.stringify({ status: 'fail', checks: [{ id: 'seveneleven.popwords', status: 'fail', message: 'minute limited', httpStatus: 429, upstreamStatus: 429, errorCode: 'CONVENIENCE_RELAY_FAILED', quotaReason: 'minute', retryAfter: 1 }] }); + const retry = JSON.stringify({ status: 'ok', cached: false, checks: [{ id: 'seveneleven.popwords', status: 'skipped', message: 'optional data unavailable', httpStatus: 200 }] }); + const { dir, result } = execute(`printf '%s' '${original}' > health-checks.json`, false, retry); + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(join(dir, 'health-checks-attempt-1.json'), 'utf8')).toContain('minute limited'); + expect(readFileSync(join(dir, 'health-check-summary.txt'), 'utf8')).toContain('firstFailedIds=seveneleven.popwords'); + expect(JSON.parse(readFileSync(join(dir, 'health-checks.json'), 'utf8')).checks[0].status).toBe('skipped'); +}); diff --git a/tests/workflows/health-summary.test.ts b/tests/workflows/health-summary.test.ts index 40c63304..030ded7b 100644 --- a/tests/workflows/health-summary.test.ts +++ b/tests/workflows/health-summary.test.ts @@ -2,12 +2,13 @@ import { readFileSync } from 'node:fs'; import { runInNewContext } from 'node:vm'; import { describe, expect, it } from 'vitest'; +import { isValidHealthPayload, retryMinuteChecks } from '../../scripts/ops/health-check-retry.js'; const workflow = readFileSync('.github/workflows/health-checks.yml', 'utf8').replace(/\r\n/g, '\n'); const script = workflow.split("node <<'NODE'\n")[1].split('\n NODE')[0]; describe('Health Checks 알림 요약', () => { - it('긴 상세 오류보다 모든 실패·저하 ID를 앞에 두고 Actions에는 전체를 남긴다', () => { + it('긴 상세 오류보다 모든 실패·저하 ID를 앞에 두고 Actions에는 전체를 남긴다', async () => { const checks = Array.from({ length: 12 }, (_, index) => ({ id: `service-${index}`, status: index % 2 ? 'degraded' : 'fail', @@ -24,7 +25,8 @@ describe('Health Checks 알림 요약', () => { }, exitCode: 0, }; - runInNewContext(script, { + await runInNewContext(script.replace("await import('./scripts/ops/health-check-retry.ts')", 'helpers'), { + helpers: { isValidHealthPayload, retryMinuteChecks }, require: () => ({ readFileSync: () => JSON.stringify({ status: 'fail', checks }), writeFileSync: (name: string, text: string) => { diff --git a/vitest.config.ts b/vitest.config.ts index 28ee5d34..935048af 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -30,7 +30,7 @@ export default defineConfig({ reporter: ['text', 'text-summary', 'html', 'json'], // 커버리지 대상 파일 - include: ['src/**/*.ts', 'scripts/relay/cache.ts', 'scripts/relay/logging.ts', 'scripts/relay/observed.ts', 'scripts/relay/oliveyoung.ts', 'scripts/relay/dtryx.ts', 'scripts/relay/http-relay.ts', 'scripts/relay/convenience.ts', 'scripts/relay/gs25-session.ts', 'scripts/relay/browser.ts', 'scripts/relay/lifecycle.ts', 'scripts/relay/ownership.ts', 'scripts/relay/guard-controller.ts', 'scripts/relay/guard-runtime.ts', 'scripts/relay/supervisor.ts', 'scripts/relay/quota.ts', 'scripts/relay/consumer.ts', 'scripts/relay/browser-environment.ts', 'scripts/relay/http-lifecycle.ts', 'scripts/relay/transport-lifecycle.ts'], + include: ['scripts/ops/health-check-retry.ts', 'src/**/*.ts', 'scripts/relay/cache.ts', 'scripts/relay/logging.ts', 'scripts/relay/observed.ts', 'scripts/relay/oliveyoung.ts', 'scripts/relay/dtryx.ts', 'scripts/relay/http-relay.ts', 'scripts/relay/convenience.ts', 'scripts/relay/gs25-session.ts', 'scripts/relay/browser.ts', 'scripts/relay/lifecycle.ts', 'scripts/relay/ownership.ts', 'scripts/relay/guard-controller.ts', 'scripts/relay/guard-runtime.ts', 'scripts/relay/supervisor.ts', 'scripts/relay/quota.ts', 'scripts/relay/consumer.ts', 'scripts/relay/browser-environment.ts', 'scripts/relay/http-lifecycle.ts', 'scripts/relay/transport-lifecycle.ts'], // 커버리지 제외 파일 exclude: [