diff --git a/docs/macos-ops-runbook.md b/docs/macos-ops-runbook.md index 9b3e0457..945b56f2 100644 --- a/docs/macos-ops-runbook.md +++ b/docs/macos-ops-runbook.md @@ -18,10 +18,11 @@ ## GUI 세션 활성화 -실제 경로를 설치 계획에 맞춘다. 이전 수동 숨김 작업 `page.aka.daiso-oliveyoung-hide`를 사용 중이면 검토 후 그 작업만 먼저 `launchctl bootout`한다. 중계 브라우저는 종료하지 않는다. 새 숨김 작업은 `page.aka.daiso-ops-hide`, 감시기는 `page.aka.daiso-ops-monitor`이다. +실제 경로를 설치 계획에 맞춘다. 이전 수동 숨김 작업 `page.aka.daiso-oliveyoung-hide`를 사용 중이면 검토 후 그 작업만 먼저 `launchctl bootout`한다. 중계 브라우저는 종료하지 않는다. 기존 plist를 보존하더라도 다음 로그인 때 중복 실행되지 않도록 이전 작업을 disable한다. 새 숨김 작업은 `page.aka.daiso-ops-hide`, 감시기는 `page.aka.daiso-ops-monitor`이다. ```sh launchctl bootout "gui/$(id -u)/page.aka.daiso-oliveyoung-hide" +launchctl disable "gui/$(id -u)/page.aka.daiso-oliveyoung-hide" launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/page.aka.daiso-ops-hide.plist" launchctl bootstrap "gui/$(id -u)" "$HOME/Library/LaunchAgents/page.aka.daiso-ops-monitor.plist" ``` @@ -56,4 +57,4 @@ launchctl bootout "gui/$(id -u)/page.aka.daiso-ops-hide" launchctl bootout "gui/$(id -u)/page.aka.daiso-ops-monitor" ``` -그 뒤 두 `page.aka.daiso-ops-*.plist`와 필요 없는 해당 plist 백업, `ops-helpers`, `ops-status.json`만 검토하여 삭제한다. 중계·터널 plist와 `relay.env`, `dtryx.env`, `state`, 인증·쿼터 원장은 보존한다. 기존 수동 숨김 도우미로 되돌리려면 보존한 기존 plist를 다시 bootstrap한다. 재부팅 후 실동작은 별도로 확인한다. +그 뒤 두 `page.aka.daiso-ops-*.plist`와 필요 없는 해당 plist 백업, `ops-helpers`, `ops-status.json`만 검토하여 삭제한다. 중계·터널 plist와 `relay.env`, `dtryx.env`, `state`, 인증·쿼터 원장은 보존한다. 기존 수동 숨김 도우미로 되돌리려면 이전 `page.aka.daiso-oliveyoung-hide` 작업을 `launchctl enable`한 뒤 보존한 기존 plist를 다시 bootstrap한다. 재부팅 후 실동작은 별도로 확인한다. diff --git a/docs/relay-observability.md b/docs/relay-observability.md index 309d2eb1..dc426367 100644 --- a/docs/relay-observability.md +++ b/docs/relay-observability.md @@ -85,3 +85,5 @@ Zyte의 키·계정 상태와 관계없이 유료 경로는 계속 차단한다. GS25 세션은 재고 요청이 없어도 중계 시작 시와 분당 준비한다. access 만료 5분 전에는 정상 갱신 경로를 이용하며, 실제 요청과 자동 준비가 중복 갱신과 실패 대기를 공유한다. 편의점 인증 health의 `authentication`은 토큰 대신 상태·만료 시각·로그인 필요 여부·재시도 시각만 반환한다. `login_required`는 사용자 입력이 필요하고 `retry_wait`는 일시적인 갱신 실패 대기다. 전자는 확인 즉시 사용자에게 알리며 과거 로그인 대기 기록만으로 현재 복구를 생략하지 않는다. 공개 MCP 외부 스모크는 CLI와 같은 12개 서비스를 검사한다. 소매 재고는 상품 존재뿐 아니라 실제 매장 확인과 알려진 수량을 요구하며 정상 수량0과 미확인을 구분한다. 일본 다이소 상세는 다른 상품도 원본 진입을 직렬화하여 첫429 이후 공유 대기를 우회하지 못한다. 외부429와 자체 분당·일일 한도는 계속 구분한다. 메가박스의 기존 상세 캐시·중복 요청 공유·취소 기한·지연 경고 기준은 유지하며 원본 지연을 해결했다고 간주하지 않는다. + +주기 중계 health의 일시 통신 오류·3초 기한 초과·HTTP5xx에는 같은 검사 안에서 한 번만 확인 GET을 수행한다. 확인 성공 시 `recovered-network-error`, `recovered-timeout`, `recovered-http-5xx` 계열 진단을 보존하고 최종 준비 상태를 기록한다. 두 번 모두 실패하면 실패 상태를 유지한다. 인증401/403, 잘못된 응답, 준비되지 않은 상태, 설정 오류는 반복하지 않는다. 이 확인은 사용자 요청이나 실제 재고 API에 적용하지 않는다. HTTP 진단에는 상태 코드만 저장하고 원문·토큰은 저장하지 않는다. diff --git a/scripts/ops/mcp-smoke.ts b/scripts/ops/mcp-smoke.ts index fa49f92a..1db17454 100644 --- a/scripts/ops/mcp-smoke.ts +++ b/scripts/ops/mcp-smoke.ts @@ -226,8 +226,16 @@ export const MCP_SMOKE_SCENARIOS: McpSmokeScenario[] = [ label: `${service} 실제 매장 재고`, toolName: `${service}_check_inventory`, args: { - keyword: service === 'gs25' ? '콜라' : service === 'oliveyoung' ? '선크림' : '커피', - storeLimit: 1, + keyword: + service === 'gs25' + ? '콜라' + : service === 'oliveyoung' + ? '선크림' + : service === 'seveneleven' + ? '핫식스' + : '커피', + // 이마트24는 수량이 없는 점포를 응답에서 생략하므로 후보 10곳을 한 번에 묶어 조회합니다. + storeLimit: service === 'emart24' ? 10 : 1, timeoutMs: 20000, ...(service === 'seveneleven' || service === 'emart24' ? { storeKeyword: '강남' } : {}), ...(service === 'gs25' || service === 'cu' || service === 'oliveyoung' diff --git a/src/api/healthCheckDefinitions.ts b/src/api/healthCheckDefinitions.ts index 4d767dbb..a6ce447a 100644 --- a/src/api/healthCheckDefinitions.ts +++ b/src/api/healthCheckDefinitions.ts @@ -193,7 +193,7 @@ export const HEALTH_CHECKS: HealthCheckDefinition[] = [ service: 'emart24', target: 'inventory', mode: 'deep', - path: '/api/emart24/inventory?keyword=%EC%BB%A4%ED%94%BC&storeKeyword=%EA%B0%95%EB%82%A8&limit=1', + path: '/api/emart24/inventory?keyword=%EC%BB%A4%ED%94%BC&storeKeyword=%EA%B0%95%EB%82%A8&storeLimit=10', collectionKey: 'stores', requiredFields: ['storeCode', 'storeName', 'name'], degradedFailurePatterns: EMART24_UPSTREAM_403_PATTERNS, diff --git a/src/utils/routeHealth.ts b/src/utils/routeHealth.ts index abd0a12c..4b0310ad 100644 --- a/src/utils/routeHealth.ts +++ b/src/utils/routeHealth.ts @@ -72,7 +72,7 @@ export function validReason(value: unknown): boolean { return ( value === undefined || (typeof value === 'string' && - [ + ([ 'ok', 'http', 'network', @@ -89,7 +89,10 @@ export function validReason(value: unknown): boolean { 'response-too-large', 'cancelled', 'invalid-input', - ].includes(value)) + 'recovered-network-error', + 'recovered-timeout', + ].includes(value) || + /^(?:http-[1-5][0-9]{2}|recovered-http-5[0-9]{2})$/.test(value))) ); } export function validTime(value: unknown): value is number { diff --git a/src/utils/routeProbes.ts b/src/utils/routeProbes.ts index 84eb361c..af227ba5 100644 --- a/src/utils/routeProbes.ts +++ b/src/utils/routeProbes.ts @@ -1,7 +1,8 @@ /** 주기 검사는 재고 중계를 소비하지 않고 직접 경로와 인증 상태만 기록합니다. */ import { cgvTransportFromBindings } from '../services/cgv/relayTransport.js'; import { ROUTE_KEYS, writeRouteChecks, type RouteCheck, type RelayCheck } from './routeHealth.js'; -import { boundedRouteJson, checkDirectRoute } from './directRoutes.js'; +import { boundedRouteJson, checkDirectRoute, DirectRouteError } from './directRoutes.js'; +import { HttpError } from './http.js'; export interface RouteProbeBindings { UPSTREAM_ROUTE_HEALTH?: DurableObjectNamespace; @@ -23,6 +24,27 @@ export interface RouteProbeBindings { DTRYX_ACCESS_CLIENT_SECRET?: string; } +/** 예외 원문 대신 허용된 진단 코드만 저장합니다. */ +function probeFailureReason(error: unknown): string { + if (error instanceof HttpError) + return Number.isInteger(error.status) && error.status >= 100 && error.status <= 599 + ? `http-${error.status}` + : 'http-error'; + if (error instanceof DirectRouteError) + return [ + 'timeout', + 'network-error', + 'invalid-response', + 'response-too-large', + 'cancelled', + 'pinned-relay', + ].includes(error.reason) + ? error.reason + : 'invalid-response'; + if (error instanceof Error && error.name === 'AbortError') return 'cancelled'; + return 'network-error'; +} + async function checkRelay( group: RelayCheck['group'], rawUrl: string | undefined, @@ -62,16 +84,28 @@ async function checkRelay( headers['CF-Access-Client-Id'] = id; headers['CF-Access-Client-Secret'] = secret; } - try { - const result = await boundedRouteJson(url, { method: 'GET', headers }, 3000); - if (!result || typeof result !== 'object' || Array.isArray(result)) - return failure('invalid-response'); - const data = result as Record; - const healthy = group === 'oliveyoung' ? data.state === 'ready' : data.status === 'ok'; - return healthy ? { group, healthy, checkedAt } : failure('unavailable'); - } catch { - return failure('network-error'); - } + const attempt = async (): Promise => { + try { + const result = await boundedRouteJson(url, { method: 'GET', headers }, 3000); + if (!result || typeof result !== 'object' || Array.isArray(result)) + return failure('invalid-response'); + const data = result as Record; + const healthy = group === 'oliveyoung' ? data.state === 'ready' : data.status === 'ok'; + return healthy ? { group, healthy, checkedAt } : failure('unavailable'); + } catch (error) { + return failure(probeFailureReason(error)); + } + }; + const first = await attempt(); + // 일시적인 전송 장애만 건강 GET으로 한 번 확인합니다. 재고 요청은 재시도하지 않습니다. + if ( + first.reason !== 'network-error' && + first.reason !== 'timeout' && + !/^http-5\d{2}$/.test(first.reason || '') + ) + return first; + const confirmed = await attempt(); + return confirmed.healthy ? { ...confirmed, reason: `recovered-${first.reason}` } : confirmed; } export async function refreshRouteChecks(env: RouteProbeBindings): Promise { @@ -82,8 +116,8 @@ export async function refreshRouteChecks(env: RouteProbeBindings): Promise const checkedAt = Date.now(); try { checks.push(await checkDirectRoute(key)); - } catch { - checks.push({ key, direct: false, checkedAt, reason: 'network-error' }); + } catch (error) { + checks.push({ key, direct: false, checkedAt, reason: probeFailureReason(error) }); } }); const cgv = cgvTransportFromBindings(env); diff --git a/tests/api/health-emart-bounds.test.ts b/tests/api/health-emart-bounds.test.ts new file mode 100644 index 00000000..72b0acc1 --- /dev/null +++ b/tests/api/health-emart-bounds.test.ts @@ -0,0 +1,28 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { HEALTH_CHECKS } from '../../src/api/healthCheckDefinitions.js'; +import { handleEmart24CheckInventory } from '../../src/api/emart24Handlers.js'; +import { MCP_SMOKE_SCENARIOS } from '../../scripts/ops/mcp-smoke.js'; +import * as lookup from '../../src/services/emart24/inventoryLookup.js'; +afterEach(() => vi.restoreAllMocks()); +it('Health와 MCP의 이마트24 점포 상한이 실제 핸들러 인자로 동일하게 전달된다', async () => { + const definition = HEALTH_CHECKS.find((check) => check.id === 'emart24.inventory')!; + const smoke = MCP_SMOKE_SCENARIOS.find((check) => check.toolName === 'emart24_check_inventory')!; + const params = new URL(definition.path, 'https://example.invalid').searchParams; + expect(params.get('storeLimit')).toBe(String(smoke.args.storeLimit)); + expect(params.has('limit')).toBe(false); + const upstream = vi + .spyOn(lookup, 'lookupEmart24Inventory') + .mockRejectedValue(new Error('요청 인자 확인용 중단')); + const context = { + env: {}, + req: { query: (key: string) => params.get(key) ?? undefined }, + header: vi.fn(), + json: vi.fn(), + }; + await handleEmart24CheckInventory( + context as unknown as Parameters[0], + ); + expect(upstream).toHaveBeenCalledWith( + expect.objectContaining({ storeLimit: 10, keyword: '커피', storeKeyword: '강남' }), + ); +}); diff --git a/tests/durableObjects/upstreamRouteHealth.test.ts b/tests/durableObjects/upstreamRouteHealth.test.ts index 671a3861..0225e5c0 100644 --- a/tests/durableObjects/upstreamRouteHealth.test.ts +++ b/tests/durableObjects/upstreamRouteHealth.test.ts @@ -125,7 +125,10 @@ describe('durable state edge cases', () => { it('accepts an equal-time probe when no request failure exists', async () => { const s = store(); const checkedAt = Date.now(); - await s.call('/checks', { checks: [{ key: 'cu-stock', direct: false, checkedAt }], relays: [] }); + await s.call('/checks', { + checks: [{ key: 'cu-stock', direct: false, checkedAt }], + relays: [], + }); await s.call('/checks', { checks: [{ key: 'cu-stock', direct: true, checkedAt }], relays: [] }); expect(await (await s.call('/snapshot')).json()).toMatchObject({ routes: { 'cu-stock': { direct: true, checkedAt } }, @@ -151,8 +154,14 @@ describe('durable state edge cases', () => { { checks: [], relays: [{ group: 'dtryx', healthy: 'yes', checkedAt: 1 }] }, { checks: [{ key: 'cu-stock', direct: true, checkedAt: -1 }], relays: [] }, { checks: [{ key: 'cu-stock', direct: true, checkedAt: Date.now() + 90000 }], relays: [] }, - { checks: Array(ROUTE_KEYS.length + 1).fill({ key: 'cu-stock', direct: true, checkedAt: 1 }), relays: [] }, - { checks: [], relays: Array(ROUTE_GROUPS.length + 1).fill({ group: 'dtryx', healthy: true, checkedAt: 1 }) }, + { + checks: Array(ROUTE_KEYS.length + 1).fill({ key: 'cu-stock', direct: true, checkedAt: 1 }), + relays: [], + }, + { + checks: [], + relays: Array(ROUTE_GROUPS.length + 1).fill({ group: 'dtryx', healthy: true, checkedAt: 1 }), + }, ])('rejects invalid values %j', async (data) => { expect((await store().call('/checks', data)).status).toBe(400); }); @@ -198,3 +207,46 @@ it('ignores a delayed failure superseded by a success after its full cooldown', routes: { 'cu-stock': { direct: true } }, }); }); + +it.each([ + 'http-401', + 'http-503', + 'recovered-network-error', + 'recovered-timeout', + 'recovered-http-503', +])('정해진 중계 진단 %s를 스냅샷에 보존한다', async (reason) => { + const s = store(); + const response = await s.call('/checks', { + checks: [], + relays: [ + { + group: 'convenience', + healthy: reason.startsWith('recovered-'), + checkedAt: Date.now(), + reason, + }, + ], + }); + expect(response.status).toBe(204); + const saved = (await (await s.call('/snapshot')).json()) as { + relays: { convenience: { reason: string } }; + }; + expect(saved.relays.convenience.reason).toBe(reason); +}); +it.each([ + 'http-099', + 'http-600', + 'http-503-token', + 'recovered-http-401', + 'recovered-private-value', +])('범위를 벗어나거나 원문이 섞인 진단 %s는 거절한다', async (reason) => { + const s = store(); + expect( + ( + await s.call('/checks', { + checks: [], + relays: [{ group: 'convenience', healthy: false, checkedAt: Date.now(), reason }], + }) + ).status, + ).toBe(400); +}); diff --git a/tests/scripts/mcp-smoke-evidence.test.ts b/tests/scripts/mcp-smoke-evidence.test.ts new file mode 100644 index 00000000..4443089e --- /dev/null +++ b/tests/scripts/mcp-smoke-evidence.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; +import { MCP_SMOKE_SCENARIOS } from '../../scripts/ops/mcp-smoke.js'; + +const scenario = (name: string) => MCP_SMOKE_SCENARIOS.find((item) => item.toolName === name)!; + +// 운영 응답의 구조만 보존하고 점포 식별 정보는 테스트용 값으로 바꿉니다. +const sevenInventory = { + keyword: '핫식스', + storeKeyword: '강남', + product: { + itemCode: '8801056240998', + itemName: '칠성)핫식스더킹퍼플355ml', + salePrice: 2500, + imageUrl: '', + }, + stockAvailable: true, + stockError: null, + note: '실시간 재고 데이터가 포함되어 있습니다.', + inventory: { + totalStoreCount: 100, + inStockStoreCount: 1, + count: 1, + stores: [ + { storeCode: 's1', storeName: '테스트점', stockQuantity: 9, isSoldOut: false, distanceM: 0 }, + ], + }, +}; + +const emartInventory = { + keyword: '커피', + pluCd: '8801115139836', + productCandidates: [], + selectionReason: 'explicit_code', + location: null, + storeFilters: { + storeKeyword: '강남', + appliedStoreKeyword: '강남', + storeLimit: 1, + directBizNos: [], + }, + nearbyStores: { totalCount: 71, stores: [{ storeCode: 's1', storeName: '테스트점' }] }, + inventory: { + goodsInfo: { viewPrice: 2800, goodsNm: '서울)강릉커피라떼250ml', pluCd: '8801115139836' }, + count: 0, + stores: [], + }, +}; + +describe('운영 재고 응답 기반 smoke 회귀', () => { + it('세븐일레븐 재고 대표 검색어는 실제 재고를 확인한 핫식스다', () => { + const entry = scenario('seveneleven_check_inventory'); + expect(entry.args).toMatchObject({ keyword: '핫식스', storeLimit: 1, storeKeyword: '강남' }); + expect(entry.validate(sevenInventory)).toBeNull(); + }); + + it('세븐일레븐 상품은 찾았지만 재고가 미확인이면 실패한다', () => { + const payload = structuredClone(sevenInventory); + payload.stockAvailable = false; + payload.inventory.inStockStoreCount = 0; + payload.inventory.stores[0].stockQuantity = -1; + expect(scenario('seveneleven_check_inventory').validate(payload)).toBeTypeOf('string'); + }); + + it('이마트24는 수량이 없는 후보를 품절로 추정하지 않고 최대 10개 점포를 묶어 조회한다', () => { + const entry = scenario('emart24_check_inventory'); + expect(entry.args).toMatchObject({ storeLimit: 10, storeKeyword: '강남', timeoutMs: 20000 }); + expect(entry.productCode).toBe('pluCd'); + for (const service of ['gs25', 'seveneleven', 'cu', 'oliveyoung']) { + expect(scenario(`${service}_check_inventory`).args.storeLimit).toBe(1); + } + }); + + it('이마트24 묶음 응답에서 반환된 0, 0, 4, 7의 실제 수량을 인정한다', () => { + const payload = { + ...emartInventory, + inventory: { + ...emartInventory.inventory, + count: 4, + stores: [0, 0, 4, 7].map((bizQty, index) => ({ + bizNo: `s${index}`, + storeName: `테스트${index}점`, + bizQty, + address: '', + phone: '', + distanceM: null, + })), + }, + }; + expect(scenario('emart24_check_inventory').validate(payload)).toBeNull(); + const unknown = { + ...payload, + inventory: { + ...payload.inventory, + stores: payload.inventory.stores.map((store) => ({ ...store, bizQty: null })), + }, + }; + expect(scenario('emart24_check_inventory').validate(unknown)).toBeTypeOf('string'); + }); + + it('이마트24 상품 정보와 매장 목록이 있어도 수량 응답이 없으면 실패한다', () => { + expect(scenario('emart24_check_inventory').validate(emartInventory)).toBeTypeOf('string'); + }); +}); diff --git a/tests/scripts/mcp-smoke.test.ts b/tests/scripts/mcp-smoke.test.ts index a19dac85..9311bd68 100644 --- a/tests/scripts/mcp-smoke.test.ts +++ b/tests/scripts/mcp-smoke.test.ts @@ -153,7 +153,11 @@ describe('runMcpSmoke', () => { ]) { expect(callTool).toHaveBeenCalledWith({ name: 'check_inventory', - arguments: expect.objectContaining({ service, [field]: 'p1', storeLimit: 1 }), + arguments: expect.objectContaining({ + service, + [field]: 'p1', + storeLimit: service === 'emart24' ? 10 : 1, + }), }); } expect(callTool.mock.calls.every(([call]) => PUBLIC_TOOL_NAMES.includes(call.name))).toBe(true); diff --git a/tests/utils/routeProbes-no-relay.test.ts b/tests/utils/routeProbes-no-relay.test.ts index e6713113..2f434a65 100644 --- a/tests/utils/routeProbes-no-relay.test.ts +++ b/tests/utils/routeProbes-no-relay.test.ts @@ -36,3 +36,47 @@ it('실제 검사 함수는 직접 원본 요청과 중계 건강 GET만 호출 expect(cgvCalls.every(([, init]) => init.method === 'GET')).toBe(true); expect(storageFetch).toHaveBeenCalledTimes(1); }); + +it('bounds both scheduled health attempts when transport ignores cancellation', async () => { + vi.useFakeTimers(); + const fetcher = vi.fn(async (url: URL) => { + if (url.hostname === 'relay.example') return new Promise(() => {}); + return Response.json({ status: 'ok', state: 'ready' }); + }); + vi.stubGlobal('fetch', fetcher); + const storageFetch = vi.fn(async () => Response.json({})); + const namespace = { + idFromName: () => ({}), + get: () => ({ fetch: storageFetch }), + } as unknown as DurableObjectNamespace; + try { + const refreshed = refreshRouteChecks({ + UPSTREAM_ROUTE_HEALTH: namespace, + OY_RELAY_URL: 'https://relay.example', + OY_RELAY_TOKEN: 'test', + }); + await vi.waitFor(() => { + expect(fetcher.mock.calls.some(([url]) => url.hostname === 'relay.example')).toBe(true); + }); + await vi.advanceTimersByTimeAsync(6000); + await refreshed; + const relayCalls = (fetcher.mock.calls as unknown as [URL, RequestInit][]).filter( + ([url]) => url.hostname === 'relay.example', + ); + expect(relayCalls).toHaveLength(2); + expect( + relayCalls.every(([url, init]) => url.pathname === '/health' && init.method === 'GET'), + ).toBe(true); + expect(relayCalls.every(([, init]) => init.signal?.aborted)).toBe(true); + const init = (storageFetch.mock.calls[0] as unknown as [string, RequestInit])[1]; + const payload = JSON.parse(init.body as string); + expect( + payload.relays.find((row: { group: string }) => row.group === 'oliveyoung'), + ).toMatchObject({ + healthy: false, + reason: 'timeout', + }); + } finally { + vi.useRealTimers(); + } +}); diff --git a/tests/utils/routeProbes-reliability.test.ts b/tests/utils/routeProbes-reliability.test.ts new file mode 100644 index 00000000..cd3464c6 --- /dev/null +++ b/tests/utils/routeProbes-reliability.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, expect, it, vi } from 'vitest'; +import { refreshRouteChecks } from '../../src/utils/routeProbes.js'; +import { HttpError } from '../../src/utils/http.js'; +import type { RelayCheck } from '../../src/utils/routeHealth.js'; +vi.mock('../../src/utils/directRoutes.js', async (importOriginal) => ({ + ...(await importOriginal()), + checkDirectRoute: vi.fn(async (key) => ({ key, direct: true, checkedAt: 100 })), + boundedRouteJson: vi.fn(), +})); +import { boundedRouteJson, DirectRouteError } from '../../src/utils/directRoutes.js'; +const health = vi.mocked(boundedRouteJson); +const storageFetch = vi.fn(async () => Response.json({})); +const namespace = { + idFromName: () => ({}), + get: () => ({ fetch: storageFetch }), +} as unknown as DurableObjectNamespace; +const env = { + UPSTREAM_ROUTE_HEALTH: namespace, + OY_RELAY_URL: 'https://relay.example', + OY_RELAY_TOKEN: 'private-token', +}; +function recordedRelay(): RelayCheck { + const init = (storageFetch.mock.calls[0] as unknown as [string, RequestInit])[1]; + return JSON.parse(init.body as string).relays.find( + (row: RelayCheck) => row.group === 'oliveyoung', + ); +} +beforeEach(() => { + health.mockReset(); + storageFetch.mockClear(); +}); +it.each([ + [new Error('private-url-and-token'), 'network-error'], + [new DirectRouteError('network-error'), 'network-error'], + [new DirectRouteError('timeout'), 'timeout'], + [new HttpError(503, 'private-status', 'private-body'), 'http-503'], +])( + 'confirms a transient failure once and preserves its safe recovery reason: %s', + async (error, reason) => { + health.mockRejectedValueOnce(error).mockResolvedValueOnce({ state: 'ready' }); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: true, reason: `recovered-${reason}` }); + expect(health).toHaveBeenCalledTimes(2); + for (const [url, init, timeout] of health.mock.calls) { + expect(String(url)).toBe('https://relay.example/health'); + expect(init.method).toBe('GET'); + expect(timeout).toBe(3000); + } + }, +); +it.each([ + [new Error('private'), 'network-error'], + [new DirectRouteError('timeout'), 'timeout'], + [new HttpError(503, 'private', 'private'), 'http-503'], +])('records a persistent failure after exactly two attempts: %s', async (error, reason) => { + health.mockRejectedValue(error); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: false, reason }); + expect(health).toHaveBeenCalledTimes(2); +}); +it('records the final confirmation failure instead of hiding it behind the initial transport error', async () => { + health + .mockRejectedValueOnce(new Error('private')) + .mockRejectedValueOnce(new HttpError(403, '', 'private')); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: false, reason: 'http-403' }); + expect(health).toHaveBeenCalledTimes(2); +}); +it.each([ + [new HttpError(401, '', 'private'), 'http-401'], + [new HttpError(403, '', 'private'), 'http-403'], + [new HttpError(429, '', 'private'), 'http-429'], + [new HttpError(99, '', 'private'), 'http-error'], + [new HttpError(600, '', 'private'), 'http-error'], + [new HttpError(503.5, '', 'private'), 'http-error'], + [new DOMException('private', 'AbortError'), 'cancelled'], + [new DirectRouteError('pinned-relay'), 'pinned-relay'], + [new DirectRouteError('invalid-response'), 'invalid-response'], + [new DirectRouteError('response-too-large'), 'response-too-large'], + [new DirectRouteError('cancelled'), 'cancelled'], + [new DirectRouteError('private-arbitrary-reason'), 'invalid-response'], +])( + 'does not retry authentication, semantic or unsafe failure reasons: %s', + async (error, reason) => { + health.mockRejectedValueOnce(error).mockResolvedValue({ state: 'ready' }); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: false, reason }); + expect(health).toHaveBeenCalledTimes(1); + }, +); +it.each([ + [null, 'invalid-response'], + [[], 'invalid-response'], + [{ state: 'warming' }, 'unavailable'], +])('does not retry malformed or unready health responses: %s', async (body, reason) => { + health.mockResolvedValueOnce(body).mockResolvedValue({ state: 'ready' }); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: false, reason }); + expect(health).toHaveBeenCalledTimes(1); +}); +it('does not turn a malformed confirmation into a successful recovery', async () => { + health.mockRejectedValueOnce(new DirectRouteError('timeout')).mockResolvedValueOnce([]); + await refreshRouteChecks(env); + expect(recordedRelay()).toMatchObject({ healthy: false, reason: 'invalid-response' }); + expect(health).toHaveBeenCalledTimes(2); +}); diff --git a/tests/utils/routeProbes.test.ts b/tests/utils/routeProbes.test.ts index a53225d4..b58ff05f 100644 --- a/tests/utils/routeProbes.test.ts +++ b/tests/utils/routeProbes.test.ts @@ -1,7 +1,8 @@ -import { afterEach, expect, it, vi } from 'vitest'; +import { beforeEach, expect, it, vi } from 'vitest'; import { refreshRouteChecks } from '../../src/utils/routeProbes.js'; import { ROUTE_KEYS } from '../../src/utils/routeHealth.js'; -vi.mock('../../src/utils/directRoutes.js', () => ({ +vi.mock('../../src/utils/directRoutes.js', async (importOriginal) => ({ + ...(await importOriginal()), checkDirectRoute: vi.fn(async (key) => ({ key, direct: key !== 'gs25-stock', checkedAt: 100 })), boundedRouteJson: vi.fn(async () => ({ status: 'ok' })), })); @@ -11,7 +12,17 @@ const namespace = { idFromName: vi.fn(() => ({})), get: vi.fn(() => ({ fetch: fetcher })), } as unknown as DurableObjectNamespace; -afterEach(() => vi.clearAllMocks()); +beforeEach(() => { + vi.mocked(checkDirectRoute) + .mockReset() + .mockImplementation(async (key) => ({ + key, + direct: key !== 'gs25-stock', + checkedAt: 100, + })); + vi.mocked(boundedRouteJson).mockReset().mockResolvedValue({ status: 'ok' }); + fetcher.mockClear(); +}); it('skips scheduling work when namespace is absent', async () => { await refreshRouteChecks({}); expect(checkDirectRoute).not.toHaveBeenCalled(); @@ -99,7 +110,7 @@ it('rejects incomplete Access credentials, unsafe URL and invalid health bodies' OY_RELAY_TOKEN: 'token', }); } - vi.mocked(boundedRouteJson).mockRejectedValueOnce(new Error('private')); + vi.mocked(boundedRouteJson).mockRejectedValue(new Error('private')); await refreshRouteChecks({ UPSTREAM_ROUTE_HEALTH: namespace, DTRYX_RELAY_URL: 'https://relay.example',