diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aedec31..3bb6954a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,4 +85,4 @@ jobs: - name: Install dependencies run: npm ci - name: Native helper regression tests - run: npm test -- tests/scripts/hide-readiness.test.ts tests/scripts/macos-ops.test.ts --maxWorkers=1 + run: npm test -- tests/scripts/hide-readiness.test.ts tests/scripts/macos-ops.test.ts tests/relay/recover-launching-guard.test.ts --maxWorkers=1 diff --git a/scripts/relay/recover-abandoned-guard.ts b/scripts/relay/recover-abandoned-guard.ts index 58e94ab0..6da6f06b 100644 --- a/scripts/relay/recover-abandoned-guard.ts +++ b/scripts/relay/recover-abandoned-guard.ts @@ -3,11 +3,13 @@ import { constants } from 'node:fs'; import { open } from 'node:fs/promises'; import { bounded } from './lifecycle.js'; import { processSnapshot, type ProcessIdentity } from './ownership.js'; +import { recoverLaunchingGuard, processReferencesDirectory } from './recover-launching-guard.js'; import { recoverExitedGuard } from './recover-exited-guard.js'; export async function recoverAbandonedGuard( markerPath: string, deadline: number, snapshot = processSnapshot, + references = processReferencesDirectory, ): Promise { const remaining = () => Math.max(0, deadline - Date.now()); try { @@ -24,7 +26,10 @@ export async function recoverAbandonedGuard( stat.mode & 0o077 ) return false; - const marker = JSON.parse(await bounded(handle.readFile('utf8'), remaining())); + const raw = await bounded(handle.readFile('utf8'), remaining()); + const marker = JSON.parse(raw); + if (marker?.state === 'launching' && !marker.browser) + return await recoverLaunchingGuard(markerPath, raw, stat, deadline, snapshot, references); const root = marker?.browser as ProcessIdentity | undefined; if ( !root || diff --git a/scripts/relay/recover-launching-guard.ts b/scripts/relay/recover-launching-guard.ts new file mode 100644 index 00000000..74b0fa21 --- /dev/null +++ b/scripts/relay/recover-launching-guard.ts @@ -0,0 +1,79 @@ +/** 실행 환경 태그까지 두 번 확인하여 시작 도중 종료한 감시자의 기록만 보존합니다. */ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { randomUUID } from 'node:crypto'; +import { constants } from 'node:fs'; +import { open, lstat, rename } from 'node:fs/promises'; +import { dirname, basename } from 'node:path'; +import { bounded } from './lifecycle.js'; +import { processSnapshot } from './ownership.js'; +export function environmentSnapshotArgs(platform: string, uid: number): string[] { + return [platform === 'darwin' ? '-Eww' : 'eww', '-U', String(uid), '-o', 'pid=,command=']; +} +/** 원문 환경은 반환하거나 기록하지 않고 해당 태그·PID의 존재 여부만 반환합니다. */ +export async function processReferencesDirectory( + directory: string, + guardPid: number, + deadline: number, + capture: ( + args: string[], + options: { timeout: number; maxBuffer: number; env: NodeJS.ProcessEnv }, + ) => Promise<{ stdout: string }> = (args, options) => + promisify(execFile)('/bin/ps', args, options), +): Promise { + const nonce = randomUUID(); + const { stdout } = await capture(environmentSnapshotArgs(process.platform, process.getuid!()), { + timeout: Math.max(1, Math.min(2000, deadline - Date.now())), + maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, DAISO_RECOVERY_SNAPSHOT: nonce }, + }); + const containsPid = (pid: number) => new RegExp(`^\\s*${pid}\\s`, 'm').test(stdout); + // 환경 조회 자체가 생략되거나 불완전한 관측이면 회수를 거절합니다. + if (!containsPid(process.pid) || !stdout.includes(`DAISO_RECOVERY_SNAPSHOT=${nonce}`)) + throw new Error('Incomplete process environment snapshot'); + return containsPid(guardPid) || stdout.includes(directory); +} +export async function recoverLaunchingGuard( + markerPath: string, + raw: string, + expected: { dev: number; ino: number }, + deadline: number, + snapshot = processSnapshot, + references = processReferencesDirectory, +): Promise { + const remaining = () => Math.max(0, deadline - Date.now()); + const marker = JSON.parse(raw); + const crashDir = marker.crashDir; + if ( + !Number.isSafeInteger(marker.guardPid) || + marker.guardPid <= 1 || + typeof crashDir !== 'string' || + dirname(crashDir) !== dirname(markerPath) || + !basename(crashDir).startsWith(basename(markerPath) + '.') || + !/\.[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\.crashpad$/i.test(crashDir) + ) + return false; + const handle = await open( + markerPath, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const stat = await bounded(handle.stat(), remaining()); + if (stat.dev !== expected.dev || stat.ino !== expected.ino) return false; + for (let round = 0; round < 2; round++) { + const rows = await bounded(snapshot(), remaining()); + if (rows.some((row) => row.pid === marker.guardPid || row.command?.includes(crashDir))) + return false; + if (await bounded(references(crashDir, marker.guardPid, deadline), remaining())) return false; + } + const content = Buffer.alloc(Buffer.byteLength(raw)); + await bounded(handle.read(content, 0, content.length, 0), remaining()); + if (content.toString() !== raw || (await handle.stat()).size !== content.length) return false; + const current = await bounded(lstat(markerPath), remaining()); + if (current.dev !== stat.dev || current.ino !== stat.ino || !remaining()) return false; + await rename(markerPath, `${markerPath}.exited-${randomUUID()}`); + return true; + } finally { + await handle.close(); + } +} diff --git a/tests/relay/recover-launching-guard.test.ts b/tests/relay/recover-launching-guard.test.ts new file mode 100644 index 00000000..0013d949 --- /dev/null +++ b/tests/relay/recover-launching-guard.test.ts @@ -0,0 +1,243 @@ +import { mkdtemp, writeFile, rm, readdir, stat, rename } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { randomUUID } from 'node:crypto'; +import { afterEach, expect, it, vi } from 'vitest'; +import { recoverAbandonedGuard } from '../../scripts/relay/recover-abandoned-guard.js'; +import { + recoverLaunchingGuard, + processReferencesDirectory, + environmentSnapshotArgs, +} from '../../scripts/relay/recover-launching-guard.js'; +const dirs: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + vi.useRealTimers(); + for (const dir of dirs) await rm(dir, { recursive: true, force: true }); +}); +async function fixture(change: Record = {}) { + const dir = await mkdtemp(join(tmpdir(), 'launch-recovery-')); + dirs.push(dir); + const marker = join(dir, 'owner.json'); + const value = { + guardPid: 12345, + state: 'launching', + crashDir: marker + '.' + randomUUID() + '.crashpad', + ...change, + }; + const raw = JSON.stringify(value); + await writeFile(marker, raw, { mode: 0o600 }); + return { dir, marker, value, raw }; +} +it('시작 실패 마커는 감시자와 환경 태그의 소멸을 두 번 확인한 뒤 보존한다', async () => { + const f = await fixture(); + const snapshot = vi.fn().mockResolvedValue([]); + const references = vi.fn().mockResolvedValue(false); + expect(await recoverAbandonedGuard(f.marker, Date.now() + 5000, snapshot, references)).toBe(true); + expect(snapshot).toHaveBeenCalledTimes(2); + expect(references).toHaveBeenCalledTimes(2); + expect((await readdir(f.dir)).filter((x) => x.startsWith('owner.json.exited-'))).toHaveLength(1); +}); +it('실제 프로세스 및 환경 관측으로도 존재하지 않는 감시자 기록만 회수한다', async () => { + const f = await fixture({ guardPid: 99999999 }); + expect(await recoverAbandonedGuard(f.marker, Date.now() + 10000)).toBe(true); +}); +it.each([ + { guardPid: 0 }, + { guardPid: 'bad' }, + { guardPid: 1.5 }, + { crashDir: undefined }, + { crashDir: '/other/file.crashpad' }, + { crashDir: 'bad' }, +])('불완전한 시작 기록은 거절한다 %j', async (change) => { + const f = await fixture(change); + expect( + await recoverAbandonedGuard( + f.marker, + Date.now() + 5000, + async () => [], + async () => false, + ), + ).toBe(false); +}); +it('다른 마커 이름과 UUID가 없는 경로를 거절한다', async () => { + for (const suffix of ['other.' + randomUUID() + '.crashpad', 'owner.json.not-uuid.crashpad']) { + const f = await fixture(); + await writeFile(f.marker, JSON.stringify({ ...f.value, crashDir: join(f.dir, suffix) })); + expect( + await recoverAbandonedGuard( + f.marker, + Date.now() + 5000, + async () => [], + async () => false, + ), + ).toBe(false); + } +}); +it('브라우저 정체가 있는 기록은 기존의 더 엄격한 회수 절차를 유지한다', async () => { + const f = await fixture({ browser: {} }); + expect( + await recoverAbandonedGuard( + f.marker, + Date.now() + 5000, + async () => [], + async () => false, + ), + ).toBe(false); +}); +it('살아 있는 감시자·명령 인수·환경 태그는 어느 관측에서든 회수를 막는다', async () => { + const row = { pid: 99, ppid: 1, pgid: 99, rss: 0, birth: 'birth', command: 'node' }; + for (const kind of ['guard', 'command', 'tag', 'second-tag', 'unrelated']) { + const f = await fixture(); + const rows = + kind === 'guard' + ? [{ ...row, pid: f.value.guardPid }] + : kind === 'command' + ? [{ ...row, command: f.value.crashDir }] + : [{ ...row, command: kind === 'unrelated' ? undefined : 'node' }]; + const references = vi.fn().mockResolvedValue(kind === 'tag'); + if (kind === 'second-tag') references.mockResolvedValueOnce(false).mockResolvedValue(true); + expect( + await recoverAbandonedGuard(f.marker, Date.now() + 5000, async () => rows, references), + ).toBe(kind === 'unrelated'); + } +}); +it('파일 정체가 바뀌거나 내용이 수정되면 보존 이동을 거절한다', async () => { + for (const kind of ['content', 'size', 'inode']) { + const f = await fixture(); + let count = 0; + const refs = async () => { + if (++count === 2) { + if (kind === 'content') await writeFile(f.marker, f.raw.replace('12345', '54321')); + else if (kind === 'size') await writeFile(f.marker, f.raw + ' '); + else { + await writeFile(f.marker + '.new', f.raw, { mode: 0o600 }); + await rename(f.marker + '.new', f.marker); + } + } + return false; + }; + expect(await recoverAbandonedGuard(f.marker, Date.now() + 5000, async () => [], refs)).toBe( + false, + ); + } +}); +it('열린 파일의 장치·inode가 기대와 다르면 프로세스 검사 전에 거절한다', async () => { + const f = await fixture(); + const s = await stat(f.marker); + for (const expected of [ + { dev: s.dev + 1, ino: s.ino }, + { dev: s.dev, ino: s.ino + 1 }, + ]) + expect( + await recoverLaunchingGuard( + f.marker, + f.raw, + expected, + Date.now() + 5000, + async () => [], + async () => false, + ), + ).toBe(false); +}); +it('불확실한 프로세스 관측과 기한 만료에는 기존 기록을 유지한다', async () => { + const f = await fixture(); + expect( + await recoverAbandonedGuard( + f.marker, + Date.now() + 5000, + async () => { + throw Error('unavailable'); + }, + async () => false, + ), + ).toBe(false); + expect( + await recoverAbandonedGuard( + f.marker, + Date.now() + 5000, + async () => [], + async () => { + throw Error('unavailable'); + }, + ), + ).toBe(false); + vi.useFakeTimers(); + const deadline = Date.now() + 5000; + let count = 0; + expect( + await recoverAbandonedGuard( + f.marker, + deadline, + async () => [], + async () => { + if (++count === 2) vi.spyOn(Date, 'now').mockReturnValue(deadline + 1); + return false; + }, + ), + ).toBe(false); +}); +it('환경 원문을 반환하지 않고 태그·PID와 관측 완전성만 검사한다', async () => { + for (const kind of ['clean', 'tag', 'guard', 'missing-self', 'missing-env']) { + const capture = vi.fn(async (_args: string[], options: { env: NodeJS.ProcessEnv }) => ({ + stdout: + (kind === 'missing-self' ? '' : `${process.pid} node\n`) + + (kind === 'missing-env' + ? '' + : `999 /bin/ps DAISO_RECOVERY_SNAPSHOT=${options.env.DAISO_RECOVERY_SNAPSHOT}\n`) + + (kind === 'guard' + ? '12345 browser\n' + : kind === 'tag' + ? '55 browser BREAKPAD_DUMP_LOCATION=/private/owned.crashpad\n' + : ''), + })); + const result = processReferencesDirectory( + '/private/owned.crashpad', + 12345, + Date.now() - 1, + capture, + ); + if (kind.startsWith('missing')) await expect(result).rejects.toThrow('Incomplete'); + else expect(await result).toBe(kind !== 'clean'); + expect(capture.mock.calls[0][1]).toMatchObject({ timeout: 1, maxBuffer: 16 * 1024 * 1024 }); + } +}); + +it('운영체제별 ps 환경 조회 인수를 명시한다', () => { + expect(environmentSnapshotArgs('darwin', 501)).toEqual([ + '-Eww', + '-U', + '501', + '-o', + 'pid=,command=', + ]); + expect(environmentSnapshotArgs('linux', 1000)).toEqual([ + 'eww', + '-U', + '1000', + '-o', + 'pid=,command=', + ]); +}); +it('명령 인수에는 없는 실제 자식의 환경 태그가 남으면 회수하지 않는다', async () => { + const f = await fixture({ guardPid: 99999999 }); + const child = spawn( + process.execPath, + ['-e', 'process.stdout.write("ready"); setInterval(() => {}, 1000)'], + { + env: { ...process.env, BREAKPAD_DUMP_LOCATION: f.value.crashDir }, + stdio: ['ignore', 'pipe', 'ignore'], + }, + ); + const exited = once(child, 'exit'); + try { + await once(child.stdout!, 'data'); + expect(await recoverAbandonedGuard(f.marker, Date.now() + 10000)).toBe(false); + } finally { + child.kill('SIGTERM'); + await exited; + } + expect(await recoverAbandonedGuard(f.marker, Date.now() + 10000)).toBe(true); +}, 30000);