From 5380cad8dae2233981cc0ee9af6c2df643f0464c Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 18 Nov 2026 15:23:30 +0530 Subject: [PATCH 1/2] ci(bench): add JVM and WASI binding nightly lanes (#858) Generalize the binding nightly job and reporting to cover a JVM lane (14 IDs) and a WASI lane (3 IDs) with their own testbeds, provenance, exact-result validation and notification paths. v3-only, like the existing binding lanes. Co-Authored-By: Claude Sonnet 5.5 --- .github/bench/bindings-report.cjs | 107 ++++++++- .github/bench/bindings-report.test.cjs | 97 +++++++- .github/bench/bindings-workflow.test.cjs | 150 +++++++++++++ .github/bench/bindings.sh | 62 +++++- .github/bench/report.test.cjs | 14 ++ .github/workflows/bench-report.yml | 5 +- .github/workflows/bench.yml | 59 ++++- CHANGELOG.md | 2 + CLAUDE.md | 6 +- README.md | 6 +- ROADMAP.md | 2 +- benchmarks/bindings839/README.md | 8 + benchmarks/bindings839/nightly.py | 100 +++++++++ benchmarks/bindings839/test_nightly.py | 179 +++++++++++++++ docs/BENCHMARKS.md | 9 + docs/BINDING_BENCHMARKS.md | 207 ++++++++++++++---- docs/TEST_COVERAGE.md | 5 +- docs/bench-coverage-v3.md | 6 +- docs/benchmarks/839-bindings/README.md | 6 + docs/benchmarks/839-bindings/dispositions.md | 6 + docs/benchmarks/839-bindings/results.md | 6 + .../2026-11-18-binding-nightlies-design.md | 154 +++++++++++++ 22 files changed, 1128 insertions(+), 68 deletions(-) create mode 100644 .github/bench/bindings-workflow.test.cjs create mode 100644 benchmarks/bindings839/nightly.py create mode 100644 benchmarks/bindings839/test_nightly.py create mode 100644 docs/superpowers/specs/2026-11-18-binding-nightlies-design.md diff --git a/.github/bench/bindings-report.cjs b/.github/bench/bindings-report.cjs index 21c98217b8..bdcb434069 100644 --- a/.github/bench/bindings-report.cjs +++ b/.github/bench/bindings-report.cjs @@ -2,9 +2,13 @@ const fs = require('node:fs'); const { audit } = require('./report.cjs'); function expected(binding) { - if (!['python', 'browser', 'node', 'c'].includes(binding)) throw new Error('unknown binding'); + if (!['python', 'browser', 'node', 'c', 'jvm', 'wasi'].includes(binding)) throw new Error('unknown binding'); + if (binding === 'wasi') return ['repl_startup_empty', 'repl_script/1k', 'repl_query_roundtrip/1k'].map(s => `binding/wasi/${s}`); const suffixes = ['prepared_execute/1k', 'cursor_batch_1/1k', 'cursor_batch_256/1k', 'set_roundtrip/2', 'predicate/1k', 'aggregate/1k']; if (binding === 'browser') suffixes.push('indexeddb_write/1k', 'indexeddb_reopen/1k'); + if (binding === 'jvm') suffixes.push('prepare/point', 'register_predicate/first', 'register_aggregate/first', + 'fact_log_batch_256/1k', 'options_error/negative_cache', 'options_readonly_open/1k', + 'file_transact/1k', 'log_writer_append_finish/1k'); return suffixes.map(s => `binding/${binding}/${s}`); } function validate(binding, metrics) { @@ -16,12 +20,104 @@ function validate(binding, metrics) { } return ids; } +// JSON.parse alone silently overwrites duplicate keys. Parse syntax first, then +// inspect object keys (including escaped spellings) in the valid token stream. +function parseMetrics(text) { + const result = JSON.parse(text); + const tokens = text.match(/"(?:\\.|[^"\\])*"|[{}\[\]:,]|[^\s{}\[\]:,]+/g) || []; + const objects = []; + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + if (token === '{') objects.push(new Set()); + else if (token === '}') objects.pop(); + else if (token.startsWith('"') && tokens[i + 1] === ':') { + const key = JSON.parse(token), keys = objects.at(-1); + if (keys.has(key)) throw new Error(`duplicate JSON key: ${key}`); + keys.add(key); + } + } + return result; +} + +function verifyEvidence(binding, metrics, directory, identity) { + const path = require('node:path'); + const { spawnSync } = require('node:child_process'); + const sha = bytes => require('node:crypto').createHash('sha256').update(bytes).digest('hex'); + const read = name => fs.readFileSync(path.join(directory, name), 'utf8'); + const provenance = parseMetrics(read('binding-provenance.json')); + for (const field of ['coreSha', 'workflowSha', 'bindingSha']) { + if (provenance[field] !== identity[field] || + (!(binding === 'wasi' && field === 'bindingSha') && !/^[a-f0-9]{40}$/.test(provenance[field]))) { + throw new Error(`wrong provenance: ${field}`); + } + } + const artifact = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm'; + if (provenance.artifactPath !== artifact || sha(fs.readFileSync(path.join(directory, artifact))) !== provenance.artifactSha256) { + throw new Error('wrong executed artifact'); + } + const harness = provenance.harnessFiles; + const required = binding === 'jvm' ? ['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'] : ['nightly.py', 'wasi.py', 'validate.py']; + if (!harness || required.some(file => !harness[file])) throw new Error('missing harness hashes'); + // WASI is built and driven by the measured checkout; reporting follows the + // workflow checkout. Resolve the measured source explicitly, not __dirname. + const harnessRoot = binding === 'jvm' ? directory : path.resolve('benchmarks/bindings839'); + for (const [file, digest] of Object.entries(harness)) { + if (path.isAbsolute(file) || file.split('/').includes('..') || + sha(fs.readFileSync(path.join(harnessRoot, file))) !== digest) throw new Error('wrong harness hash'); + } + if (sha(JSON.stringify(Object.fromEntries(Object.entries(harness).sort()))) !== provenance.harnessSha256) { + throw new Error('wrong combined harness hash'); + } + const runtime = provenance.runtime; + if (binding === 'jvm') { + if (runtime?.distribution !== 'temurin' || runtime.vendor !== 'Eclipse Adoptium' || runtime.version !== '17.0.17+10' || + JSON.stringify(runtime.flags) !== JSON.stringify(['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'])) throw new Error('wrong JVM runtime'); + } else if (runtime?.version !== 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)' || + runtime.sha256 !== 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635' || + JSON.stringify(runtime.flags) !== JSON.stringify(['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'])) { + throw new Error('wrong WASI runtime'); + } + const rawPath = path.join(directory, 'benchmark/samples.jsonl'); + // Keep #839's validator authoritative for sample IDs/counts/iterations and + // the complete/check markers, including rejection of smoke streams. + const validation = spawnSync('python3', ['-I', path.join(__dirname, '../../benchmarks/bindings839/validate.py'), binding, rawPath], { encoding: 'utf8' }); + if (validation.status !== 0) throw new Error(`invalid raw samples: ${validation.stderr || validation.error}`); + const records = read('benchmark/samples.jsonl').trim().split('\n').map(parseMetrics); + const environment = records.find(r => r.type === 'environment'); + if (binding === 'jvm') { + if (environment.java_vendor !== runtime.vendor || environment.java_version !== '17.0.17' || + environment.vm_version !== runtime.version || JSON.stringify(environment.jvm_arguments) !== JSON.stringify(runtime.flags) || + environment.warmup_memory !== 100 || environment.warmup_storage !== 5 || + environment.build?.core_revision !== identity.coreSha || environment.build?.binding_revision !== identity.bindingSha) { + throw new Error('wrong JVM sample environment'); + } + } else if (environment.core_revision !== identity.coreSha || environment.wasm_sha256 !== provenance.artifactSha256 || + environment.runtime_version !== runtime.version || environment.runtime_sha256 !== runtime.sha256 || + JSON.stringify(environment.runtime_flags) !== JSON.stringify(runtime.flags) || environment.runtime_cache !== false || + environment.storage !== 'in-memory; no preopened directories' || environment.warmups !== 3 || environment.samples !== 10) { + throw new Error('wrong WASI sample environment'); + } + for (const id of expected(binding)) { + const means = records.filter(r => r.type === 'sample' && r.id === id).map(r => r.total_ns / r.iterations).sort((a, b) => a - b); + const median = (means[4] + means[5]) / 2; + if (Math.abs(metrics[id].latency.value - median) > Math.abs(median) * 1e-12) throw new Error(`wrong raw-sample median: ${id}`); + } + return provenance; +} + function main(command) { const read = p => fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : ''; const binding = process.env.BINDING; const ids = expected(binding); if (command === 'validate') { - validate(binding, JSON.parse(read('binding/binding-results.json'))); + const metrics = parseMetrics(read('binding/binding-results.json')); + validate(binding, metrics); + if (['jvm', 'wasi'].includes(binding)) verifyEvidence(binding, metrics, 'binding', { + coreSha: process.env.BENCH_SHA, workflowSha: process.env.WORKFLOW_SHA, + bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(), + }); const executed = read('binding-execution.txt').split('\n').filter(s => s.startsWith('ok: ')).map(s => s.slice(4)); if (executed.length !== ids.length || new Set(executed).size !== ids.length || executed.some(id => !ids.includes(id))) throw new Error('executed IDs do not match shortlist'); return; @@ -32,9 +128,10 @@ function main(command) { const execution = read('binding-execution.txt').replace(/^ok: (.+)$/gm, 'Benchmarking $1: Analyzing'); const result = audit(ids, execution, read('binding-bencher.txt'), process.env.BENCHER_OUTCOME); const report = { - binding, coreSha: process.env.BENCH_SHA, bindingSha: read('binding-sha.txt').trim(), + binding, coreSha: process.env.BENCH_SHA, bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(), workflowSha: process.env.WORKFLOW_SHA, ref: process.env.BENCH_REF, historyBranch: process.env.BINDING_HISTORY, - testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result, + suite: `binding_${binding}`, testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result, + provenance: read('binding/binding-provenance.json') ? parseMetrics(read('binding/binding-provenance.json')) : null, }; fs.writeFileSync('binding-report.json', JSON.stringify(report, null, 2) + '\n'); const summary = `### Binding benchmark: ${binding}\n\nCore: ${report.coreSha}; binding: ${report.bindingSha}; workflow: ${report.workflowSha}\n\nHistory branch: ${report.historyBranch}; testbed: ${report.testbed}; executed ${result.executed.length}/${ids.length}; uploaded ${result.uploaded.length}/${ids.length}; alerts ${result.alerts.length}; upload ${result.uploadStatus}\n\n${result.errors.join('\n')}\n`; @@ -42,7 +139,7 @@ function main(command) { if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); if (result.errors.length) process.exitCode = 1; } -module.exports = { expected, validate }; +module.exports = { expected, validate, parseMetrics, verifyEvidence }; if (require.main === module) { try { main(process.argv[2]); } catch (e) { console.error(e.message); process.exitCode = 1; } } diff --git a/.github/bench/bindings-report.test.cjs b/.github/bench/bindings-report.test.cjs index cdf921d40a..063f002244 100644 --- a/.github/bench/bindings-report.test.cjs +++ b/.github/bench/bindings-report.test.cjs @@ -3,8 +3,8 @@ const assert = require('node:assert/strict'); const { expected, validate } = require('./bindings-report.cjs'); const data = binding => Object.fromEntries(expected(binding).map(id => [id, { latency: { value: 100 } }])); test('exact dedicated routes and finite positive latency only', () => { - for (const binding of ['python', 'browser', 'node', 'c']) { - assert.equal(validate(binding, data(binding)).length, binding === 'browser' ? 8 : 6); + for (const binding of ['python', 'browser', 'node', 'c', 'jvm', 'wasi']) { + assert.equal(validate(binding, data(binding)).length, ({ browser: 8, jvm: 14, wasi: 3 }[binding] || 6)); const missing = data(binding); delete missing[expected(binding)[0]]; assert.throws(() => validate(binding, missing), /IDs/); const extra = data(binding); extra['native/query'] = { latency: { value: 1 } }; @@ -44,3 +44,96 @@ test('CLI verifies execution and records provenance even when upload is partial' assert.ok(report.errors.length); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); + +// New lanes must keep distinct inventories, not alias the six P16 cases. +test('JVM lifecycle/storage and WASI REPL shortlists are exact', () => { + assert.deepEqual(expected('wasi'), [ + 'binding/wasi/repl_startup_empty', 'binding/wasi/repl_script/1k', + 'binding/wasi/repl_query_roundtrip/1k', + ]); + assert.deepEqual(expected('jvm').slice(6), [ + 'prepare/point', 'register_predicate/first', 'register_aggregate/first', + 'fact_log_batch_256/1k', 'options_error/negative_cache', + 'options_readonly_open/1k', 'file_transact/1k', 'log_writer_append_finish/1k', + ].map(id => `binding/jvm/${id}`)); +}); + +const { verifyEvidence, parseMetrics } = require('./bindings-report.cjs'); +const crypto = require('node:crypto'); +const sha = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +test('duplicate JSON metric keys fail instead of being overwritten', () => { + const id = expected('jvm')[0]; + assert.throws(() => parseMetrics(`{"${id}":{"latency":{"value":1}},"${id}":{"latency":{"value":2}}}`), /duplicate/); + assert.throws(() => parseMetrics('{"a":{"value":1,"value":2}}'), /duplicate/); + assert.deepEqual(parseMetrics('{"a":{"value":1},"b":{"value":2}}'), {a: {value: 1}, b: {value: 2}}); +}); + +for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, provenance and artifact bytes before upload`, () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-evidence-')); + try { + fs.mkdirSync(path.join(dir, 'benchmark')); + const coreSha = 'a'.repeat(40), workflowSha = 'b'.repeat(40), bindingSha = binding === 'jvm' ? 'c'.repeat(40) : null; + const ids = expected(binding); + const environment = binding === 'jvm' ? { + java_vendor: 'Eclipse Adoptium', java_version: '17.0.17', vm_version: '17.0.17+10', + jvm_arguments: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'], warmup_memory: 100, warmup_storage: 5, + build: {core_revision: coreSha, binding_revision: bindingSha}, + } : { + core_revision: coreSha, wasm_sha256: sha('actual artifact'), + runtime_version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', + runtime_sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', + runtime_flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'], + runtime_cache: false, storage: 'in-memory; no preopened directories', warmups: 3, samples: 10, + }; + const records = [{ type: 'environment', mode: 'benchmark', ...environment }]; + const metrics = {}; + for (const id of ids) { + const iterations = id.endsWith('repl_query_roundtrip/1k') ? 20 : binding === 'wasi' || /(?:file_transact|log_writer_append_finish)\/1k$/.test(id) ? 1 : 20; + for (let sample = 0; sample < 10; sample++) records.push({type: 'sample', id, sample, iterations, total_ns: (sample + 1) * iterations, ns_per_op: sample + 1}); + records.push({type: 'checked', id}); + metrics[id] = { latency: { value: 5.5 } }; + } + records.push({type: 'complete', mode: 'benchmark', count: ids.length, ids}); + const raw = path.join(dir, 'benchmark/samples.jsonl'); + fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n')); + const artifactPath = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm'; + fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact'); + const harnessFiles = binding === 'jvm' + ? Object.fromEntries(['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'].map(p => [p, sha(p)])) + : Object.fromEntries(['nightly.py', 'wasi.py', 'validate.py'].map(p => [p, sha(fs.readFileSync(path.join(__dirname, '../../benchmarks/bindings839', p)))])); + if (binding === 'jvm') for (const file of Object.keys(harnessFiles)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true}); + fs.writeFileSync(path.join(dir, file), file); + } + const provenance = { coreSha, workflowSha, bindingSha, artifactPath, artifactSha256: sha('actual artifact'), harnessFiles, + harnessSha256: sha(JSON.stringify(Object.fromEntries(Object.entries(harnessFiles).sort()))), + runtime: binding === 'jvm' + ? { distribution: 'temurin', vendor: 'Eclipse Adoptium', version: '17.0.17+10', flags: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'] } + : { version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'] }, + }; + const file = path.join(dir, 'binding-provenance.json'); + fs.writeFileSync(file, JSON.stringify(provenance)); + const identity = { coreSha, workflowSha, bindingSha }; + assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha); + metrics[ids[0]].latency.value = 6; + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /median/); + metrics[ids[0]].latency.value = 5.5; + for (const field of ['coreSha', 'workflowSha', 'bindingSha']) { + if (field === 'bindingSha' && binding === 'wasi') continue; + assert.throws(() => verifyEvidence(binding, metrics, dir, {...identity, [field]: 'd'.repeat(40)}), /provenance/); + } + fs.writeFileSync(file, JSON.stringify({...provenance, runtime: {...provenance.runtime, flags: []}})); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /runtime/); + fs.writeFileSync(file, JSON.stringify(provenance)); + fs.writeFileSync(path.join(dir, artifactPath), 'different artifact'); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /artifact/); + fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact'); + const wrongEnvironment = {...records[0], ...(binding === 'jvm' ? {warmup_memory: 3} : {runtime_cache: true})}; + fs.writeFileSync(raw, [wrongEnvironment, ...records.slice(1)].map(r => JSON.stringify(r)).join('\n')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /environment/); + fs.writeFileSync(raw, records.slice(0, -1).map(r => JSON.stringify(r)).join('\n')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /samples/); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/.github/bench/bindings-workflow.test.cjs b/.github/bench/bindings-workflow.test.cjs new file mode 100644 index 0000000000..e3d1575af4 --- /dev/null +++ b/.github/bench/bindings-workflow.test.cjs @@ -0,0 +1,150 @@ +// Run workflow shell/notification bodies locally; no network or GitHub writes. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/bench.yml'), 'utf8'); +const bindingJob = workflow.split('\n bindings:\n')[1]; +function body(name, kind = 'run') { + const step = bindingJob.split(` - name: ${name}\n`)[1]; + assert.ok(step, `missing step ${name}`); + return step.split(` ${kind}: |\n`)[1]?.split(/\n - /)[0].replace(/^ /gm, '').trimEnd() + || step.split(` ${kind}: |\n`)[1].split(/\n - /)[0].replace(/^ /gm, '').trimEnd(); +} +function shell(name, env, setup = () => {}, prefix = '') { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-workflow-')); + try { + setup(cwd); + const result = spawnSync('bash', ['-e', '-c', prefix + body(name).replace(/\$\{\{ secrets\.BENCHER_API_TOKEN \}\}/g, 'test-token')], { + cwd, encoding: 'utf8', env: {...process.env, GITHUB_OUTPUT: path.join(cwd, 'out'), GITHUB_ENV: path.join(cwd, 'env'), GITHUB_STEP_SUMMARY: path.join(cwd, 'summary'), ...env}, + }); + const read = file => fs.existsSync(path.join(cwd, file)) ? fs.readFileSync(path.join(cwd, file), 'utf8') : ''; + return {...result, output: read('out'), vars: read('env'), summary: read('summary')}; + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } +} +test('new rows use the actual binding suite job names', () => { + assert.match(bindingJob, /name: bench \/ binding_\$\{\{ matrix.binding \}\}/); + assert.match(bindingJob, /binding: jvm, repo: minigraf-java/); + assert.match(bindingJob, /binding: wasi/); + assert.match(workflow, /jvm-ref:/); +}); +test('main skips with a summary; a missing v3 harness fails', () => { + const main = shell('Check v3 binding suite availability', {BENCH_REF: 'main', BINDING: 'wasi'}); + assert.equal(main.status, 0, main.stderr); + assert.match(main.summary, /Skipping/); + assert.doesNotMatch(main.output, /enabled=true/); + const missing = shell('Check v3 binding suite availability', {BENCH_REF: 'v3', BINDING: 'wasi'}); + assert.notEqual(missing.status, 0); +}); +for (const binding of ['jvm', 'wasi']) test(`${binding} uploads only to its own testbed without inherited history`, () => { + for (const history of ['v3', 'binding-experiment/experiment/v3']) { + const result = shell('Upload foreign runtime metrics', {BINDING: binding, BINDING_HISTORY: history, BENCH_SHA: 'a'.repeat(40), BENCH_THRESHOLD: '0.999'}, undefined, + 'bencher() { printf "%s\\0" "$@"; }\n'); + assert.equal(result.status, 0, result.stderr); + const args = result.stdout.split('\0'); + assert.equal(args[args.indexOf('--testbed') + 1], `binding-${binding}-ubuntu24-v1`); + assert.equal(args[args.indexOf('--branch') + 1], history); + assert.equal(args[args.indexOf('--threshold-max-sample-size') + 1], '30'); + assert.equal(args[args.indexOf('--threshold-upper-boundary') + 1], '0.999'); + assert.ok(!args.some(a => a.startsWith('--start-point'))); + } +}); + +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +for (const binding of ['jvm', 'wasi']) test(`${binding} repeated exact IDs, immediate upload failure and failed lookup`, async () => { + const suite = `binding_${binding}`, id = `binding/${binding}/example`; + const alert = name => `- ${name} (Latency): https://bencher.dev/console/projects/minigraf/alerts/123`; + const original = {...process.env}; + try { + Object.assign(process.env, {SUITE: suite, BENCH_REF: 'v3', BENCH_SHA: 'a'.repeat(40)}); + for (const scenario of ['repeat', 'different-id', 'upload-failure', 'lookup-failure', 'existing-issue', 'experiment-prior']) { + const calls = [], warnings = []; + const github = {rest: {actions: { + listWorkflowRuns: async () => ({data: {workflow_runs: [{id: 1, display_title: 'Nightly Benchmarks (main)'}, {id: 2, display_title: 'Nightly Benchmarks (v3)', html_url: 'prior'}]}}), + listJobsForWorkflowRun: async args => { assert.equal(args.run_id, 2); return {data: {jobs: [{id: 3, name: `bench / ${suite}`}, {id: 4, name: 'bench / binding_python'}]}}; }, + downloadJobLogsForWorkflowRun: async args => { + assert.equal(args.job_id, 3); + if (scenario === 'lookup-failure') throw new Error('unavailable logs'); + return {data: `History branch: ${scenario === 'experiment-prior' ? 'binding-experiment/v3/java-only' : 'v3'}; testbed: binding-${binding}-ubuntu24-v1;\n` + alert(scenario === 'different-id' ? `${id}/other` : id)}; + }, + }, issues: { + listForRepo: async () => ({data: scenario === 'existing-issue' ? [{number: 8, title: `Benchmark regression: ${suite} (v3)`}] : []}), + create: async args => { calls.push(args); return {data: {number: 9}}; }, + createComment: async args => { calls.push(args); }, + }}}; + const summary = {addHeading() {return this;}, addRaw() {return this;}, addList() {return this;}, addBreak() {return this;}, async write() {}}; + const mockRequire = name => { assert.equal(name, 'fs'); return {readFileSync: () => scenario === 'upload-failure' ? 'Unauthorized' : alert(id)}; }; + await new AsyncFunction('require', 'github', 'context', 'core', 'console', body('Report regression', 'script'))( + mockRequire, github, {repo: {owner: 'test', repo: 'test'}, runId: 10, serverUrl: 'https://example.com'}, {warning: text => warnings.push(text), summary}, {log() {}}); + assert.equal(calls.length, ['repeat', 'upload-failure', 'existing-issue'].includes(scenario) ? 1 : 0); + if (scenario === 'lookup-failure') assert.ok(warnings.some(w => w.includes('Could not read'))); + if (scenario === 'upload-failure') assert.match(calls[0].body, /API or upload error/); + if (scenario === 'repeat') assert.match(calls[0].body, /two runs in a row/); + if (scenario === 'existing-issue') assert.equal(calls[0].issue_number, 8); + } + } finally { + for (const key of ['SUITE', 'BENCH_REF', 'BENCH_SHA']) { + if (original[key] === undefined) delete process.env[key]; else process.env[key] = original[key]; + } + } +}); + +test('experimental binding and core refs are isolated, including WASI without a repo', () => { + for (const [binding, ref, bindingRef, history] of [ + ['jvm', 'v3', 'v3', 'v3'], + ['jvm', 'v3', 'java-only', 'binding-experiment/v3/java-only'], + ['jvm', 'candidate', 'v3', 'binding-experiment/candidate/v3'], + ['wasi', 'candidate', 'v3', 'binding-experiment/candidate/v3'], + ]) { + const result = shell('Pin binding source', {BINDING: binding, BENCH_REF: ref, BINDING_REF: bindingRef}, undefined, + 'git() { printf "%s\\n" binding-sha; }\n'); + assert.equal(result.status, 0, result.stderr); + assert.match(result.vars, new RegExp(`BINDING_HISTORY=${history}\\n`)); + } +}); + +test('JVM manual builder owns the patch and wrong Cargo resolutions fail', () => { + const helper = path.join(__dirname, 'bindings.sh'); + for (const resolution of ['correct', 'release-pin', 'other-checkout', 'duplicate']) { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-build-')); + try { + fs.mkdirSync(path.join(cwd, 'binding/bench/manual'), {recursive: true}); + fs.mkdirSync(path.join(cwd, 'bin')); + fs.writeFileSync(path.join(cwd, 'binding/bench/nightly.sh'), '# contract placeholder\n'); + fs.writeFileSync(path.join(cwd, 'binding/bench/manual/build.sh'), `set -euo pipefail + test ! -e .cargo/config.toml + test "$CARGO_BUILD_JOBS" = 4 + test "$CARGO_TARGET_DIR" = "$PWD/target" + test "$CORE_REVISION" = "$BENCH_SHA" + mkdir .cargo + echo 'owned by manual build' > .cargo/config.toml + `); + fs.writeFileSync(path.join(cwd, 'bin/git'), '#!/bin/sh\nprintf "%s\\n" "$BENCH_SHA"\n', {mode: 0o755}); + const pkg = {name: 'minigraf', source: resolution === 'release-pin' ? 'git+release' : null, + manifest_path: path.join(resolution === 'other-checkout' ? '/wrong' : cwd, 'Cargo.toml')}; + const metadata = {packages: resolution === 'duplicate' ? [pkg, pkg] : [pkg]}; + fs.writeFileSync(path.join(cwd, 'metadata.json'), JSON.stringify(metadata)); + fs.writeFileSync(path.join(cwd, 'bin/cargo'), '#!/bin/sh\ncat ../metadata.json\n', {mode: 0o755}); + const result = spawnSync('bash', [helper, 'build', 'jvm'], {cwd, encoding: 'utf8', + env: {...process.env, PATH: `${cwd}/bin:${process.env.PATH}`, BENCH_SHA: 'a'.repeat(40)}}); + assert.equal(result.status === 0, resolution === 'correct', result.stderr); + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } + } +}); + +test('JVM run refuses stale output and smoke that emits metrics', () => { + for (const scenario of ['stale', 'smoke-metrics', 'failed-exit']) { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-run-')); + try { + fs.mkdirSync(path.join(cwd, 'binding/bench'), {recursive: true}); + fs.writeFileSync(path.join(cwd, 'binding/bench/nightly.sh'), scenario === 'failed-exit' ? 'exit 7\n' : 'echo "{}" > binding-results.json\n'); + if (scenario === 'stale') fs.writeFileSync(path.join(cwd, 'binding/binding-results.json'), '{}'); + const args = [path.join(__dirname, 'bindings.sh'), 'run', 'jvm']; + if (scenario === 'smoke-metrics') args.push('--test'); + const result = spawnSync('bash', args, {cwd, encoding: 'utf8'}); + assert.notEqual(result.status, 0, scenario); + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } + } +}); diff --git a/.github/bench/bindings.sh b/.github/bench/bindings.sh index e0170e257b..fc9c0245e2 100644 --- a/.github/bench/bindings.sh +++ b/.github/bench/bindings.sh @@ -5,7 +5,67 @@ mode=$1 binding=$2 core_dir=$PWD export CARGO_TARGET_DIR="$PWD/binding/target/p16" -export CARGO_BUILD_JOBS=2 +export CARGO_BUILD_JOBS=4 +[[ "$mode" = build || "$mode" = run ]] +if [[ "$binding" = wasi ]]; then + if [[ "$mode" = build ]]; then + test "$(git rev-parse HEAD)" = "$BENCH_SHA" + test -f benchmarks/bindings839/nightly.py + test -f benchmarks/bindings839/wasi.py + test -f benchmarks/bindings839/validate.py + # No binding repo. Build the CLI from this once-resolved core checkout. + export CARGO_TARGET_DIR="$core_dir/target" + cargo build --locked --release --target wasm32-wasip1 --bin minigraf + mkdir binding + cp target/wasm32-wasip1/release/minigraf.wasm binding/minigraf.wasm + curl -sSfL 'https://github.com/bytecodealliance/wasmtime/releases/download/v49.0.2/wasmtime-v49.0.2-x86_64-linux.tar.xz' \ + -o binding/wasmtime.tar.xz + tar -xf binding/wasmtime.tar.xz -C binding + echo 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635 binding/wasmtime-v49.0.2-x86_64-linux/wasmtime' | sha256sum --check --strict + else + shift 2 + python3 -I benchmarks/bindings839/nightly.py \ + --runtime binding/wasmtime-v49.0.2-x86_64-linux/wasmtime \ + --wasm binding/minigraf.wasm --core-dir "$core_dir" --output binding "$@" + fi + exit +fi +if [[ "$binding" = jvm ]]; then + # manual/build.sh and Gradle use target/release explicitly. It alone owns + # the patch, so do not create .cargo/config.toml before calling it. + export CARGO_TARGET_DIR="$core_dir/binding/target" + cd binding + test -f bench/nightly.sh + if [[ "$mode" = build ]]; then + test "$(git -C "$core_dir" rev-parse HEAD)" = "$BENCH_SHA" + CORE_REVISION="$BENCH_SHA" BINDING_REVISION="$(git rev-parse HEAD)" \ + bash bench/manual/build.sh "$core_dir" + cargo metadata --format-version 1 --locked > target-metadata.json + python3 -I - "$core_dir" <<'PY_METADATA' +import json, pathlib, sys +packages = json.load(open('target-metadata.json'))['packages'] +core = [p for p in packages if p['name'] == 'minigraf'] +assert len(core) == 1 and core[0]['source'] is None +assert pathlib.Path(core[0]['manifest_path']).parent.resolve() == pathlib.Path(sys.argv[1]).resolve() +PY_METADATA + else + shift 2 + # Contract in docs/BINDING_BENCHMARKS.md; failures must propagate unchanged. + test ! -e binding-results.json + test ! -e binding-provenance.json + if [[ "${1:-}" = --test ]]; then + test ! -e smoke + else + test ! -e benchmark + fi + bash bench/nightly.sh "$@" + if [[ "${1:-}" = --test ]]; then + test ! -e binding-results.json + test ! -e binding-provenance.json + fi + fi + exit +fi cd binding if [[ "$mode" = build ]]; then mkdir -p .cargo diff --git a/.github/bench/report.test.cjs b/.github/bench/report.test.cjs index bef897b2ee..290aadf08f 100644 --- a/.github/bench/report.test.cjs +++ b/.github/bench/report.test.cjs @@ -96,3 +96,17 @@ test('CLI retains provenance, IDs and summary on success and early failure', () assert.equal(JSON.parse(fs.readFileSync(path.join(cwd, 'bench-report.json'))).sha, env.BENCH_SHA); } finally { fs.rmSync(cwd, { recursive: true, force: true }); } }); + +for (const runtime of ['jvm', 'wasi']) test(`${runtime} requires selected = executed = uploaded`, () => { + const { expected } = require('./bindings-report.cjs'); + const selected = expected(runtime); + const execution = selected.map(id => `Benchmarking ${id}: Analyzing`).join('\n'); + const uploaded = selected.map(id => `- ${id} (Latency): https://bencher.dev/console/projects/minigraf/perf?x=1`).join('\n'); + assert.deepEqual(audit(selected, execution, uploaded, 'success').errors, []); + for (const bad of ['', execution + '\n' + execution, execution.split('\n').slice(1).join('\n')]) { + assert.ok(audit(selected, bad, uploaded, 'success').errors.length); + } + for (const bad of ['', uploaded + '\n' + uploaded, uploaded.split('\n').slice(1).join('\n')]) { + assert.ok(audit(selected, execution, bad, 'success').errors.length); + } +}); diff --git a/.github/workflows/bench-report.yml b/.github/workflows/bench-report.yml index ccdd53aa63..7e59a36535 100644 --- a/.github/workflows/bench-report.yml +++ b/.github/workflows/bench-report.yml @@ -3,10 +3,10 @@ name: Benchmark reporting checks on: push: branches: [main, 'v[0-9]*'] - paths: ['.github/bench/**', '.github/workflows/bench*.yml'] + paths: ['.github/bench/**', '.github/workflows/bench*.yml', 'benchmarks/bindings839/**'] pull_request: branches: [main, 'v[0-9]*'] - paths: ['.github/bench/**', '.github/workflows/bench*.yml'] + paths: ['.github/bench/**', '.github/workflows/bench*.yml', 'benchmarks/bindings839/**'] permissions: contents: read @@ -20,3 +20,4 @@ jobs: with: node-version: '24' - run: node --test .github/bench/*.test.cjs + - run: python3 -I -m unittest discover -s benchmarks/bindings839 -p 'test_*.py' diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 558a63d6c9..a0cd4568dd 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -21,16 +21,22 @@ on: default: main binding-ref: - description: Binding ref for all P16 repos (phase B may select the PR branch) + description: Common binding ref (use jvm-ref for a JVM-only experiment) required: false default: v3 + jvm-ref: + description: Override only minigraf-java (other bindings keep binding-ref) + required: false + default: '' + permissions: contents: read issues: write actions: read # read the previous run's job logs to confirm an alert repeats env: + CARGO_BUILD_JOBS: "4" # One threshold per branch/testbed/measure; never race per-suite models. # 0.999 retains the noisy file/concurrency setting and existing testbed history. BENCH_THRESHOLD: "0.999" @@ -568,9 +574,11 @@ jobs: - { binding: browser, repo: minigraf-wasm } - { binding: node, repo: minigraf-node } - { binding: c, repo: minigraf-c } + - { binding: jvm, repo: minigraf-java } + - { binding: wasi } env: BINDING: ${{ matrix.binding }} - BINDING_REF: ${{ inputs.binding-ref || 'v3' }} + BINDING_REF: ${{ matrix.binding == 'wasi' && 'v3' || (matrix.binding == 'jvm' && inputs.jvm-ref) || inputs.binding-ref || 'v3' }} BENCH_SHA: ${{ needs.compile.outputs.sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} steps: @@ -580,10 +588,17 @@ jobs: - name: Check v3 binding suite availability id: source run: | - if [ -f benches/minigraf_bench/prepared.rs ]; then + # A missing harness on v3 is an error, never a successful empty run. + if [ "$BENCH_REF" = v3 ] || [ -f src/query/engine/mod.rs ]; then + test -f benches/minigraf_bench/prepared.rs + if [ "$BINDING" = wasi ]; then + test -f benchmarks/bindings839/nightly.py + test -f benchmarks/bindings839/wasi.py + test -f benchmarks/bindings839/validate.py + fi echo 'enabled=true' >> "$GITHUB_OUTPUT" else - echo 'Skipping v3-only binding jobs for this measured checkout.' >> "$GITHUB_STEP_SUMMARY" + echo 'Skipping v3-only binding jobs for this measured main/v2 checkout.' >> "$GITHUB_STEP_SUMMARY" fi - uses: actions/checkout@v7 if: steps.source.outputs.enabled == 'true' @@ -591,29 +606,33 @@ jobs: ref: ${{ github.workflow_sha }} path: .bench-workflow sparse-checkout-cone-mode: false - sparse-checkout: /.github/bench/ + sparse-checkout: | + /.github/bench/ + /benchmarks/bindings839/ persist-credentials: false - uses: actions/checkout@v7 - if: steps.source.outputs.enabled == 'true' + if: steps.source.outputs.enabled == 'true' && matrix.binding != 'wasi' with: repository: project-minigraf/${{ matrix.repo }} - ref: ${{ inputs.binding-ref || 'v3' }} + ref: ${{ env.BINDING_REF }} path: binding persist-credentials: false - name: Pin binding source if: steps.source.outputs.enabled == 'true' run: | set -o pipefail - git -C binding rev-parse HEAD | tee binding-sha.txt + if [ "$BINDING" != wasi ]; then + git -C binding rev-parse HEAD | tee binding-sha.txt + fi history="$BENCH_REF" - if [ "$BINDING_REF" != v3 ]; then + if [ "$BINDING_REF" != v3 ] || [ "$BENCH_REF" != v3 ]; then history="binding-experiment/$BENCH_REF/$BINDING_REF" fi echo "BINDING_HISTORY=$history" >> "$GITHUB_ENV" - uses: dtolnay/rust-toolchain@stable if: steps.source.outputs.enabled == 'true' with: - targets: wasm32-unknown-unknown + targets: wasm32-unknown-unknown, wasm32-wasip1 - uses: actions/setup-python@v6 if: steps.source.outputs.enabled == 'true' with: @@ -622,6 +641,11 @@ jobs: if: steps.source.outputs.enabled == 'true' with: node-version: '20' + - uses: actions/setup-java@v5 + if: steps.source.outputs.enabled == 'true' && matrix.binding == 'jvm' + with: + distribution: temurin + java-version: '17.0.17+10' - name: Build binding with measured core if: steps.source.outputs.enabled == 'true' run: bash .bench-workflow/.github/bench/bindings.sh build "$BINDING" @@ -652,7 +676,7 @@ jobs: --threshold-max-sample-size 30 --threshold-upper-boundary "$BENCH_THRESHOLD" \ --err --adapter json --file binding/binding-results.json 2>&1 | tee binding-bencher.txt - name: Report regression - if: steps.bencher.outcome == 'failure' && env.BINDING_REF == 'v3' + if: steps.bencher.outcome == 'failure' && env.BINDING_REF == 'v3' && env.BENCH_REF == 'v3' uses: actions/github-script@v9 env: SUITE: binding_${{ matrix.binding }} @@ -714,7 +738,13 @@ jobs: const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({ owner, repo, job_id: prevJob.id, }); - previous = parseAlerts(typeof log === 'string' ? log : Buffer.from(log).toString('utf8')); + const text = typeof log === 'string' ? log : Buffer.from(log).toString('utf8'); + // A JVM-only experiment can share this run title. Its + // alerts must never count as a production repetition. + const testbed = `binding-${suite.slice('binding_'.length)}-ubuntu24-v1`; + if (text.includes(`History branch: ${ref}; testbed: ${testbed};`)) { + previous = parseAlerts(text); + } } } } catch (err) { @@ -808,3 +838,8 @@ jobs: binding-report.json binding/binding-results.json binding/target-metadata.json + binding/binding-provenance.json + binding/benchmark/ + binding/smoke/ + binding/minigraf.wasm + binding/bench/manual/build/evidence/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b778bd5b4..5b569d3ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **JVM/WASI nightly workflow support** (#858, core portion): 14 JVM and 3 WASI routes, pinned runtimes, measured-core/artifact provenance, strict raw-sample reporting and independent histories. Java adapter integration, default-branch rollout and live baseline/notification evidence remain pending; mobile lanes remain excluded pending qualification. No runtime, file-format or Rust test inventory change. See [the adapter contract](docs/BINDING_BENCHMARKS.md). + - **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import admits an empty blob, a nonempty v8 image with a selectable meta and complete, verified contents (an empty v8 image included), or a supported v7 image that migrates completely, and rejects anything else (such as a torn or zero-filled initial image) before replacing anything; it validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. Exports after interrupted initial creation fill an absent second meta slot with zeros and still propagate failed reads of existing pages. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). diff --git a/CLAUDE.md b/CLAUDE.md index ff484769bc..ced0844d9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,8 +15,10 @@ verifies numeric comparisons for all 427 native and 26 binding IDs, at least thr independent v3 runs per ID, preserved history after main advances, actual scheduled routes and isolated live notification delivery/cleanup. Fixes #853/#854 are merged; checksummed evidence outlives Actions retention. This changes no database runtime -behavior or Rust test counts. Additional JVM/WASI and qualified mobile nightly -lanes are tracked separately in #858. +behavior or Rust test counts. Core workflow support for 14 JVM and 3 WASI nightly IDs is implemented for #858 +([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter, default-branch rollout +and live baseline/notification evidence remain pending. Mobile lanes remain +excluded pending qualification. Rust test counts are unchanged. The [#842 semantic permutation map](docs/benchmarks/842-semantic-permutations/README.md) ties the excluded numeric, bind, join and forced-plan permutations to 128 diff --git a/README.md b/README.md index e3cf225198..8f8d878684 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,10 @@ The [manual v7-to-v8 migration sweep](docs/benchmarks/838-migration/README.md) ( The [regression-history follow-up](docs/benchmarks/840-regression/completion.md) (#840) verifies numeric comparisons for all 427 native and 26 binding IDs, independent v3 history, scheduled main/v3 routes and isolated live notification delivery with -cleanup. Raw evidence and offline verifiers are retained in Git. JVM/WASI and -qualified mobile nightly expansion are tracked separately in #858. +cleanup. Raw evidence and offline verifiers are retained in Git. Core workflow support for 14 JVM and 3 WASI nightly IDs is implemented for #858 +([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter, default-branch rollout +and live baseline/notification evidence remain pending. Mobile lanes remain +excluded pending qualification; Rust test counts are unchanged. The [semantic permutation map](docs/benchmarks/842-semantic-permutations/README.md) (#842) ties excluded numeric, bind, join and forced-plan permutations to diff --git a/ROADMAP.md b/ROADMAP.md index 2c5f1982b0..f175a203af 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -133,7 +133,7 @@ Scope — release-gate testing and documentation: - Benchmarks at 100K–10M facts and a published performance envelope (#394; on `v3`: `examples/perf_envelope.rs`, results in `docs/BENCHMARKS.md`; follow-ups #528–#532, with #532 superseded by #551's removal of repeated committed-fact net-assert; [audit and measurements](docs/benchmarks/532-net-assert-audit.md)) - Acceptance evidence for memory/scaling (#837): named-test and Linux RSS gates, nonlinear/multistratum recursion and custom chunk bounds; [validation status](docs/benchmarks/837-acceptance-gaps.md) - Regression-history/reporting evidence (#840) is complete: [all 453 numeric comparisons, preserved independent v3 history, scheduled routes and live notification/cleanup evidence](docs/benchmarks/840-regression/completion.md). Separate main/v3 fixes #853/#854 are merged; raw evidence is retained beyond artifact expiry. -- Nightly runtime expansion (#858): add Java/JVM and WASI; qualify Swift/iOS simulator and Android emulator timings against physical-device workloads before enabling representative mobile lanes. The platform restriction from #839 applies only to that manual evidence task. +- Nightly runtime expansion (#858): core workflow support for 14 Java/JVM and 3 WASI IDs is implemented ([contract](docs/BINDING_BENCHMARKS.md)); Java adapter integration, default-branch rollout, live baselines/notifications and durable evidence remain pending. Swift/iOS simulator and Android emulator lanes stay excluded until qualification against physical-device workloads. The platform restriction from #839 applies only to that accepted manual evidence task. - Manual migration evidence (#838): native v7-to-v8 through 200k facts; Chromium/Firefox/WebKit through 20k facts, with separate read-only/reopen controls and bounded post-migration cursor churn. [Results, frozen fixtures and exclusions](docs/benchmarks/838-migration/README.md); no new nightly route or sustained-pin bound. - Semantic permutations (#842): [correctness-target map and control limits](docs/benchmarks/842-semantic-permutations/README.md) for X07. The exclusion stands; no nightly route or acceptance gate is added. - Failure paths (#843): [correctness-target map](docs/benchmarks/843-failure-correctness/README.md) for X08. The exclusion stands; the heavy WAL replay test gap is #865. diff --git a/benchmarks/bindings839/README.md b/benchmarks/bindings839/README.md index 8ed84ad3d0..aba3ee9c92 100644 --- a/benchmarks/bindings839/README.md +++ b/benchmarks/bindings839/README.md @@ -5,6 +5,14 @@ The [scope and coverage plan](../../docs/benchmarks/839-bindings/README.md) and manual runs from existing GitHub nightly benchmarks. No release acceptance or latency threshold is implied. +## Nightly integration (#858) + +`nightly.py` adapts the unchanged WASI driver/validator for hosted Ubuntu 24.04 +nightlies. See [the lane and adapter contracts](../../docs/BINDING_BENCHMARKS.md) +for runtime pins, provenance, fresh-output rules and raw-sample recomputation. +The dedicated-bench instructions below describe historical #839 reproduction; +#858 separately authorizes these hosted JVM/WASI lanes. + ## WASI reproduction Use a clean core checkout at the recorded revision and Rust 1.99.1. Copy the diff --git a/benchmarks/bindings839/nightly.py b/benchmarks/bindings839/nightly.py new file mode 100644 index 0000000000..c3e9ff147f --- /dev/null +++ b/benchmarks/bindings839/nightly.py @@ -0,0 +1,100 @@ +"""Run the unchanged #839 WASI driver, then publish a validated nightly stream. + +Metrics are nanoseconds: median of ten sample means (total_ns / iterations). +A driver failure, including teardown after the last sample, publishes no metrics. +""" +import argparse +import hashlib +import json +import os +import pathlib +import statistics +import subprocess +import sys + +if not __debug__: + raise RuntimeError('do not disable Python validation with -O') + +HERE = pathlib.Path(__file__).resolve().parent +# -I excludes ambient imports; trust only this checkout's sibling validator. +sys.path.insert(0, str(HERE)) +from validate import validate, WASI + +# Official x86_64-linux release binary, also retained in #839's three raw streams. +WASMTIME_VERSION = 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)' +WASMTIME_SHA256 = 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635' +FLAGS = ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'] + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def metrics(records): + validate(records, 'wasi') + return {name: {'latency': {'value': statistics.median( + r['total_ns'] / r['iterations'] for r in records + if r['type'] == 'sample' and r['id'] == name)}} + for name in sorted('binding/wasi/' + suffix for suffix in WASI)} + + +def check_environment(env, core_sha, wasm_sha): + assert env['core_revision'] == core_sha, 'wrong measured core' + assert env['wasm_sha256'] == wasm_sha, 'wrong executed WASM' + assert env['runtime_sha256'] == WASMTIME_SHA256, 'wrong Wasmtime binary' + assert env['runtime_version'] == WASMTIME_VERSION, 'wrong Wasmtime version' + assert env['runtime_flags'] == FLAGS, 'wrong runtime flags (no preopens allowed)' + assert env['runtime_cache'] is False + assert env['storage'] == 'in-memory; no preopened directories' + assert env['warmups'] == 3 and env['samples'] == 10 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--runtime', type=pathlib.Path, required=True) + parser.add_argument('--wasm', type=pathlib.Path, required=True) + parser.add_argument('--core-dir', type=pathlib.Path, required=True) + parser.add_argument('--output', type=pathlib.Path, required=True) + parser.add_argument('--test', action='store_true') + args = parser.parse_args() + core_sha = subprocess.check_output(['git', '-C', str(args.core_dir), 'rev-parse', 'HEAD'], text=True).strip() + assert core_sha == os.environ['BENCH_SHA'], 'checkout differs from once-resolved SHA' + workflow_sha = os.environ['WORKFLOW_SHA'] + assert len(workflow_sha) == 40 and all(c in '0123456789abcdef' for c in workflow_sha) + # Check the executable before running even --version. + assert sha(args.runtime) == WASMTIME_SHA256, 'wrong Wasmtime binary' + assert subprocess.check_output([str(args.runtime.resolve()), '--version'], text=True).strip() == WASMTIME_VERSION + wasm_sha = sha(args.wasm) + harness_files = {name: sha(HERE / name) for name in ('nightly.py', 'wasi.py', 'validate.py')} + args.output.mkdir(parents=True, exist_ok=True) + result_path = args.output / 'binding-results.json' + provenance_path = args.output / 'binding-provenance.json' + assert not result_path.exists() and not provenance_path.exists(), 'stale nightly outputs' + mode = 'smoke' if args.test else 'benchmark' + output = args.output / mode # driver refuses an existing directory + subprocess.run([sys.executable, '-I', str(HERE / 'wasi.py'), + '--runtime', str(args.runtime.resolve()), '--wasm', str(args.wasm.resolve()), + '--core-revision', core_sha, '--output', str(output), '--mode', mode], check=True) + records = [json.loads(line) for line in (output / 'samples.jsonl').read_text().splitlines()] + check_environment(records[0], core_sha, wasm_sha) + assert sha(args.wasm) == wasm_sha and sha(args.runtime) == WASMTIME_SHA256, 'artifact changed during run' + if args.test: + assert not any(r['type'] == 'sample' for r in records), 'smoke emitted timing data' + return + result = metrics(records) + provenance = dict(coreSha=core_sha, workflowSha=workflow_sha, bindingSha=None, + harnessFiles=harness_files, + harnessSha256=hashlib.sha256(json.dumps(harness_files, sort_keys=True, separators=(',', ':')).encode()).hexdigest(), + artifactSha256=wasm_sha, artifactPath='minigraf.wasm', + runtime=dict(version=WASMTIME_VERSION, sha256=WASMTIME_SHA256, flags=FLAGS), + environment=records[0]) + with provenance_path.open('x') as stream: + stream.write(json.dumps(provenance, indent=2) + '\n') + with result_path.open('x') as stream: + stream.write(json.dumps(result, indent=2) + '\n') + for name in result: + print('ok: ' + name) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/bindings839/test_nightly.py b/benchmarks/bindings839/test_nightly.py new file mode 100644 index 0000000000..950c162c62 --- /dev/null +++ b/benchmarks/bindings839/test_nightly.py @@ -0,0 +1,179 @@ +"""Nightly conversion keeps #839's strict oracle and raw-sample statistic.""" +import copy +import contextlib +import hashlib +import io +import os +import importlib.util +import json +import pathlib +import statistics +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +def fixture(): + # Synthetic unit-test data, never qualification or baseline measurements. + # Self-contained so the main CI companion needs no historical evidence copy. + records = [dict(type='environment', mode='benchmark', core_revision='c' * 40, + wasm_sha256='d' * 64, + runtime_sha256='d0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', + runtime_version='wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', + runtime_flags=['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'], + storage='in-memory; no preopened directories', runtime_cache=False, + warmups=3, samples=10)] + ids = ['binding/wasi/' + suffix for suffix in + ('repl_startup_empty', 'repl_script/1k', 'repl_query_roundtrip/1k')] + for index, name in enumerate(ids): + iterations = 20 if index == 2 else 1 + for sample in range(10): + total = (101 + sample * 17) * iterations + index + records.append(dict(type='sample', id=name, sample=sample, + iterations=iterations, total_ns=total, ns_per_op=total / iterations)) + records.append(dict(type='checked', id=name)) + records.append(dict(type='complete', mode='benchmark', count=3, ids=ids)) + return records + + +class NightlyTests(unittest.TestCase): + def setUp(self): + self.assertTrue((HERE / 'nightly.py').is_file(), 'nightly adapter is required') + spec = importlib.util.spec_from_file_location('nightly', HERE / 'nightly.py') + self.adapter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.adapter) + self.records = fixture() + + def test_raw_recomputation_of_every_metric(self): + metrics = self.adapter.metrics(self.records) + self.assertEqual(len(metrics), 3) + for name, metric in metrics.items(): + samples = [r for r in self.records if r.get('id') == name and r['type'] == 'sample'] + expected = statistics.median(r['total_ns'] / r['iterations'] for r in samples) + self.assertEqual(metric, {'latency': {'value': expected}}) + + def test_rejects_invalid_or_incomplete_streams(self): + sample = next(i for i, r in enumerate(self.records) if r['type'] == 'sample') + variants = [self.records[:-1], self.records + [self.records[-1]], self.records + [self.records[sample]]] + for key, value in [('total_ns', 0), ('total_ns', -1), ('iterations', 2), + ('ns_per_op', float('nan')), ('ns_per_op', float('inf')), + ('ns_per_op', 123), ('id', 'binding/wasi/extra')]: + records = copy.deepcopy(self.records) + records[sample][key] = value + variants.append(records) + smoke = copy.deepcopy(self.records) + smoke[0]['mode'] = 'smoke' + variants.append(smoke) + for records in variants: + with self.subTest(records=records[sample]): + with self.assertRaises((AssertionError, ValueError)): + self.adapter.metrics(records) + + def test_runtime_pin_and_flags_are_enforced(self): + env = self.records[0] + self.adapter.check_environment(env, env['core_revision'], env['wasm_sha256']) + for key, value in [('runtime_sha256', '0' * 64), ('runtime_version', 'wasmtime 49.0.3'), + ('runtime_flags', ['run', '--dir', '.']), ('runtime_cache', True), + ('warmups', 0), ('samples', 1), ('core_revision', 'a' * 40), + ('wasm_sha256', 'b' * 64)]: + with self.subTest(key=key), self.assertRaises((AssertionError, ValueError)): + self.adapter.check_environment(dict(env, **{key: value}), env['core_revision'], env['wasm_sha256']) + + def test_optimized_adapter_is_rejected(self): + result = subprocess.run([sys.executable, '-I', '-O', str(HERE / 'nightly.py'), '--help'], + capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn('do not disable', result.stderr) + + def test_publish_only_after_successful_exit_and_reject_stale_outputs(self): + for mode in ('success', 'smoke', 'late-exit-failure'): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + runtime, wasm = root / 'runtime', root / 'minigraf.wasm' + runtime.write_bytes(b'test runtime') + wasm.write_bytes(b'test artifact') + runtime_sha = hashlib.sha256(runtime.read_bytes()).hexdigest() + records = copy.deepcopy(self.records) + records[0]['runtime_sha256'] = runtime_sha + records[0]['wasm_sha256'] = hashlib.sha256(wasm.read_bytes()).hexdigest() + if mode == 'smoke': + records = [r for r in records if r['type'] != 'sample'] + records[0]['mode'] = records[-1]['mode'] = 'smoke' + + def driver(command, **kwargs): + output = pathlib.Path(command[command.index('--output') + 1]) + output.mkdir() # the real driver's fresh-directory rule + (output / 'samples.jsonl').write_text('\n'.join(map(json.dumps, records))) + if mode == 'late-exit-failure': + # Even a complete stream cannot override process failure. + raise subprocess.CalledProcessError(1, command) + + argv = ['nightly.py', '--runtime', str(runtime), '--wasm', str(wasm), + '--core-dir', str(root), '--output', str(root)] + if mode == 'smoke': + argv.append('--test') + with mock.patch.object(sys, 'argv', argv), \ + mock.patch.dict(os.environ, BENCH_SHA=records[0]['core_revision'], WORKFLOW_SHA='a' * 40), \ + mock.patch.object(self.adapter, 'WASMTIME_SHA256', runtime_sha), \ + mock.patch.object(self.adapter.subprocess, 'check_output', side_effect=[records[0]['core_revision'], self.adapter.WASMTIME_VERSION]), \ + mock.patch.object(self.adapter.subprocess, 'run', side_effect=driver), \ + contextlib.redirect_stdout(io.StringIO()) as stdout: + if mode == 'late-exit-failure': + with self.assertRaises(subprocess.CalledProcessError): + self.adapter.main() + else: + self.adapter.main() + metrics_file = root / 'binding-results.json' + self.assertEqual(metrics_file.exists(), mode == 'success') + self.assertEqual((root / 'binding-provenance.json').exists(), mode == 'success') + self.assertEqual(stdout.getvalue().count('ok: '), 3 if mode == 'success' else 0) + if mode == 'success': + self.assertEqual(json.loads(metrics_file.read_text()), self.adapter.metrics(records)) + with mock.patch.object(sys, 'argv', argv), \ + mock.patch.dict(os.environ, BENCH_SHA=records[0]['core_revision'], WORKFLOW_SHA='a' * 40), \ + mock.patch.object(self.adapter, 'WASMTIME_SHA256', runtime_sha), \ + mock.patch.object(self.adapter.subprocess, 'check_output', side_effect=[records[0]['core_revision'], self.adapter.WASMTIME_VERSION]), \ + mock.patch.object(self.adapter.subprocess, 'run') as run: + with self.assertRaisesRegex(AssertionError, 'stale'): + self.adapter.main() + run.assert_not_called() + + def test_driver_failure_cannot_publish_metrics(self): + # Exercise the real runner/REPL parser with wrong-but-plausible output. + # This substitute runtime is test-only; production's binary pin rejects it. + with tempfile.TemporaryDirectory() as tmp: + tmp = pathlib.Path(tmp) + runtime = tmp / 'fake-runtime' + runtime.write_text('''#!/usr/bin/env python3 +import sys +if '--version' in sys.argv: + print('fake runtime') + sys.exit(0) +script = sys.stdin.read() +if script: + print('✓ Transacted successfully (tx: 1)') + print('?n\\n' + '-' * 20) + # 1000 rows with the right sum, but wrong values. + print('\\n'.join(map(str, [0, 0, 3, 3] + list(range(4, 1000))))) + print('\\n1000 result(s) found.') +''') + runtime.chmod(0o755) + wasm = tmp / 'fake.wasm' + wasm.write_bytes(b'fixture') + output = tmp / 'run' + result = subprocess.run([sys.executable, '-I', str(HERE / 'wasi.py'), + '--runtime', str(runtime), '--wasm', str(wasm), + '--core-revision', 'test', '--mode', 'benchmark', + '--output', str(output)], capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn('wrong row values', result.stderr) + records = [json.loads(line) for line in (output / 'samples.jsonl').read_text().splitlines()] + with self.assertRaises(AssertionError): + self.adapter.metrics(records) + self.assertFalse((output / 'binding-results.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 93826bb942..359c9fc4d3 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -2,6 +2,15 @@ This document records reproducible local benchmark snapshots. Completed release-performance history belongs in the [CHANGELOG](../CHANGELOG.md); continuous CI history is available on [Bencher](https://bencher.dev/perf/minigraf/plots). +## JVM/WASI nightly workflow support (#858, unreleased) + +The core workflow defines 14 JVM and 3 WASI routes with separate Ubuntu 24.04 +histories, pinned runtimes, exact-result checks, raw sample medians and artifact +provenance. See [binding benchmarks](BINDING_BENCHMARKS.md) for timing boundaries +and the Java adapter contract. Java integration, default-branch rollout and live +baselines/notifications remain pending. No mobile lane is enabled before +qualification, and no new runtime timings are claimed by this implementation. + ## Failure paths (#843, unreleased) The [X08 map](benchmarks/843-failure-correctness/README.md) assigns invalid diff --git a/docs/BINDING_BENCHMARKS.md b/docs/BINDING_BENCHMARKS.md index 26b12ecca4..b0c6eafe10 100644 --- a/docs/BINDING_BENCHMARKS.md +++ b/docs/BINDING_BENCHMARKS.md @@ -1,18 +1,31 @@ # Binding benchmarks (#776 P16) -The remaining manual work is tracked in [#839's scope and coverage plan](benchmarks/839-bindings/README.md). -For that issue, new timings come only from Device Farm and the dedicated Linux -bench; macOS timing is excluded. GitHub may build and check correctness, and the -existing nightly measurements described here continue unchanged. - -These are foreign-runtime measurements, independent of native Criterion data. -The `bindings` matrix in `.github/workflows/bench.yml` selects the whole fixed -shortlist in each binding's `bench/nightly.*` entry point. Core and binding PRs -must land before enabling the default-v3 route. The default-main workflow needs -its identical companion; schedules then select v3 at 14:00 UTC. The 02:00 main -checkout skips all four jobs before building or uploading. Manual runs may use -`binding-ref` to select the common `bench/776-p16-bindings` PR branch during -phase B; no dispatch is part of phase A. +The accepted [#839 manual evidence](benchmarks/839-bindings/README.md) remains +historical evidence. #858 adds core workflow support for JVM (14 IDs) and WASI +(3 IDs), alongside the existing 26 Python/browser/Node/C IDs. The Java nightly +adapter and identical default-branch CI companion must land before rollout. +The main companion must also carry `benchmarks/bindings839/` reporting/test +dependencies, so its reporting CI can run without the v3 historical archives. +Three independent v3 runs (including a scheduled run), live notification checks, +durable evidence and measured lane costs are still pending. This change does +not complete #858 or activate a mobile simulator/emulator lane. + +The `bindings` matrix in `.github/workflows/bench.yml` measures the once-resolved +core SHA. JVM checks out `minigraf-java`; WASI has no binding repository. The +14:00 UTC schedule measures v3; main/v2 checkouts skip with a step-summary note +and no uploads. A required harness or artifact missing on v3 fails the job. +Reporting helpers come from `github.workflow_sha`; the WASI driver comes from +the measured core. Both identities are recorded. + +Manual dispatch can set `jvm-ref` for a Java-only experiment, leaving every +other binding on `binding-ref` (default `v3`). `binding-ref` still selects a +common ref for all binding repositories, so use it only when that ref exists +in all of them. WASI ignores binding refs and always uses the measured core. +Non-v3 core or binding refs use `binding-experiment//` +histories, with no production notifications. An experimental prior job cannot +satisfy the production two-run repeat policy. + +## Existing Python/browser/Node/C lanes All cases use 1,000 deterministic in-memory integer facts, plus one two-element set fact. The suffixes below expand under `binding/{python,browser,node,c}/`: @@ -64,7 +77,7 @@ Artifacts retain both commit hashes, workflow hash, resolved Cargo metadata, executed IDs, raw metrics, upload output and the completeness report. Bencher uses `binding-{python,browser,node,c}-ubuntu24-v1` testbeds, Python 3.11, Node 20 and Playwright 1.55.0 Chromium on Ubuntu 24.04. Native histories are not -inherited. An experimental `binding-ref` uses a separate +inherited. An experimental core or binding ref uses the separate `binding-experiment//` history branch and does not post production regression issues. Each testbed has its own 0.999 t-test threshold with a 30-point window; alerts use the existing two-consecutive-run issue policy, while missing @@ -76,27 +89,145 @@ confirm every ID and upload, establish the new testbed histories, and revise this allowance from actual Actions evidence. No latency or RSS measurement was performed locally. These latency cases do not enforce RSS bounds. -## Deliberate dispositions requiring release sign-off - -- Java/JVM: no JVM nightly latency history. Python exercises the shared UniFFI - adapter, but does not measure JVM allocation, JNA or JVM callbacks. The - [#839 manual series](benchmarks/839-bindings/results.md) supplies separate - release evidence; acceptance remains pending. -- Android: ART/ABI and device storage timing require a fixed physical device; - emulator timings on a hosted Linux runner are not a substitute. Manual - device benchmarks and release sign-off remain required. -- Swift: ARC/callback and Apple-device storage timing require stable Apple - hardware. Manual iOS benchmarks and release sign-off remain required for #839; - macOS timing is explicitly excluded from that issue's approved scope. - Linux UniFFI tests do not provide these timings. -- WASI: browser Wasm does not measure the WASI runtime. Tier 2 WASI REPL startup - and runtime overhead remain manual; it exposes no matching foreign prepared - object or synchronous host callback API in the audited package. -- Options validation/error paths and fact-log/LogWriter conversions are outside - this shortlist. Native tests/benchmarks cover engine work, not these foreign - conversions. They remain explicitly unmeasured follow-up scope. Browser - persistence coverage is write-through/open only; migration and pinned-reader - churn remain manual. Deliberate callback errors stay correctness tests. - -G15 therefore gains scheduled representatives for four host runtimes; the -listed manual/excluded paths need explicit release sign-off before #776 closes. +## JVM lane (#858) + +Suite `binding_jvm` uses testbed `binding-jvm-ubuntu24-v1` on Ubuntu 24.04, +Temurin **17.0.17+10** (vendor `Eclipse Adoptium`) and exactly +`-Xms512m -Xmx512m -XX:+UseG1GC`. The environment of the measured JVM records +vendor, full VM version and actual input arguments; the reporter checks them. +The manual #839 harness remains authoritative for operations and exact results. +The six common suffixes above expand under `binding/jvm/`; eight more are: + +| Suffix | Timed boundary | +| --- | --- | +| `prepare/point` | Create a prepared point query; validate it outside timing | +| `register_predicate/first` | First predicate registration on a fresh database | +| `register_aggregate/first` | First aggregate registration on a fresh database | +| `fact_log_batch_256/1k` | Open, fully drain and close a 1k-record fact log; includes foreign record conversion | +| `options_error/negative_cache` | Reject a negative cache option and check the error code | +| `options_readonly_open/1k` | Open an already seeded file with read-only/cache/memory options | +| `file_transact/1k` | One 1k-fact file transaction; reopen/validate outside timing | +| `log_writer_append_finish/1k` | Append 1k records and finish; reopened full records checked outside timing | + +Short operations use 100 warmups and ten samples of 20 invocations. The two +write cases use five warmups and ten single-invocation samples. JVM startup is +outside these timed calls; JIT warmups do not prove stationarity. JNA, foreign +allocation/conversion and callback overhead remain included. Neither JVM nor +WASI results are engine-only latency. + +### Contract for minigraf-java's nightly adapter + +The core helper runs from the measured checkout with the Java repository at +`binding/`. **`bench/manual/build.sh CORE_CHECKOUT` owns the Cargo patch**; the +core helper does not create one for JVM. It sets `CARGO_BUILD_JOBS=4`, +`CARGO_TARGET_DIR=/target`, `CORE_REVISION=BENCH_SHA`, and +`BINDING_REVISION=`. The manual builder must keep its +metadata assertion before compilation and run `ResultChecksTest`. After it +returns, the helper runs `cargo metadata --locked` again and requires exactly +one `minigraf` package, `source=null`, at the measured checkout's manifest. +The manual builder's existing `target/release` and Gradle copy paths agree. + +The later binding-repo implementation must provide **`bash bench/nightly.sh`**, +invoked from the binding root after the build. Its only optional argument is +**`--test`**. It inherits `BENCH_SHA`, `WORKFLOW_SHA`, `JAVA_HOME` and +`CARGO_BUILD_JOBS=4`; read the actual binding HEAD with Git. Do not rebuild or +repatch in this entry point. Launch the installed manual harness with the pinned +JDK and flags, preserving #839's timing and exact-result checks. Require successful +process exit and all teardown before publishing anything accepted for upload. +A complete record printed before a later teardown failure is insufficient. + +- `--test`: fresh `smoke/` directory, one untimed invocation per case; retain raw + checks, emit no sample records, `binding-results.json` or + `binding-provenance.json`. Include a wrong-but-plausible result rejection test + in the binding adapter's own test suite. +- No arguments: fresh `benchmark/` directory. Preserve the manual stream as + `benchmark/samples.jsonl` (copy/rename `minigraf-bench.jsonl` only after successful + process exit). It must pass core `validate.py jvm`: one benchmark environment, + ten samples per exact ID (`sample=0..9`, `total_ns`, `iterations`, `ns_per_op`), + one `checked` record per ID and one matching `complete` record. +- Write `binding-results.json` as `{ "binding/jvm/": { "latency": { + "value": N } } }`, with exactly the 14 IDs. `N` is **nanoseconds**, the median + of the ten `total_ns / iterations` values (average of sorted indices 4 and 5). + The core reporter recomputes every metric from retained raw samples. +- Print exactly one `ok: binding/jvm/` line per ID, only after all + validation and teardown succeeds. Other output must not start with `ok: `. + Missing, extra or duplicate IDs/keys and nonpositive/nonfinite metrics fail. +- Write `binding-provenance.json` with `coreSha`, `workflowSha`, `bindingSha` + (actual 40-character Git SHAs), `artifactPath: "benchmark/libminigraf_ffi.so"`, + `artifactSha256`, `harnessFiles`, `harnessSha256`, and `runtime` as below. + Retain at that artifact path the native bytes **actually loaded or extracted + from the installed harness's packaged JAR**, and verify them against the build + library. A hash of an unused build-directory library is insufficient. Prevent + JNA/library-path overrides from selecting another native library. +- `harnessFiles` maps binding-relative source filenames to lowercase SHA-256. + Include all harness/adapter source and configuration files, at minimum + `bench/nightly.sh`, `bench/manual/build.sh`, `bench/manual/build.gradle.kts`, + and `bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/`'s + `Benchmark.kt` and `ResultChecks.kt`. Exclude generated build/results files. + `harnessSha256` hashes the UTF-8 JSON of this map with lexically sorted keys, + no spaces and no final newline. The reporter rehashes the source files. +- `runtime` is `{ "distribution": "temurin", "vendor": "Eclipse Adoptium", + "version": "17.0.17+10", "flags": ["-Xms512m", "-Xmx512m", "-XX:+UseG1GC"] }`. + Derive vendor/version/flags from the actual measured JVM environment, not a + shell label. Preserve raw `java_vendor`, `java_version` (`17.0.17`), + `vm_version` (`17.0.17+10`), `jvm_arguments`, `warmup_memory=100`, + `warmup_storage=5`, and `build.core_revision`/`build.binding_revision`. + +Existing output directories or metric/provenance files must fail, never satisfy +another invocation. The core helper also rejects stale files and smoke metrics. +The core reporter fails before upload on missing evidence, mismatched identities, +artifact/harness hashes, runtime configuration, samples or medians. + +## WASI lane (#858) + +Suite `binding_wasi` uses `binding-wasi-ubuntu24-v1` on Ubuntu 24.04. The workflow +builds `minigraf.wasm` from the measured core with `--locked --release` and +`wasm32-wasip1`, then runs `benchmarks/bindings839/nightly.py`. It wraps the +unchanged `wasi.py` and `validate.py`; smoke and benchmark directories are fresh. +The adapter publishes metrics and `ok:` lines only after successful driver exit +and complete stream validation. `benchmark/samples.jsonl` retains all 30 samples; +metrics are the median of ten sample means in nanoseconds. + +| ID | Timed boundary | +| --- | --- | +| `binding/wasi/repl_startup_empty` | New Wasmtime process, uncached compilation, empty REPL and exit | +| `binding/wasi/repl_script/1k` | New process, uncached compilation, 1k-fact transaction and full query output | +| `binding/wasi/repl_query_roundtrip/1k` | Query in a persistent seeded REPL, including pipes, flushing and host line framing | + +All three use three warmups and ten samples. Startup/script samples contain one +invocation; persistent-query samples average 20 invocations. Every invocation +checks exact rows, count and sum, successful exit and empty stderr. Persistent +process teardown must also pass. Startup/compilation and pipe/framing overheads +are not subtracted; these are actual REPL measurements. + +Wasmtime is pinned to **49.0.2**, official `x86_64-linux` binary SHA-256 +`d0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635` +(the binary recorded in #839's retained WASI streams). The checksum is checked +before execution and the version must be +`wasmtime 49.0.2 (3c8a3e79a 2026-10-02)`. +Arguments are exactly `run -C cache=n -C parallel-compilation=n`, followed by the +WASM path: **no preopened directory**. The adapter records core/workflow SHAs, +WASM SHA-256, runtime version/checksum/flags, and individual plus combined +SHA-256 hashes of `nightly.py`, `wasi.py` and `validate.py`. + +## Rollout, histories and remaining exclusions + +Both new lanes share the existing selected = executed = uploaded audit, +0.999 t-test upper boundary and 30-sample window. They inherit no native or other +binding history; v3 uploads use no `--start-point`, so main advances cannot reset +them. Evidence (raw samples, provenance, executed IDs, metrics, native/WASM +artifact, Cargo metadata and upload/report output) is retained for **30 days**. +A durable checksummed archive and measured setup/build/measure durations remain +post-rollout work; no new hosted-runtime timings or cost claims were collected +for this core implementation. The 60-minute timeout is a limit, not a cost estimate. +Changing JDK, Wasmtime, runner configuration or timing boundaries requires a new +testbed version. + +#839's maintainer acceptance applies to its recorded physical/dedicated-bench +evidence, not new nightly baselines. Android and Swift/iOS simulator/emulator +lanes remain excluded pending #858's matched physical B1/B2/V qualification. +Desktop macOS Swift remains excluded. JVM now has representative lifecycle, +options and log conversion routes; other bindings' unmeasured permutations, +wide values, scaling, RSS bounds and deliberate callback failures remain outside +this shortlist. WASI exposes no matching prepared-object or host-callback API. +Browser migration and pinned-reader churn remain manual. diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 9c7fc550cf..0c60e6888d 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -100,7 +100,10 @@ The initial four-run schedule archive and five new measurement runs are retained Sixteen isolated live notification scenarios verify four actual deliveries and cleanup of all 46 synthetic resources. Fresh extraction reproduces the evidence reports, and removing delivery/comparison evidence is rejected. No Rust tests -are added; the inventory above is unchanged. Additional runtime lanes are #858. +are added; the inventory above is unchanged. The #858 core workflow adds 14 JVM and 3 WASI routes with raw-sample and +provenance checks ([contract](BINDING_BENCHMARKS.md)). The Java adapter, +default-branch rollout and live baselines/notifications remain pending; mobile +lanes remain excluded pending qualification. No Rust tests are added. **#843 failure-path map**: [138 curated correctness tests](benchmarks/843-failure-correctness/README.md) (35 cells; 5 tests exist only in the `test-hooks` build) cover the failure paths excluded from diff --git a/docs/bench-coverage-v3.md b/docs/bench-coverage-v3.md index 577ff94c4e..424e1e0aa0 100644 --- a/docs/bench-coverage-v3.md +++ b/docs/bench-coverage-v3.md @@ -5,8 +5,10 @@ establishes numeric comparisons for all 427 native and 26 binding IDs with at least three independent v3 runs each. Merged fixes #853/#854 preserve the v3 head after a measured main advance. Actual scheduled routes, isolated live notification delivery and cleanup are verified; complete evidence is archived beyond 30 days. -JVM/WASI and qualified iOS-simulator/Android-emulator expansion are tracked in -#858. The original validation snapshot below is retained unchanged; its null +The #858 core workflow now defines 14 JVM and 3 WASI routes +([contract](BINDING_BENCHMARKS.md)); Java adapter integration, default-branch +rollout and live baselines/notifications remain pending. No mobile lane is +enabled before qualification. The original validation snapshot below is retained unchanged; its null baselines and X05 open status describe that earlier snapshot. Audit date: 2026-11-03. P01–P17 are merged. This closes out the implementation diff --git a/docs/benchmarks/839-bindings/README.md b/docs/benchmarks/839-bindings/README.md index 4032db92ce..a5aca67ad2 100644 --- a/docs/benchmarks/839-bindings/README.md +++ b/docs/benchmarks/839-bindings/README.md @@ -1,5 +1,11 @@ # Binding release evidence (#839) +Current status (#858 follow-up): #839 was accepted by both maintainer roles for +evidence revision `10ff8003…`, merged in #852 as `cdad3e72…`. The pending-review +wording below is the preserved historical record. New JVM/WASI nightly support +and mobile qualification are separate #858 work; see +[the current lane contract](../../BINDING_BENCHMARKS.md). + Status: runtime evidence collected; binding and release-maintainer acceptance is pending. This document records execution and coverage, not a release sign-off. diff --git a/docs/benchmarks/839-bindings/dispositions.md b/docs/benchmarks/839-bindings/dispositions.md index cfee186e43..4ca3ee72d2 100644 --- a/docs/benchmarks/839-bindings/dispositions.md +++ b/docs/benchmarks/839-bindings/dispositions.md @@ -1,5 +1,11 @@ # Remaining API and lifecycle dispositions +Current status (#858 follow-up): #839 was accepted by both maintainer roles for +evidence revision `10ff8003…`, merged in #852 as `cdad3e72…`. The pending-review +wording below is the preserved historical record. New JVM/WASI nightly support +and mobile qualification are separate #858 work; see +[the current lane contract](../../BINDING_BENCHMARKS.md). + These are **proposals for binding and release-maintainer acceptance**, not approved exclusions. If a disposition is rejected, its applicable cases must be measured on the bench or Device Farm before #839 closes. Existing P16 coverage diff --git a/docs/benchmarks/839-bindings/results.md b/docs/benchmarks/839-bindings/results.md index a4f0d8311c..3ff58df628 100644 --- a/docs/benchmarks/839-bindings/results.md +++ b/docs/benchmarks/839-bindings/results.md @@ -1,5 +1,11 @@ # Manual measurements (pending acceptance) +Current status (#858 follow-up): #839 was accepted by both maintainer roles for +evidence revision `10ff8003…`, merged in #852 as `cdad3e72…`. The pending-review +wording below is the preserved historical record. New JVM/WASI nightly support +and mobile qualification are separate #858 work; see +[the current lane contract](../../BINDING_BENCHMARKS.md). + Each cell is the median of ten sample means, in milliseconds. Runs remain separate. These runtimes and operations are not equivalent workloads or a ranking. diff --git a/docs/superpowers/specs/2026-11-18-binding-nightlies-design.md b/docs/superpowers/specs/2026-11-18-binding-nightlies-design.md new file mode 100644 index 0000000000..ca7f2c6391 --- /dev/null +++ b/docs/superpowers/specs/2026-11-18-binding-nightlies-design.md @@ -0,0 +1,154 @@ +# Spec — 007-binding-nightlies (#858) + +**Mission Type:** mixed (CI workflow/reporting code, binding benchmark harness code, empirical qualification, durable evidence and docs) + +**Base branch:** `v3` for core work. `main` gets a companion PR with the identical shared-CI change, because scheduled runs use the default branch's `bench.yml`. Binding harness changes land on the binding repos' `v3` branches. No v2.x measurement or backport. + +## 1. User Intent + +> next v3.0.0 issue #858 + +Issue #858, "Bench: add JVM/WASI nightly lanes and qualify iOS/Android emulator coverage" (milestone v3.0.0), is the controlling scope. Maintainer clarification in the issue (2026-11-05): the #839 physical-device/dedicated-bench restriction is not a blanket rule. "A hosted iOS simulator or Android emulator may be included in nightlies **if evidence shows sufficiently close performance to the corresponding physical-device workloads**." This issue does not assume that condition is met and does not allow emulator results to be treated as physical-device results without qualification. + +Planning answers (2026-11-18): + +> Scope: A+B in one mission — JVM/WASI lanes plus the reviewed mobile qualification protocol, measurement and per-category decision; #858 closes when all are done. + +> Device Farm: Yes, fresh matched runs — new physical series at the candidate revision, baseline and injected-slowdown variant, same device models as #839. + +> Thresholds: the draft numbers (§4 AC12), fixed now in the spec. + +> Shortlist: all 14 JVM + 3 WASI cases, v3-only like the existing binding lanes. + +## 2. Goal + +Extend the default-branch-owned nightly benchmark system with Linux/JVM and WASI lanes that measure the once-resolved core of each v3 run. Each lane gets exact-result validation, the existing selected = executed = uploaded reporting contract, its own Bencher history and working regression notification. Separately, qualify the iOS simulator and the Android emulator on GitHub-hosted runners against fresh, matched physical-device runs on AWS Device Farm. Use thresholds pre-registered in this spec. Enable a simulator/emulator nightly lane only for the cases that pass, and give every other case an explicit, evidenced disposition. No database runtime, public API, file format, core dependency or binary-size change. + +## 3. Context & Investigation Findings + +Full investigation: `Planner-Discussion/Claude-01.md` and `Codex-01.md`. Key references: + +### Nightly system (identical on `main` and `origin/v3` today) +- `.github/workflows/bench.yml:1-43`: cron `0 2 * * *` measures `main`, `0 14 * * *` measures `v3`. Workflow concurrency group `bench`. Scheduled runs always use the default branch's file. +- `bench.yml:46-98`: the `compile` job resolves the measured SHA once. Helpers come from `github.workflow_sha` (`.bench-workflow` sparse checkout). +- `bench.yml:557-810`: the `bindings` job (python/browser/node/c). It is skipped when `benches/minigraf_bench/prepared.rs` is absent, which in practice means v3 only. It uses one `binding-ref` input for every repo, `binding-sha.txt`, testbed `binding--ubuntu24-v1`, t-test upper boundary `0.999` and max sample size 30, regression issues with a two-run repeat policy (`:654-790`) whose lookup assumes job name `bench / `, and artifacts kept for 30 days only. +- `.github/bench/bindings.sh:10-25` patches the binding to the measured core and asserts it through `cargo metadata`. Its build/run cases cover only the four existing runtimes. +- `.github/bench/bindings-report.cjs:4-17` hard-codes the runtime list and the 6 (+2 browser) IDs. `:23-43` expects Bencher JSON metrics plus exactly one `ok: ` line per ID. `report.cjs:20-47` (`audit`) rejects missing/extra/duplicate IDs and failed uploads. +- `.github/workflows/bench-report.yml` runs only the Node reporting tests (`bindings-report.test.cjs`, `report.test.cjs`, `history.test.cjs`). Changes under `benchmarks/bindings839/` do not trigger it. +- The #776/#840 snapshot had 427 native IDs. `docs/bench-coverage-v3.md:28-40` records #863's six further routed candidates. Current counts come from actual-run evidence, not a frozen number. + +### Harnesses to reuse (#839, accepted; evidence merged in #852) +- JVM: `minigraf-java` v3 `bench/manual/` (`Benchmark.kt:79-240`, `ResultChecks.kt` + `ResultChecksTest`, `build.sh:7-52`, `build.gradle.kts:6-15`). 14 cases, JSONL output, smoke/benchmark modes, `-Xms512m -Xmx512m -XX:+UseG1GC`, 100 short-op warmups / 5 write warmups, 10 samples of 20 ops (1 op for the two write cases), Java 17 compatible. `build.sh` does its own core patching and refuses an existing `.cargo/config.toml`, which conflicts with `bindings.sh`'s patch step. One of them has to own the patch. +- WASI: core v3 `benchmarks/bindings839/wasi.py` (+ `validate.py`, `test_wasi.py`, `test_guardrails.py`, `README.md:8-35`). 3 REPL cases, Wasmtime with `-C cache=n -C parallel-compilation=n`, no preopened dir (in-memory), 3 warmups, 10 samples, 20 ops per query sample. It refuses Python `-O`, and `--core-revision` is only a label. `.github/workflows/wasm.yml:105-128` already pins Wasmtime `v49.0.2` (its test flags preopen a dir and must not be copied into timing). +- `benchmarks/bindings839/validate.py:11-49` is a strict validator for the 14-case and 3-case sets: full sets only, rejects smoke. Partial mobile activation needs an equally strict selection contract, not a loosened allowlist. +- Swift/iOS: `minigraf-swift` v3 `bench/device-farm/`. `Tests/BenchmarkTests.swift:152-181` rejects simulator timing. `build.sh:27-61` builds device + arm64-simulator libraries but records only the device library hash/ABI. `:70-81` auto-picks a simulator (not pinned). `.github/workflows/bench-839-build.yml` builds the IPA/XCTest. +- Android: `minigraf-android` v3 `bench/device-farm/`. It is ARM64-only (`build.sh:32-65`, `build.gradle.kts:16-32` `arm64-v8a`), non-debuggable Release. `BenchmarkTest.kt:88-134` defaults to smoke, and `-e mode benchmark` times. It records model/fingerprint/ABIs but has no first-class physical-versus-emulated identity. +- Physical baselines: `docs/benchmarks/839-bindings/results.md` (v3). Pixel 7 / Android 14 and iPhone 15 (`iPhone15,4`) / iOS 18.5, core `33e5dffa…`, AWS Device Farm public devices (a model/OS pin, not a reserved unit), with visible spread. These are historical and **not** the qualification reference (see §6). +- Status drift: `docs/benchmarks/839-bindings/README.md:3-4`, `dispositions.md:34-48` and `docs/BINDING_BENCHMARKS.md:79-102` still say "pending sign-off". #839 accepted both maintainer roles for evidence revision `10ff8003…` (merged in #852 as `cdad3e72…`). +- #840 completion record and tooling: `docs/benchmarks/840-regression/completion.md:14-129` (fresh independent runs, inherited main observations excluded, isolated notification scenarios, checksummed archives that reproduce byte-for-byte). This is the pattern for the evidence work here. + +### Environment facts checked during planning +- `minigraf`, `minigraf-swift`, `minigraf-android`, `minigraf-java` are public repos, so hosted `macos-15` and `ubuntu-24.04` minutes are free. Hosted macOS concurrency/queue time is still a limit to document. +- The local AWS CLI has a `minigraf-devicefarm` profile. Device Farm work uses it, and no account identifiers go into public artifacts. + +## 4. Acceptance Criteria + +### Part A — JVM and WASI nightly lanes + +**Common lane contract.** AC3 and AC5–AC9 apply to **every added lane**: JVM, WASI, and each mobile configuration activated under AC14. Read "binding SHA" as the binding repository SHA wherever the lane has one (JVM, Swift, Android), and read the suite names in AC9 as each lane's own suite (`binding_jvm`, `binding_wasi`, and each mobile lane's suite). Each lane has a pinned runtime configuration, records the identity actually executed, and has its own configuration-specific and experimental history. Runtime-specific details are in AC4 (JVM/WASI) and AC10 (mobile). + +1. **Workflow on both branches.** The JVM and WASI lanes are in `bench.yml` on `main` and `v3`. After both PRs merge, `.github/workflows/bench.yml`, `.github/workflows/bench-report.yml` and `.github/bench/` are byte-identical across the two branches. Main-branch (v2) runs skip the new lanes the way they skip the existing binding lanes: no upload, and a step-summary note. A v3 run where a required harness/artifact is missing fails; it does not report success with nothing measured. +2. **Shortlist.** The JVM lane measures exactly the 14 IDs `binding/jvm/{prepared_execute/1k, cursor_batch_1/1k, cursor_batch_256/1k, set_roundtrip/2, predicate/1k, aggregate/1k, prepare/point, register_predicate/first, register_aggregate/first, fact_log_batch_256/1k, options_error/negative_cache, options_readonly_open/1k, file_transact/1k, log_writer_append_finish/1k}`. The WASI lane measures exactly `binding/wasi/{repl_startup_empty, repl_script/1k, repl_query_roundtrip/1k}`. Timing boundaries are unchanged from #839 and documented in `docs/BINDING_BENCHMARKS.md`. The existing native routes and the 26 existing binding IDs are unchanged. +3. **Measured-core provenance.** Each run of an added lane records the run's once-resolved core SHA, workflow SHA, binding SHA (JVM), harness/driver SHA-256, and the SHA-256 of the executed artifact (JVM native library as loaded or packaged; `minigraf.wasm`). The JVM build is asserted to use the measured core checkout, at least as strictly as `bindings.sh`'s metadata check. A binding's release pin or a supplied revision label cannot satisfy this. +4. **Runtime contracts.** JVM runs on a pinned JDK release (distribution and full version, not only a major version like `jvmToolchain(17)`), records the JDK vendor/version and the JVM flags actually used, and keeps the #839 heap/GC/warmup/sample policy. WASI uses a Wasmtime version pinned by version **and** binary checksum, cache and parallel compilation disabled, no preopened directory, and the #839 warmup/sample policy. Lane docs state the startup-compilation and pipe/framing overheads. Neither lane claims engine-only latency. +5. **Exact results.** Every timed invocation checks the complete expected result (rows/sets/counts/sums, and error code where applicable). A wrong or empty result, nonzero exit, unexpected stderr (WASI) or failed teardown fails the job before upload. Untimed/smoke mode emits no timing data. Each new path has a negative test that shows a wrong-but-plausible result is rejected. +6. **Reporting contract.** The shortlist definition and `bindings-report.test.cjs` cover `jvm` and `wasi`. Metrics are Bencher JSON in nanoseconds. Each metric can be recomputed from the retained raw samples with the documented statistic (median of 10 sample means, matching P16/#839). Selected = executed = uploaded is enforced, with one `ok:` per ID. Missing, extra, duplicate, nonfinite or nonpositive values fail. +7. **Histories.** The new testbeds are `binding-jvm-ubuntu24-v1` and `binding-wasi-ubuntu24-v1` (or equally explicit versioned names), with no inheritance from native or other binding histories. Non-v3 binding refs keep using `binding-experiment/...` history. The #840 head-reset bug does not come back: a main advance does not reset v3 history. +8. **Usable baselines.** After merge, every new ID of an added lane has a finite numeric baseline and upper limit on its v3 testbed. These come from at least 3 independent v3 runs (separate workflow runs, at least one of them a real 14:00 UTC scheduled event), with t-test upper boundary 0.999 and max sample size 30 confirmed through the Bencher API. Inherited main data, re-uploads and #839 samples do not count. +9. **Notification.** For each added lane's suite, isolated live scenarios (the #840 pattern, without polluting production histories) show three things: a repeated exact-ID alert on the same branch/suite opens/comments the regression issue; an upload failure notifies immediately; a failed prior-log lookup warns with an empty prior set. Synthetic resources are removed afterwards. + +### Part B — iOS simulator / Android emulator qualification + +10. **Configurations pinned.** The spec's qualification report names, for each platform, the physical reference and the candidate CI configuration, and records them on every run. Physical references: iPhone 15 / iOS 18.5 and Pixel 7 / Android 14 on Device Farm (same models as #839; if unavailable, the closest available model/OS, documented, before any timing). Candidate CI: hosted macOS runner image + Xcode + simulator runtime + device type; hosted Linux runner + emulator API level + system image + ABI + acceleration. The Generator must verify empirically that the Android configuration runs with hardware acceleration and must disclose any guest/host ABI translation. Simulator/emulator runs record their own artifact hash, not the device library's. +11. **Matched builds.** Physical and simulated runs for one comparison use the same core, binding and harness revisions, Release builds, and identical cases and timing boundaries. The 14-case mobile suite is the candidate set. Any case that cannot run identically on both is reported as inconclusive with the reason. +12. **Pre-registered gate (per case, per platform configuration).** + - **Datasets.** For each platform configuration (physical, and each candidate simulator/emulator), collect three sets of independent runs (fresh app/process each): baseline **B1**, a second baseline **B2**, and slowdown variant **V**. Each set has **N runs, N ≥ 5**. N, the run order (for example interleaved B1/V/B2 batches) and the retry rule are recorded in the protocol **before timing**. A run's value is the median of its 10 sample means. + - **Set membership is fixed.** A retry is allowed only for a run that produced no valid stream (provider/infrastructure failure, validation failure), up to a retry cap declared before timing. Every attempt is retained with its reason. A valid run is never replaced or discarded, whatever its value. + - **Closeness** uses B1: median(sim B1 run values) ÷ median(physical B1 run values) within **[0.67, 1.5]**. + - **Repeatability** uses B1: CV = **sample** standard deviation (n − 1 denominator) ÷ arithmetic mean of the run values. Sim CV **≤ 15%** and sim CV **≤ 2 ×** physical CV. + - **Regression signal.** The slowdown variant does real extra work on the timed path, keeps correct answers, is built identically for both platforms, and has its expected magnitude stated before timing. On each platform: Δ = median(V run values) ÷ median(B1 run values) − 1 (the fractional increase, not the multiplier). Detection = one-sided Welch t-test of V > B1 on run values, **p < 0.001**. Pass requires detection on both platforms and |Δ_sim − Δ_phys| ≤ **0.5 × Δ_phys**. + - **A/A** compares B1 with B2 on each platform with the same Welch test. Pass requires p ≥ 0.001 in **both** one-sided directions, on both platforms. + - **Degenerate data.** Zero variance in a set, an undefined test statistic, a nonfinite or nonpositive value, or Δ_phys ≤ 0 makes the case **inconclusive** for that configuration. It never passes. + - The Welch test here is a two-group qualification test. It is a different procedure from Bencher's production `t_test` threshold (a prediction interval for a new value against history, upper boundary 0.999, max sample size 30). Equal nominal tail probabilities do not make their decisions equivalent, and neither setting changes the other. + - **Outcome:** pass closeness, repeatability, regression signal and A/A → **qualified**. Valid data that fails any of them → **rejected**. Too few valid runs after the declared retries, an unmatched build, an unavailable device, a provider failure or degenerate data → **inconclusive**, never reported as rejected or qualified. + - Each case is reported under its category (compute/callback, allocation/conversion, storage/startup), with the mapping stated in the report. The decision is per case; a category summary cannot override a case result. + - These numbers are fixed by this spec before any qualification timing. They may not be changed after seeing results without a new user decision recorded in the report. +13. **Qualification report.** `docs/benchmarks/858-mobile-qualification/` (or equivalent) contains: the protocol (AC12, copied before timing), every raw sample from every run (physical and simulated, including failed/aborted runs with reasons), build identities and hashes, per-case tables (physical and simulated distributions, ratio, CVs, A/B and A/A results, outcome), category summaries, and a reproduction script that recomputes every decision from the raw data. Negative and boundary controls show the decision script: rejects a fabricated "close" dataset that fails each criterion in turn; applies the sample-CV convention at the 15% boundary (for example run values 80, 90, 100, 110, 120 have sample CV 15.81% and fail); uses fractional Δ, not the multiplier; and returns inconclusive for zero-variance, nonfinite and too-few-run inputs. +14. **Conditional activation.** Each platform configuration with at least one qualified case gets a nightly lane for exactly its qualified cases. Each lane has its own testbed (named to say simulator/emulator and its configuration, e.g. `binding-swift-ios-sim--v1`, `binding-android-emu---v1`), meets the common lane contract (AC3, AC5–AC9), and its docs say it is a qualified simulator/emulator lane, not physical-device timing. Rejected and inconclusive cases stay excluded and are listed with their evidence. Where a platform has no qualified case, the report contains a costed physical-device-lane proposal (not implemented). Physical mode stays runnable in both harnesses, and the simulator guard in Swift is replaced by an explicit platform identity, not just deleted. +14a. **Part B completion rule.** Part B, and therefore #858, is complete only when, for **both** iOS and Android, the matched B1/B2/V datasets of AC12 were actually collected on the physical reference and at least one candidate CI configuration, and every case has a decision computed from them. An outcome where the comparison was never performed does **not** complete Part B. Examples: Device Farm never produced a valid physical series, the simulator/emulator never ran, or every case is inconclusive only for those reasons. Such an outcome is reported as unfinished work with its blocker, and #858 stays open. A completed study may reject every case, and individual cases may stay inconclusive for an evidenced, case-specific reason (for example an API that cannot run identically on both). No mobile lane is required for success. + +### Shared + +15. **Durable evidence.** Raw samples, reports, Bencher API evidence (baselines/limits/models), runtime/runner metadata and the qualification data are kept beyond Actions retention in a checksummed archive (committed under `docs/benchmarks/858-*/`, or committed hashes plus an external archive location). A fresh extraction reproduces the derived reports byte-for-byte. No private host, account or device-farm identifiers. Task-owned scratch (local, bench-machine, and temporary CI refs/branches) is removed only after the archive has been verified. Historical runs, production Bencher histories, Device Farm projects/runs and retained source evidence are never deleted as cleanup. +16. **Operational limits documented.** Measured (not estimated) per-lane setup/build/measure durations, hosted runner availability/queueing observed, Device Farm minutes used, and repeatability limits are in the docs. Estimates are labelled as estimates. +17. **Docs in sync.** `docs/BINDING_BENCHMARKS.md`, `docs/bench-coverage-v3.md`, `docs/BENCHMARKS.md` (v3 content; main's v2 file only if the main companion changes something user-visible), the #839 status overlays (accepted; history kept), `CHANGELOG.md` v3 Unreleased, `ROADMAP.md`, and the #858 status statements on v3 in `CLAUDE.md` (line 19), `README.md` (line 43) and `docs/TEST_COVERAGE.md` (line 103), which are updated to the actual enabled lanes and exclusions even though Rust test counts stay unchanged. #840's historical inventory stays as historical evidence. The docs site (`../minigraf-docs`) is checked; any benchmark-coverage text there is updated with `@since v3.0.0`. The approved spec is copied to `docs/superpowers/specs/2026-11-18-binding-nightlies-design.md`. +18. **Nothing else changes.** No change to `src/`, public API, core `Cargo.toml` dependencies, file format or binary size. The existing Node reporting tests, `test_wasi.py`, `test_guardrails.py`, JVM/Android `ResultChecksTest` and the Swift oracle test pass. `bench-report.yml` also runs whenever the paths the new lanes depend on change (including `benchmarks/bindings839/**` or its successor). + +## 5. Edge Cases & Boundaries + +- **One `binding-ref` for all repos:** a JVM-only experiment branch does not exist in the Python/browser/Node/C repos. An experiment on one binding must not break the other lanes, and the SHA actually used is recorded either way. +- **Premature success:** samples printed before a later check, teardown or exit failure must not count. A provider "PASS" (Device Farm, Gradle, XCTest) is not success without the validated complete stream. +- **iOS log recovery:** #839 needed the JSONL from the automation log (artifact attachment failed). Recovered evidence must be exactly one complete stream: no duplicated application/syslog copies, no truncation, no smoke/benchmark mixing. +- **Stale outputs:** a fresh output location per invocation, so an old timing file can never satisfy a new run. +- **JIT/ART/thermal:** the configured warmups do not prove stationarity. Record conditions and spread; do not normalize away a closeness failure (#863's runner-speed normalization is not allowed for AC12). +- **Platform identity:** a simulator device name is not hardware identity. Emulator host ISA, guest ABI, image/API level and acceleration are separate recorded facts. A run that cannot prove which platform it ran on is invalid. +- **Same SHA, independent runs:** repeated runs of one revision are fine. Independence comes from separate processes/workflow runs, not distinct commits. +- **Configuration drift:** a changed runner image, Xcode, system image, ABI, JDK, Wasmtime or timing boundary needs a new testbed version, and for mobile, requalification. Do not merge histories because suffixes match. +- **Hosted runner variability:** a macOS queue timeout or runner shortage is "unavailable", not a performance failure. +- **Injected slowdown:** it must do real, representative work on the timed path (not a sleep or a timestamp offset) and keep correct answers. It must never be in a production nightly build. Synthetic latency is fine only for notification plumbing tests (AC9), never for qualification or production baselines. +- **Main schedule:** the 02:00 UTC main run must keep its current coverage and timing; the new lanes skip cleanly. + +## 6. Key Decisions + +- **One mission for A and B** (user). #858 closes only when both are done. PRs are split for review (§8). +- **Fresh matched physical runs on Device Farm** (user), not a comparison with the #839 data, because #839 is at an old revision and has no slowdown A/B. #839 stays historical evidence. +- **Thresholds fixed now** (user approved the draft, AC12). Pre-registration exists to stop choosing thresholds after seeing the data. +- **All 14 JVM + 3 WASI cases** (user). The eight lifecycle/storage JVM cases are the costs #839 added beyond P16. Lane cost is measured under AC16, not assumed. +- **v3-only measurement** (Claude + Codex), the same as the existing binding lanes. The main PR only carries the identical workflow. +- **Integrate with the existing nightly system**, not a separate one. The new lanes reuse its provenance, reporting, history and notification behaviour, and its four-runtime assumptions are resolved rather than bypassed. The job layout (a generalized matrix or sibling jobs, for example for WASI with no binding repo, or for a macOS-hosted iOS lane) is the Generator's choice. +- **Separate qualification from activation.** Experimental simulator/emulator timing during qualification goes only to experiment histories (or none), never to a production testbed. +- **Historical docs keep their wording**, with current-status overlays (#839 accepted, #840 complete). + +## 7. Out of Scope + +- Engine optimizations, API/format/dependency/size changes. +- Reopening #839 sign-offs or #776's native audit. +- A macOS desktop Swift lane (an iOS simulator on a macOS host is in scope; desktop Swift is not). +- Implementing a physical-device nightly lane (only a costed proposal when qualification fails). +- An unqualified simulator/emulator "trend monitor". +- RSS/memory bounds, scaling sweeps, published-package certification. +- v2.x JVM/WASI measurement; releases/version bumps. + +## 8. Possible Directions & Ideas (non-binding) + +- **PR sequence:** (1) core v3 + main companion: generalize the binding job/reporting for JVM + WASI, plus the Java v3 nightly adapter; (2) collect fresh baselines and run the notification scenarios; (3) Swift/Android qualification modes (platform identity, simulator/emulator builds, slowdown variant) in the binding repos; (4) the qualification runs and report; (5) activation PRs for qualified cases only, with baselines/notification for them; (6) docs + evidence archive. +- **Job layout:** extending the existing `bindings` matrix fits JVM well. WASI (no binding repo) and an iOS simulator lane (macOS host) may fit better as sibling jobs that share the same reporting helpers. +- **Suggested binding:** a thin `bench/nightly.*` entry in `minigraf-java` that reuses `Benchmark.kt` cases and emits the existing contract, rather than changing the #839 manual mode. +- **WASI:** a small adapter from `samples.jsonl` to Bencher JSON plus `ok:` lines, keeping `wasi.py` and `validate.py` the source of truth. +- **Android emulator:** the x86_64 image on `ubuntu-24.04` with KVM needs an x86_64 build of the harness; an arm64 image on an arm runner may avoid that if acceleration works. Check both before committing to one. +- **Injected slowdown idea:** a harness-only compile flag that repeats the timed operation's real work a stated number of extra times, for example 1 extra time per 2 invocations, about 1.5×. Same flag on device and simulator. +- **Prior evidence tooling:** `docs/benchmarks/840-regression/` collectors/verifiers (exact API reconciliation, lineage, negative controls, archive reproduction). +- **Heavy local work** (if any) goes on the dedicated bench machine, never the laptop. Mobile builds run on hosted CI, and timing runs on Device Farm or hosted runners. + +--- + +## Changes from Claude-02 (response to Codex-02) + +All Codex-02 findings accepted after rechecking the sources: +- **H1:** job topology moved from a decision to a non-binding suggestion (§6, §8). +- **H2:** new AC14a, the Part B completion rule. Non-execution is unfinished work, not completion. +- **M1:** AC12 now defines datasets B1/B2/V, fixed set membership and retries, sample CV, fractional Δ, the A/A rule in both directions, and degenerate data → inconclusive. The approved numbers are unchanged, and the Welch test is distinguished from Bencher's `t_test`. AC13 adds boundary controls (CV example checked: run values 80–120 give sample CV 15.81% against population 14.14%). +- **M2:** common lane contract added at the top of §4, so AC3 and AC5–AC9 cover every added lane, including activated mobile lanes. The JDK release is pinned (AC4). +- **M3:** AC17 names the #858 status lines in v3 `CLAUDE.md:19`, `README.md:43` and `docs/TEST_COVERAGE.md:103` (verified with `git show origin/v3`). +- **M4:** AC15 requires scratch cleanup only after the archive is verified. +- **Low:** 427 is worded as the snapshot; "cost is small" removed. From c7bdf985798efe635eec9e544b136a50405e0c91 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Wed, 18 Nov 2026 16:07:01 +0530 Subject: [PATCH 2/2] ci(bench): validate binding lanes from the measured checkout (#858) Run the binding validator and hash the harness from the measured checkout so the workflow is identical on main, where benchmarks/bindings839 is absent. Skip the Python harness tests when it is absent. The regression repeat lookup now searches back for the latest prior run with matching production history. Co-Authored-By: Claude Sonnet 5.5 --- .github/bench/bindings-report.cjs | 10 ++- .github/bench/bindings-report.test.cjs | 46 ++++++++++- .github/bench/bindings-workflow.test.cjs | 99 ++++++++++++++++++++++-- .github/workflows/bench-report.yml | 1 + .github/workflows/bench.yml | 21 ++--- CHANGELOG.md | 2 +- CLAUDE.md | 3 +- README.md | 3 +- ROADMAP.md | 2 +- docs/BENCHMARKS.md | 3 +- docs/BINDING_BENCHMARKS.md | 35 +++++++-- docs/TEST_COVERAGE.md | 5 +- docs/bench-coverage-v3.md | 4 +- 13 files changed, 195 insertions(+), 39 deletions(-) diff --git a/.github/bench/bindings-report.cjs b/.github/bench/bindings-report.cjs index bdcb434069..1b503ac850 100644 --- a/.github/bench/bindings-report.cjs +++ b/.github/bench/bindings-report.cjs @@ -60,9 +60,11 @@ function verifyEvidence(binding, metrics, directory, identity) { 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'] : ['nightly.py', 'wasi.py', 'validate.py']; if (!harness || required.some(file => !harness[file])) throw new Error('missing harness hashes'); - // WASI is built and driven by the measured checkout; reporting follows the - // workflow checkout. Resolve the measured source explicitly, not __dirname. - const harnessRoot = binding === 'jvm' ? directory : path.resolve('benchmarks/bindings839'); + // Python validation for both lanes belongs to the measured core, even when + // these reporting helpers come from main (which has no bindings839 harness). + // WASI provenance must hash the same validator that we execute below. + const measuredHarness = path.resolve('benchmarks/bindings839'); + const harnessRoot = binding === 'jvm' ? directory : measuredHarness; for (const [file, digest] of Object.entries(harness)) { if (path.isAbsolute(file) || file.split('/').includes('..') || sha(fs.readFileSync(path.join(harnessRoot, file))) !== digest) throw new Error('wrong harness hash'); @@ -82,7 +84,7 @@ function verifyEvidence(binding, metrics, directory, identity) { const rawPath = path.join(directory, 'benchmark/samples.jsonl'); // Keep #839's validator authoritative for sample IDs/counts/iterations and // the complete/check markers, including rejection of smoke streams. - const validation = spawnSync('python3', ['-I', path.join(__dirname, '../../benchmarks/bindings839/validate.py'), binding, rawPath], { encoding: 'utf8' }); + const validation = spawnSync('python3', ['-I', path.join(measuredHarness, 'validate.py'), binding, rawPath], { encoding: 'utf8' }); if (validation.status !== 0) throw new Error(`invalid raw samples: ${validation.stderr || validation.error}`); const records = read('benchmark/samples.jsonl').trim().split('\n').map(parseMetrics); const environment = records.find(r => r.type === 'environment'); diff --git a/.github/bench/bindings-report.test.cjs b/.github/bench/bindings-report.test.cjs index 063f002244..c24a645a43 100644 --- a/.github/bench/bindings-report.test.cjs +++ b/.github/bench/bindings-report.test.cjs @@ -58,7 +58,7 @@ test('JVM lifecycle/storage and WASI REPL shortlists are exact', () => { ].map(id => `binding/jvm/${id}`)); }); -const { verifyEvidence, parseMetrics } = require('./bindings-report.cjs'); +const { parseMetrics } = require('./bindings-report.cjs'); const crypto = require('node:crypto'); const sha = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); test('duplicate JSON metric keys fail instead of being overwritten', () => { @@ -70,7 +70,30 @@ test('duplicate JSON metric keys fail instead of being overwritten', () => { for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, provenance and artifact bytes before upload`, () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-evidence-')); + const originalCwd = process.cwd(); try { + // Model the default-branch workflow separately from the measured v3 tree. + // This fixture tests validator selection; Python's own suite tests #839's + // sample contract. It also keeps these shared Node tests runnable on main. + const measuredHarness = path.join(dir, 'benchmarks/bindings839'); + const workflowHarness = path.join(dir, '.bench-workflow/benchmarks/bindings839'); + const workflowHelpers = path.join(dir, '.bench-workflow/.github/bench'); + fs.mkdirSync(measuredHarness, {recursive: true}); + fs.mkdirSync(workflowHarness, {recursive: true}); + fs.mkdirSync(workflowHelpers, {recursive: true}); + for (const file of ['bindings-report.cjs', 'report.cjs']) { + fs.copyFileSync(path.join(__dirname, file), path.join(workflowHelpers, file)); + } + fs.writeFileSync(path.join(measuredHarness, 'validate.py'), `import json, pathlib, sys +pathlib.Path('measured-validator-ran').write_text(__file__) +records = [json.loads(line) for line in pathlib.Path(sys.argv[2]).read_text().splitlines()] +if records[-1]['type'] != 'complete': + sys.exit('missing complete marker') +`); + for (const file of ['nightly.py', 'wasi.py']) fs.writeFileSync(path.join(measuredHarness, file), file); + fs.writeFileSync(path.join(workflowHarness, 'validate.py'), "raise SystemExit('tampered workflow validator ran')\n"); + const { verifyEvidence } = require(path.join(workflowHelpers, 'bindings-report.cjs')); + process.chdir(dir); fs.mkdirSync(path.join(dir, 'benchmark')); const coreSha = 'a'.repeat(40), workflowSha = 'b'.repeat(40), bindingSha = binding === 'jvm' ? 'c'.repeat(40) : null; const ids = expected(binding); @@ -102,7 +125,7 @@ for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, prov ? Object.fromEntries(['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts', 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'].map(p => [p, sha(p)])) - : Object.fromEntries(['nightly.py', 'wasi.py', 'validate.py'].map(p => [p, sha(fs.readFileSync(path.join(__dirname, '../../benchmarks/bindings839', p)))])); + : Object.fromEntries(['nightly.py', 'wasi.py', 'validate.py'].map(p => [p, sha(fs.readFileSync(path.join(measuredHarness, p)))])); if (binding === 'jvm') for (const file of Object.keys(harnessFiles)) { fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true}); fs.writeFileSync(path.join(dir, file), file); @@ -117,6 +140,10 @@ for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, prov fs.writeFileSync(file, JSON.stringify(provenance)); const identity = { coreSha, workflowSha, bindingSha }; assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha); + assert.equal(fs.readFileSync('measured-validator-ran', 'utf8'), path.join(measuredHarness, 'validate.py')); + // main has no workflow harness at all; validation still uses measured v3. + fs.rmSync(workflowHarness, {recursive: true}); + assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha); metrics[ids[0]].latency.value = 6; assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /median/); metrics[ids[0]].latency.value = 5.5; @@ -135,5 +162,18 @@ for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, prov assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /environment/); fs.writeFileSync(raw, records.slice(0, -1).map(r => JSON.stringify(r)).join('\n')); assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /samples/); - } finally { fs.rmSync(dir, { recursive: true, force: true }); } + fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n')); + // A workflow copy cannot rescue a missing measured validator on v3. + fs.mkdirSync(workflowHarness, {recursive: true}); + fs.writeFileSync(path.join(workflowHarness, 'validate.py'), '# accepts everything\n'); + if (binding === 'wasi') { + fs.appendFileSync(path.join(measuredHarness, 'validate.py'), '# changed since measurement\n'); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /harness hash/); + } + fs.unlinkSync(path.join(measuredHarness, 'validate.py')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /validate.py|raw samples/); + } finally { + process.chdir(originalCwd); + fs.rmSync(dir, { recursive: true, force: true }); + } }); diff --git a/.github/bench/bindings-workflow.test.cjs b/.github/bench/bindings-workflow.test.cjs index e3d1575af4..fc6f6081c8 100644 --- a/.github/bench/bindings-workflow.test.cjs +++ b/.github/bench/bindings-workflow.test.cjs @@ -30,13 +30,43 @@ test('new rows use the actual binding suite job names', () => { assert.match(bindingJob, /binding: wasi/); assert.match(workflow, /jvm-ref:/); }); -test('main skips with a summary; a missing v3 harness fails', () => { - const main = shell('Check v3 binding suite availability', {BENCH_REF: 'main', BINDING: 'wasi'}); - assert.equal(main.status, 0, main.stderr); - assert.match(main.summary, /Skipping/); - assert.doesNotMatch(main.output, /enabled=true/); - const missing = shell('Check v3 binding suite availability', {BENCH_REF: 'v3', BINDING: 'wasi'}); - assert.notEqual(missing.status, 0); +test('main skips; either v3 lane fails when its measured validator is absent', () => { + for (const binding of ['jvm', 'wasi']) { + const main = shell('Check v3 binding suite availability', {BENCH_REF: 'main', BINDING: binding}); + assert.equal(main.status, 0, main.stderr); + assert.match(main.summary, /Skipping/); + assert.doesNotMatch(main.output, /enabled=true/); + for (const missing of [true, false]) { + const result = shell('Check v3 binding suite availability', {BENCH_REF: 'v3', BINDING: binding}, cwd => { + for (const file of ['benches/minigraf_bench/prepared.rs', 'benchmarks/bindings839/nightly.py', + 'benchmarks/bindings839/wasi.py', ...(missing ? [] : ['benchmarks/bindings839/validate.py'])]) { + fs.mkdirSync(path.dirname(path.join(cwd, file)), {recursive: true}); + fs.writeFileSync(path.join(cwd, file), 'fixture'); + } + }); + assert.equal(result.status === 0, !missing, result.stderr); + assert.equal(result.output.includes('enabled=true'), !missing); + } + } +}); +test('workflow checkout needs only shared helpers, never the measured Python harness', () => { + const checkout = bindingJob.split('ref: ${{ github.workflow_sha }}')[1].split('persist-credentials: false')[0]; + assert.match(checkout, /\/\.github\/bench\//); + assert.doesNotMatch(checkout, /bindings839/); +}); +test('reporting CI skips absent Python tests on main and runs them on v3', () => { + const reporting = fs.readFileSync(path.join(__dirname, '../workflows/bench-report.yml'), 'utf8'); + assert.equal((reporting.match(/'benchmarks\/bindings839\/\*\*'/g) || []).length, 2, 'keep both path triggers'); + const step = reporting.split(' - ').find(s => s.includes('python3 -I -m unittest')); + assert.match(step, /if:.*hashFiles\('benchmarks\/bindings839\/test_\*\.py'\) != ''/); + const expression = step.match(/if: \$\{\{ (.+) \}\}/)[1]; + for (const [hash, shouldRun] of [['', false], ['test-files-hash', true]]) { + const hashFiles = pattern => { + assert.equal(pattern, 'benchmarks/bindings839/test_*.py'); + return hash; + }; + assert.equal(new Function('hashFiles', `return ${expression}`)(hashFiles), shouldRun); + } }); for (const binding of ['jvm', 'wasi']) test(`${binding} uploads only to its own testbed without inherited history`, () => { for (const history of ['v3', 'binding-experiment/experiment/v3']) { @@ -148,3 +178,58 @@ test('JVM run refuses stale output and smoke that emits metrics', () => { } finally { fs.rmSync(cwd, {recursive: true, force: true}); } } }); + +for (const binding of ['jvm', 'wasi']) test(`${binding} finds the most recent production history within a bounded search`, async () => { + const original = {...process.env}; + const suite = `binding_${binding}`, id = `binding/${binding}/example`; + const alert = `- ${id} (Latency): https://bencher.dev/console/projects/minigraf/alerts/123`; + const production = `History branch: v3; testbed: binding-${binding}-ubuntu24-v1;`; + try { + Object.assign(process.env, {SUITE: suite, BENCH_REF: 'v3', BENCH_SHA: 'a'.repeat(40)}); + for (const scenario of ['repeat-after-experiment', 'clean-production', 'wrong-testbed', 'no-suite', 'lookup-failure', 'exhausted']) { + const calls = [], warnings = [], visited = []; + const github = {rest: {actions: { + listWorkflowRuns: async args => { + assert.equal(args.status, 'completed'); + assert.equal(args.per_page, 20); + return {data: {workflow_runs: [ + {id: 100, display_title: 'Nightly Benchmarks (v3)'}, // current + {id: 99, display_title: 'Nightly Benchmarks (main)'}, + ...Array.from({length: 18}, (_, i) => ({id: 98 - i, display_title: 'Nightly Benchmarks (v3)', html_url: `prior-${98 - i}`})), + ]}}; + }, + listJobsForWorkflowRun: async args => { + visited.push(args.run_id); + return {data: {jobs: scenario === 'no-suite' && args.run_id === 98 ? [] : [{id: args.run_id, name: `bench / ${suite}`}]}}; + }, + downloadJobLogsForWorkflowRun: async ({job_id}) => { + if (scenario === 'lookup-failure' && job_id === 97) throw new Error('unavailable logs'); + if (job_id === 98 || scenario === 'exhausted') { + const history = scenario === 'wrong-testbed' ? 'History branch: v3; testbed: binding-python-ubuntu24-v1;' : + `History branch: binding-experiment/v3/java-only; testbed: binding-${binding}-ubuntu24-v1;`; + return {data: `${history}\n${alert}`}; + } + return {data: `${production}\n${scenario === 'clean-production' && job_id === 97 ? '' : alert}`}; + }, + }, issues: { + listForRepo: async () => ({data: []}), + create: async args => { calls.push(args); return {data: {number: 1}}; }, + }}}; + const summary = {addHeading() {return this;}, addRaw() {return this;}, addList() {return this;}, addBreak() {return this;}, async write() {}}; + await new AsyncFunction('require', 'github', 'context', 'core', 'console', body('Report regression', 'script'))( + () => ({readFileSync: () => alert}), github, + {repo: {owner: 'test', repo: 'test'}, runId: 100, serverUrl: 'https://example.com'}, + {warning: message => warnings.push(message), summary}, {log() {}}); + const reports = ['repeat-after-experiment', 'wrong-testbed', 'no-suite'].includes(scenario); + assert.equal(calls.length, reports ? 1 : 0, scenario); + if (reports) assert.match(calls[0].body, /Previous run:\*\* prior-97/); + assert.deepEqual(visited, scenario === 'exhausted' ? Array.from({length: 18}, (_, i) => 98 - i) : [98, 97]); + if (scenario === 'lookup-failure') assert.ok(warnings.some(w => w.includes('Could not read'))); + if (scenario === 'exhausted') assert.ok(warnings.some(w => w.includes('No matching production history'))); + } + } finally { + for (const key of ['SUITE', 'BENCH_REF', 'BENCH_SHA']) { + if (original[key] === undefined) delete process.env[key]; else process.env[key] = original[key]; + } + } +}); diff --git a/.github/workflows/bench-report.yml b/.github/workflows/bench-report.yml index 7e59a36535..b0eafb2046 100644 --- a/.github/workflows/bench-report.yml +++ b/.github/workflows/bench-report.yml @@ -21,3 +21,4 @@ jobs: node-version: '24' - run: node --test .github/bench/*.test.cjs - run: python3 -I -m unittest discover -s benchmarks/bindings839 -p 'test_*.py' + if: ${{ hashFiles('benchmarks/bindings839/test_*.py') != '' }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a0cd4568dd..e9c0841b43 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -591,10 +591,12 @@ jobs: # A missing harness on v3 is an error, never a successful empty run. if [ "$BENCH_REF" = v3 ] || [ -f src/query/engine/mod.rs ]; then test -f benches/minigraf_bench/prepared.rs + if [ "$BINDING" = jvm ] || [ "$BINDING" = wasi ]; then + test -f benchmarks/bindings839/validate.py + fi if [ "$BINDING" = wasi ]; then test -f benchmarks/bindings839/nightly.py test -f benchmarks/bindings839/wasi.py - test -f benchmarks/bindings839/validate.py fi echo 'enabled=true' >> "$GITHUB_OUTPUT" else @@ -608,7 +610,6 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | /.github/bench/ - /benchmarks/bindings839/ persist-credentials: false - uses: actions/checkout@v7 if: steps.source.outputs.enabled == 'true' && matrix.binding != 'wasi' @@ -709,9 +710,9 @@ jobs: `**Run:** ${runUrl}`, ].join('\n'); } else { - // Find this suite's job in the previous completed run of the same - // branch (by run name, see run-name) and read its alerts. Runs - // from before the branch was in the run name count as main. + // Search at most 20 completed runs for this suite's most recent + // production history. A JVM-only experiment shares the v3 run + // title but must not hide an earlier production repetition. let previous = new Set(); let previousUrl = null; try { @@ -726,10 +727,8 @@ jobs: const m = /\(([^)]+)\)$/.exec(r.display_title || ''); return m ? m[1] : 'main'; }; - const prevRun = runs.workflow_runs.find( - r => r.id !== context.runId && runRef(r) === ref); - if (prevRun) { - previousUrl = prevRun.html_url; + for (const prevRun of runs.workflow_runs) { + if (prevRun.id >= context.runId || runRef(prevRun) !== ref) continue; const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({ owner, repo, run_id: prevRun.id, per_page: 100, }); @@ -744,9 +743,13 @@ jobs: const testbed = `binding-${suite.slice('binding_'.length)}-ubuntu24-v1`; if (text.includes(`History branch: ${ref}; testbed: ${testbed};`)) { previous = parseAlerts(text); + previousUrl = prevRun.html_url; + // A clean production run breaks the streak too. + break; } } } + if (!previousUrl) core.warning('No matching production history in the last 20 completed runs; using an empty prior alert set.'); } catch (err) { core.warning(`Could not read the previous run's alerts: ${err.message}`); } diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b569d3ae5..b5353875c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **JVM/WASI nightly workflow support** (#858, core portion): 14 JVM and 3 WASI routes, pinned runtimes, measured-core/artifact provenance, strict raw-sample reporting and independent histories. Java adapter integration, default-branch rollout and live baseline/notification evidence remain pending; mobile lanes remain excluded pending qualification. No runtime, file-format or Rust test inventory change. See [the adapter contract](docs/BINDING_BENCHMARKS.md). +- **JVM/WASI nightly workflow support** (#858, core portion): 14 JVM and 3 WASI routes, pinned runtimes, measured-core/artifact provenance, strict raw-sample reporting and independent histories. The Java adapter exists in `minigraf-java` at `daf7b9c` (`feat/858-nightly-adapter`). Default-branch rollout and live baseline/notification evidence remain pending; mobile lanes remain excluded pending qualification. No runtime, file-format or Rust test inventory change. See [the adapter contract](docs/BINDING_BENCHMARKS.md). - **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import admits an empty blob, a nonempty v8 image with a selectable meta and complete, verified contents (an empty v8 image included), or a supported v7 image that migrates completely, and rejects anything else (such as a torn or zero-filled initial image) before replacing anything; it validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. Exports after interrupted initial creation fill an absent second meta slot with zeros and still propagate failed reads of existing pages. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). diff --git a/CLAUDE.md b/CLAUDE.md index ced0844d9e..856ac9730a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,8 @@ independent v3 runs per ID, preserved history after main advances, actual schedu routes and isolated live notification delivery/cleanup. Fixes #853/#854 are merged; checksummed evidence outlives Actions retention. This changes no database runtime behavior or Rust test counts. Core workflow support for 14 JVM and 3 WASI nightly IDs is implemented for #858 -([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter, default-branch rollout +([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter exists in `minigraf-java` at `daf7b9c` +(`feat/858-nightly-adapter`). Default-branch rollout and live baseline/notification evidence remain pending. Mobile lanes remain excluded pending qualification. Rust test counts are unchanged. diff --git a/README.md b/README.md index 8f8d878684..cef12a2447 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ The [regression-history follow-up](docs/benchmarks/840-regression/completion.md) verifies numeric comparisons for all 427 native and 26 binding IDs, independent v3 history, scheduled main/v3 routes and isolated live notification delivery with cleanup. Raw evidence and offline verifiers are retained in Git. Core workflow support for 14 JVM and 3 WASI nightly IDs is implemented for #858 -([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter, default-branch rollout +([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter exists in `minigraf-java` at `daf7b9c` +(`feat/858-nightly-adapter`). Default-branch rollout and live baseline/notification evidence remain pending. Mobile lanes remain excluded pending qualification; Rust test counts are unchanged. diff --git a/ROADMAP.md b/ROADMAP.md index f175a203af..d17dc3bd85 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -133,7 +133,7 @@ Scope — release-gate testing and documentation: - Benchmarks at 100K–10M facts and a published performance envelope (#394; on `v3`: `examples/perf_envelope.rs`, results in `docs/BENCHMARKS.md`; follow-ups #528–#532, with #532 superseded by #551's removal of repeated committed-fact net-assert; [audit and measurements](docs/benchmarks/532-net-assert-audit.md)) - Acceptance evidence for memory/scaling (#837): named-test and Linux RSS gates, nonlinear/multistratum recursion and custom chunk bounds; [validation status](docs/benchmarks/837-acceptance-gaps.md) - Regression-history/reporting evidence (#840) is complete: [all 453 numeric comparisons, preserved independent v3 history, scheduled routes and live notification/cleanup evidence](docs/benchmarks/840-regression/completion.md). Separate main/v3 fixes #853/#854 are merged; raw evidence is retained beyond artifact expiry. -- Nightly runtime expansion (#858): core workflow support for 14 Java/JVM and 3 WASI IDs is implemented ([contract](docs/BINDING_BENCHMARKS.md)); Java adapter integration, default-branch rollout, live baselines/notifications and durable evidence remain pending. Swift/iOS simulator and Android emulator lanes stay excluded until qualification against physical-device workloads. The platform restriction from #839 applies only to that accepted manual evidence task. +- Nightly runtime expansion (#858): core workflow support for 14 Java/JVM and 3 WASI IDs is implemented ([contract](docs/BINDING_BENCHMARKS.md)); the Java adapter exists in `minigraf-java` at `daf7b9c` (`feat/858-nightly-adapter`). Default-branch rollout, live baselines/notifications and durable evidence remain pending. Swift/iOS simulator and Android emulator lanes stay excluded until qualification against physical-device workloads. The platform restriction from #839 applies only to that accepted manual evidence task. - Manual migration evidence (#838): native v7-to-v8 through 200k facts; Chromium/Firefox/WebKit through 20k facts, with separate read-only/reopen controls and bounded post-migration cursor churn. [Results, frozen fixtures and exclusions](docs/benchmarks/838-migration/README.md); no new nightly route or sustained-pin bound. - Semantic permutations (#842): [correctness-target map and control limits](docs/benchmarks/842-semantic-permutations/README.md) for X07. The exclusion stands; no nightly route or acceptance gate is added. - Failure paths (#843): [correctness-target map](docs/benchmarks/843-failure-correctness/README.md) for X08. The exclusion stands; the heavy WAL replay test gap is #865. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 359c9fc4d3..d41bbdf517 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -7,7 +7,8 @@ This document records reproducible local benchmark snapshots. Completed release- The core workflow defines 14 JVM and 3 WASI routes with separate Ubuntu 24.04 histories, pinned runtimes, exact-result checks, raw sample medians and artifact provenance. See [binding benchmarks](BINDING_BENCHMARKS.md) for timing boundaries -and the Java adapter contract. Java integration, default-branch rollout and live +and the Java adapter contract. The adapter exists in `minigraf-java` at +`daf7b9c` (`feat/858-nightly-adapter`). Default-branch rollout and live baselines/notifications remain pending. No mobile lane is enabled before qualification, and no new runtime timings are claimed by this implementation. diff --git a/docs/BINDING_BENCHMARKS.md b/docs/BINDING_BENCHMARKS.md index b0c6eafe10..99fbbdec1f 100644 --- a/docs/BINDING_BENCHMARKS.md +++ b/docs/BINDING_BENCHMARKS.md @@ -3,9 +3,8 @@ The accepted [#839 manual evidence](benchmarks/839-bindings/README.md) remains historical evidence. #858 adds core workflow support for JVM (14 IDs) and WASI (3 IDs), alongside the existing 26 Python/browser/Node/C IDs. The Java nightly -adapter and identical default-branch CI companion must land before rollout. -The main companion must also carry `benchmarks/bindings839/` reporting/test -dependencies, so its reporting CI can run without the v3 historical archives. +adapter exists in `minigraf-java` at `daf7b9c` on `feat/858-nightly-adapter`; +it and the identical default-branch CI companion must land before rollout. Three independent v3 runs (including a scheduled run), live notification checks, durable evidence and measured lane costs are still pending. This change does not complete #858 or activate a mobile simulator/emulator lane. @@ -14,16 +13,38 @@ The `bindings` matrix in `.github/workflows/bench.yml` measures the once-resolve core SHA. JVM checks out `minigraf-java`; WASI has no binding repository. The 14:00 UTC schedule measures v3; main/v2 checkouts skip with a step-summary note and no uploads. A required harness or artifact missing on v3 fails the job. -Reporting helpers come from `github.workflow_sha`; the WASI driver comes from -the measured core. Both identities are recorded. +Reporting JS/shell helpers come from `github.workflow_sha`. All core harness +Python, including `validate.py` for both JVM and WASI and the WASI driver/adapter, +runs from the measured core checkout. The WASI provenance hashes that same +`validate.py`; a different or missing workflow-checkout copy has no effect. +Both checkout identities are recorded. + +This is the F1 option (b) design: the default-branch sparse checkout needs only +`.github/bench/`, and main does not need `benchmarks/bindings839/`. The shared +`bench-report.yml` keeps its harness path triggers and runs Python unit tests +only when `hashFiles('benchmarks/bindings839/test_*.py')` finds tests. Its Node +tests have self-contained fixtures and run on both branches. This reporting-CI +guard does not apply to measurement: a missing measured v3 validator fails both +JVM and WASI jobs, and missing WASI drivers also fail. Manual dispatch can set `jvm-ref` for a Java-only experiment, leaving every other binding on `binding-ref` (default `v3`). `binding-ref` still selects a common ref for all binding repositories, so use it only when that ref exists in all of them. WASI ignores binding refs and always uses the measured core. Non-v3 core or binding refs use `binding-experiment//` -histories, with no production notifications. An experimental prior job cannot -satisfy the production two-run repeat policy. +histories, with no production notifications. + +For the two-run repeat policy (F2), search the latest 20 completed `bench.yml` +runs on the default branch, newest first, considering only runs older than the +current run with the same measured-branch title. For this suite, skip missing +jobs and logs without the production `History branch: v3; testbed: ;` line, including JVM-only experiments. Stop at the first matching +production log, even if it has no alerts: a clean production run breaks the +streak. Only exact IDs repeated in that log can notify. A failed API/log lookup +stops the search and warns with an empty prior set; exhausting the 20-run window +without a match also warns with an empty set. Upload failures still notify +immediately. This bound can leave an older production run out of reach after +many experiments; it never treats an experiment as production repetition. ## Existing Python/browser/Node/C lanes diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 0c60e6888d..e10b19ed0a 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -101,8 +101,9 @@ Sixteen isolated live notification scenarios verify four actual deliveries and cleanup of all 46 synthetic resources. Fresh extraction reproduces the evidence reports, and removing delivery/comparison evidence is rejected. No Rust tests are added; the inventory above is unchanged. The #858 core workflow adds 14 JVM and 3 WASI routes with raw-sample and -provenance checks ([contract](BINDING_BENCHMARKS.md)). The Java adapter, -default-branch rollout and live baselines/notifications remain pending; mobile +provenance checks ([contract](BINDING_BENCHMARKS.md)). The Java adapter exists +in `minigraf-java` at `daf7b9c` (`feat/858-nightly-adapter`). Default-branch +rollout and live baselines/notifications remain pending; mobile lanes remain excluded pending qualification. No Rust tests are added. **#843 failure-path map**: [138 curated correctness tests](benchmarks/843-failure-correctness/README.md) diff --git a/docs/bench-coverage-v3.md b/docs/bench-coverage-v3.md index 424e1e0aa0..db661e26fe 100644 --- a/docs/bench-coverage-v3.md +++ b/docs/bench-coverage-v3.md @@ -6,8 +6,8 @@ least three independent v3 runs each. Merged fixes #853/#854 preserve the v3 hea after a measured main advance. Actual scheduled routes, isolated live notification delivery and cleanup are verified; complete evidence is archived beyond 30 days. The #858 core workflow now defines 14 JVM and 3 WASI routes -([contract](BINDING_BENCHMARKS.md)); Java adapter integration, default-branch -rollout and live baselines/notifications remain pending. No mobile lane is +([contract](BINDING_BENCHMARKS.md)); the Java adapter exists in `minigraf-java` at +`daf7b9c` (`feat/858-nightly-adapter`). Default-branch rollout and live baselines/notifications remain pending. No mobile lane is enabled before qualification. The original validation snapshot below is retained unchanged; its null baselines and X05 open status describe that earlier snapshot.