Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 104 additions & 5 deletions .github/bench/bindings-report.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
const fs = require('node:fs');
const { audit } = require('./report.cjs');
function expected(binding) {
if (!['python', 'browser', 'node', 'c'].includes(binding)) throw new Error('unknown binding');
if (!['python', 'browser', 'node', 'c', 'jvm', 'wasi'].includes(binding)) throw new Error('unknown binding');
if (binding === 'wasi') return ['repl_startup_empty', 'repl_script/1k', 'repl_query_roundtrip/1k'].map(s => `binding/wasi/${s}`);
const suffixes = ['prepared_execute/1k', 'cursor_batch_1/1k', 'cursor_batch_256/1k', 'set_roundtrip/2', 'predicate/1k', 'aggregate/1k'];
if (binding === 'browser') suffixes.push('indexeddb_write/1k', 'indexeddb_reopen/1k');
if (binding === 'jvm') suffixes.push('prepare/point', 'register_predicate/first', 'register_aggregate/first',
'fact_log_batch_256/1k', 'options_error/negative_cache', 'options_readonly_open/1k',
'file_transact/1k', 'log_writer_append_finish/1k');
return suffixes.map(s => `binding/${binding}/${s}`);
}
function validate(binding, metrics) {
Expand All @@ -16,12 +20,106 @@ function validate(binding, metrics) {
}
return ids;
}
// JSON.parse alone silently overwrites duplicate keys. Parse syntax first, then
// inspect object keys (including escaped spellings) in the valid token stream.
function parseMetrics(text) {
const result = JSON.parse(text);
const tokens = text.match(/"(?:\\.|[^"\\])*"|[{}\[\]:,]|[^\s{}\[\]:,]+/g) || [];
const objects = [];
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
if (token === '{') objects.push(new Set());
else if (token === '}') objects.pop();
else if (token.startsWith('"') && tokens[i + 1] === ':') {
const key = JSON.parse(token), keys = objects.at(-1);
if (keys.has(key)) throw new Error(`duplicate JSON key: ${key}`);
keys.add(key);
}
}
return result;
}

function verifyEvidence(binding, metrics, directory, identity) {
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const sha = bytes => require('node:crypto').createHash('sha256').update(bytes).digest('hex');
const read = name => fs.readFileSync(path.join(directory, name), 'utf8');
const provenance = parseMetrics(read('binding-provenance.json'));
for (const field of ['coreSha', 'workflowSha', 'bindingSha']) {
if (provenance[field] !== identity[field] ||
(!(binding === 'wasi' && field === 'bindingSha') && !/^[a-f0-9]{40}$/.test(provenance[field]))) {
throw new Error(`wrong provenance: ${field}`);
}
}
const artifact = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm';
if (provenance.artifactPath !== artifact || sha(fs.readFileSync(path.join(directory, artifact))) !== provenance.artifactSha256) {
throw new Error('wrong executed artifact');
}
const harness = provenance.harnessFiles;
const required = binding === 'jvm' ? ['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts',
'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt',
'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'] : ['nightly.py', 'wasi.py', 'validate.py'];
if (!harness || required.some(file => !harness[file])) throw new Error('missing harness hashes');
// Python validation for both lanes belongs to the measured core, even when
// these reporting helpers come from main (which has no bindings839 harness).
// WASI provenance must hash the same validator that we execute below.
const measuredHarness = path.resolve('benchmarks/bindings839');
const harnessRoot = binding === 'jvm' ? directory : measuredHarness;
for (const [file, digest] of Object.entries(harness)) {
if (path.isAbsolute(file) || file.split('/').includes('..') ||
sha(fs.readFileSync(path.join(harnessRoot, file))) !== digest) throw new Error('wrong harness hash');
}
if (sha(JSON.stringify(Object.fromEntries(Object.entries(harness).sort()))) !== provenance.harnessSha256) {
throw new Error('wrong combined harness hash');
}
const runtime = provenance.runtime;
if (binding === 'jvm') {
if (runtime?.distribution !== 'temurin' || runtime.vendor !== 'Eclipse Adoptium' || runtime.version !== '17.0.17+10' ||
JSON.stringify(runtime.flags) !== JSON.stringify(['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'])) throw new Error('wrong JVM runtime');
} else if (runtime?.version !== 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)' ||
runtime.sha256 !== 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635' ||
JSON.stringify(runtime.flags) !== JSON.stringify(['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'])) {
throw new Error('wrong WASI runtime');
}
const rawPath = path.join(directory, 'benchmark/samples.jsonl');
// Keep #839's validator authoritative for sample IDs/counts/iterations and
// the complete/check markers, including rejection of smoke streams.
const validation = spawnSync('python3', ['-I', path.join(measuredHarness, 'validate.py'), binding, rawPath], { encoding: 'utf8' });
if (validation.status !== 0) throw new Error(`invalid raw samples: ${validation.stderr || validation.error}`);
const records = read('benchmark/samples.jsonl').trim().split('\n').map(parseMetrics);
const environment = records.find(r => r.type === 'environment');
if (binding === 'jvm') {
if (environment.java_vendor !== runtime.vendor || environment.java_version !== '17.0.17' ||
environment.vm_version !== runtime.version || JSON.stringify(environment.jvm_arguments) !== JSON.stringify(runtime.flags) ||
environment.warmup_memory !== 100 || environment.warmup_storage !== 5 ||
environment.build?.core_revision !== identity.coreSha || environment.build?.binding_revision !== identity.bindingSha) {
throw new Error('wrong JVM sample environment');
}
} else if (environment.core_revision !== identity.coreSha || environment.wasm_sha256 !== provenance.artifactSha256 ||
environment.runtime_version !== runtime.version || environment.runtime_sha256 !== runtime.sha256 ||
JSON.stringify(environment.runtime_flags) !== JSON.stringify(runtime.flags) || environment.runtime_cache !== false ||
environment.storage !== 'in-memory; no preopened directories' || environment.warmups !== 3 || environment.samples !== 10) {
throw new Error('wrong WASI sample environment');
}
for (const id of expected(binding)) {
const means = records.filter(r => r.type === 'sample' && r.id === id).map(r => r.total_ns / r.iterations).sort((a, b) => a - b);
const median = (means[4] + means[5]) / 2;
if (Math.abs(metrics[id].latency.value - median) > Math.abs(median) * 1e-12) throw new Error(`wrong raw-sample median: ${id}`);
}
return provenance;
}

function main(command) {
const read = p => fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : '';
const binding = process.env.BINDING;
const ids = expected(binding);
if (command === 'validate') {
validate(binding, JSON.parse(read('binding/binding-results.json')));
const metrics = parseMetrics(read('binding/binding-results.json'));
validate(binding, metrics);
if (['jvm', 'wasi'].includes(binding)) verifyEvidence(binding, metrics, 'binding', {
coreSha: process.env.BENCH_SHA, workflowSha: process.env.WORKFLOW_SHA,
bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(),
});
const executed = read('binding-execution.txt').split('\n').filter(s => s.startsWith('ok: ')).map(s => s.slice(4));
if (executed.length !== ids.length || new Set(executed).size !== ids.length || executed.some(id => !ids.includes(id))) throw new Error('executed IDs do not match shortlist');
return;
Expand All @@ -32,17 +130,18 @@ function main(command) {
const execution = read('binding-execution.txt').replace(/^ok: (.+)$/gm, 'Benchmarking $1: Analyzing');
const result = audit(ids, execution, read('binding-bencher.txt'), process.env.BENCHER_OUTCOME);
const report = {
binding, coreSha: process.env.BENCH_SHA, bindingSha: read('binding-sha.txt').trim(),
binding, coreSha: process.env.BENCH_SHA, bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(),
workflowSha: process.env.WORKFLOW_SHA, ref: process.env.BENCH_REF, historyBranch: process.env.BINDING_HISTORY,
testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result,
suite: `binding_${binding}`, testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result,
provenance: read('binding/binding-provenance.json') ? parseMetrics(read('binding/binding-provenance.json')) : null,
};
fs.writeFileSync('binding-report.json', JSON.stringify(report, null, 2) + '\n');
const summary = `### Binding benchmark: ${binding}\n\nCore: ${report.coreSha}; binding: ${report.bindingSha}; workflow: ${report.workflowSha}\n\nHistory branch: ${report.historyBranch}; testbed: ${report.testbed}; executed ${result.executed.length}/${ids.length}; uploaded ${result.uploaded.length}/${ids.length}; alerts ${result.alerts.length}; upload ${result.uploadStatus}\n\n${result.errors.join('\n')}\n`;
console.log(summary);
if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary);
if (result.errors.length) process.exitCode = 1;
}
module.exports = { expected, validate };
module.exports = { expected, validate, parseMetrics, verifyEvidence };
if (require.main === module) {
try { main(process.argv[2]); } catch (e) { console.error(e.message); process.exitCode = 1; }
}
137 changes: 135 additions & 2 deletions .github/bench/bindings-report.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ const assert = require('node:assert/strict');
const { expected, validate } = require('./bindings-report.cjs');
const data = binding => Object.fromEntries(expected(binding).map(id => [id, { latency: { value: 100 } }]));
test('exact dedicated routes and finite positive latency only', () => {
for (const binding of ['python', 'browser', 'node', 'c']) {
assert.equal(validate(binding, data(binding)).length, binding === 'browser' ? 8 : 6);
for (const binding of ['python', 'browser', 'node', 'c', 'jvm', 'wasi']) {
assert.equal(validate(binding, data(binding)).length, ({ browser: 8, jvm: 14, wasi: 3 }[binding] || 6));
const missing = data(binding); delete missing[expected(binding)[0]];
assert.throws(() => validate(binding, missing), /IDs/);
const extra = data(binding); extra['native/query'] = { latency: { value: 1 } };
Expand Down Expand Up @@ -44,3 +44,136 @@ test('CLI verifies execution and records provenance even when upload is partial'
assert.ok(report.errors.length);
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});

// New lanes must keep distinct inventories, not alias the six P16 cases.
test('JVM lifecycle/storage and WASI REPL shortlists are exact', () => {
assert.deepEqual(expected('wasi'), [
'binding/wasi/repl_startup_empty', 'binding/wasi/repl_script/1k',
'binding/wasi/repl_query_roundtrip/1k',
]);
assert.deepEqual(expected('jvm').slice(6), [
'prepare/point', 'register_predicate/first', 'register_aggregate/first',
'fact_log_batch_256/1k', 'options_error/negative_cache',
'options_readonly_open/1k', 'file_transact/1k', 'log_writer_append_finish/1k',
].map(id => `binding/jvm/${id}`));
});

const { parseMetrics } = require('./bindings-report.cjs');
const crypto = require('node:crypto');
const sha = bytes => crypto.createHash('sha256').update(bytes).digest('hex');
test('duplicate JSON metric keys fail instead of being overwritten', () => {
const id = expected('jvm')[0];
assert.throws(() => parseMetrics(`{"${id}":{"latency":{"value":1}},"${id}":{"latency":{"value":2}}}`), /duplicate/);
assert.throws(() => parseMetrics('{"a":{"value":1,"value":2}}'), /duplicate/);
assert.deepEqual(parseMetrics('{"a":{"value":1},"b":{"value":2}}'), {a: {value: 1}, b: {value: 2}});
});

for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, provenance and artifact bytes before upload`, () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-evidence-'));
const originalCwd = process.cwd();
try {
// Model the default-branch workflow separately from the measured v3 tree.
// This fixture tests validator selection; Python's own suite tests #839's
// sample contract. It also keeps these shared Node tests runnable on main.
const measuredHarness = path.join(dir, 'benchmarks/bindings839');
const workflowHarness = path.join(dir, '.bench-workflow/benchmarks/bindings839');
const workflowHelpers = path.join(dir, '.bench-workflow/.github/bench');
fs.mkdirSync(measuredHarness, {recursive: true});
fs.mkdirSync(workflowHarness, {recursive: true});
fs.mkdirSync(workflowHelpers, {recursive: true});
for (const file of ['bindings-report.cjs', 'report.cjs']) {
fs.copyFileSync(path.join(__dirname, file), path.join(workflowHelpers, file));
}
fs.writeFileSync(path.join(measuredHarness, 'validate.py'), `import json, pathlib, sys
pathlib.Path('measured-validator-ran').write_text(__file__)
records = [json.loads(line) for line in pathlib.Path(sys.argv[2]).read_text().splitlines()]
if records[-1]['type'] != 'complete':
sys.exit('missing complete marker')
`);
for (const file of ['nightly.py', 'wasi.py']) fs.writeFileSync(path.join(measuredHarness, file), file);
fs.writeFileSync(path.join(workflowHarness, 'validate.py'), "raise SystemExit('tampered workflow validator ran')\n");
const { verifyEvidence } = require(path.join(workflowHelpers, 'bindings-report.cjs'));
process.chdir(dir);
fs.mkdirSync(path.join(dir, 'benchmark'));
const coreSha = 'a'.repeat(40), workflowSha = 'b'.repeat(40), bindingSha = binding === 'jvm' ? 'c'.repeat(40) : null;
const ids = expected(binding);
const environment = binding === 'jvm' ? {
java_vendor: 'Eclipse Adoptium', java_version: '17.0.17', vm_version: '17.0.17+10',
jvm_arguments: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'], warmup_memory: 100, warmup_storage: 5,
build: {core_revision: coreSha, binding_revision: bindingSha},
} : {
core_revision: coreSha, wasm_sha256: sha('actual artifact'),
runtime_version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)',
runtime_sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635',
runtime_flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'],
runtime_cache: false, storage: 'in-memory; no preopened directories', warmups: 3, samples: 10,
};
const records = [{ type: 'environment', mode: 'benchmark', ...environment }];
const metrics = {};
for (const id of ids) {
const iterations = id.endsWith('repl_query_roundtrip/1k') ? 20 : binding === 'wasi' || /(?:file_transact|log_writer_append_finish)\/1k$/.test(id) ? 1 : 20;
for (let sample = 0; sample < 10; sample++) records.push({type: 'sample', id, sample, iterations, total_ns: (sample + 1) * iterations, ns_per_op: sample + 1});
records.push({type: 'checked', id});
metrics[id] = { latency: { value: 5.5 } };
}
records.push({type: 'complete', mode: 'benchmark', count: ids.length, ids});
const raw = path.join(dir, 'benchmark/samples.jsonl');
fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n'));
const artifactPath = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm';
fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact');
const harnessFiles = binding === 'jvm'
? Object.fromEntries(['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts',
'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt',
'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'].map(p => [p, sha(p)]))
: Object.fromEntries(['nightly.py', 'wasi.py', 'validate.py'].map(p => [p, sha(fs.readFileSync(path.join(measuredHarness, p)))]));
if (binding === 'jvm') for (const file of Object.keys(harnessFiles)) {
fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true});
fs.writeFileSync(path.join(dir, file), file);
}
const provenance = { coreSha, workflowSha, bindingSha, artifactPath, artifactSha256: sha('actual artifact'), harnessFiles,
harnessSha256: sha(JSON.stringify(Object.fromEntries(Object.entries(harnessFiles).sort()))),
runtime: binding === 'jvm'
? { distribution: 'temurin', vendor: 'Eclipse Adoptium', version: '17.0.17+10', flags: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'] }
: { version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'] },
};
const file = path.join(dir, 'binding-provenance.json');
fs.writeFileSync(file, JSON.stringify(provenance));
const identity = { coreSha, workflowSha, bindingSha };
assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha);
assert.equal(fs.readFileSync('measured-validator-ran', 'utf8'), path.join(measuredHarness, 'validate.py'));
// main has no workflow harness at all; validation still uses measured v3.
fs.rmSync(workflowHarness, {recursive: true});
assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha);
metrics[ids[0]].latency.value = 6;
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /median/);
metrics[ids[0]].latency.value = 5.5;
for (const field of ['coreSha', 'workflowSha', 'bindingSha']) {
if (field === 'bindingSha' && binding === 'wasi') continue;
assert.throws(() => verifyEvidence(binding, metrics, dir, {...identity, [field]: 'd'.repeat(40)}), /provenance/);
}
fs.writeFileSync(file, JSON.stringify({...provenance, runtime: {...provenance.runtime, flags: []}}));
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /runtime/);
fs.writeFileSync(file, JSON.stringify(provenance));
fs.writeFileSync(path.join(dir, artifactPath), 'different artifact');
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /artifact/);
fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact');
const wrongEnvironment = {...records[0], ...(binding === 'jvm' ? {warmup_memory: 3} : {runtime_cache: true})};
fs.writeFileSync(raw, [wrongEnvironment, ...records.slice(1)].map(r => JSON.stringify(r)).join('\n'));
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /environment/);
fs.writeFileSync(raw, records.slice(0, -1).map(r => JSON.stringify(r)).join('\n'));
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /samples/);
fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n'));
// A workflow copy cannot rescue a missing measured validator on v3.
fs.mkdirSync(workflowHarness, {recursive: true});
fs.writeFileSync(path.join(workflowHarness, 'validate.py'), '# accepts everything\n');
if (binding === 'wasi') {
fs.appendFileSync(path.join(measuredHarness, 'validate.py'), '# changed since measurement\n');
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /harness hash/);
}
fs.unlinkSync(path.join(measuredHarness, 'validate.py'));
assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /validate.py|raw samples/);
} finally {
process.chdir(originalCwd);
fs.rmSync(dir, { recursive: true, force: true });
}
});
Loading
Loading