diff --git a/.github/bench/bindings-report.cjs b/.github/bench/bindings-report.cjs index 21c98217b8..1b503ac850 100644 --- a/.github/bench/bindings-report.cjs +++ b/.github/bench/bindings-report.cjs @@ -2,9 +2,13 @@ const fs = require('node:fs'); const { audit } = require('./report.cjs'); function expected(binding) { - if (!['python', 'browser', 'node', 'c'].includes(binding)) throw new Error('unknown binding'); + if (!['python', 'browser', 'node', 'c', 'jvm', 'wasi'].includes(binding)) throw new Error('unknown binding'); + if (binding === 'wasi') return ['repl_startup_empty', 'repl_script/1k', 'repl_query_roundtrip/1k'].map(s => `binding/wasi/${s}`); const suffixes = ['prepared_execute/1k', 'cursor_batch_1/1k', 'cursor_batch_256/1k', 'set_roundtrip/2', 'predicate/1k', 'aggregate/1k']; if (binding === 'browser') suffixes.push('indexeddb_write/1k', 'indexeddb_reopen/1k'); + if (binding === 'jvm') suffixes.push('prepare/point', 'register_predicate/first', 'register_aggregate/first', + 'fact_log_batch_256/1k', 'options_error/negative_cache', 'options_readonly_open/1k', + 'file_transact/1k', 'log_writer_append_finish/1k'); return suffixes.map(s => `binding/${binding}/${s}`); } function validate(binding, metrics) { @@ -16,12 +20,106 @@ function validate(binding, metrics) { } return ids; } +// JSON.parse alone silently overwrites duplicate keys. Parse syntax first, then +// inspect object keys (including escaped spellings) in the valid token stream. +function parseMetrics(text) { + const result = JSON.parse(text); + const tokens = text.match(/"(?:\\.|[^"\\])*"|[{}\[\]:,]|[^\s{}\[\]:,]+/g) || []; + const objects = []; + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + if (token === '{') objects.push(new Set()); + else if (token === '}') objects.pop(); + else if (token.startsWith('"') && tokens[i + 1] === ':') { + const key = JSON.parse(token), keys = objects.at(-1); + if (keys.has(key)) throw new Error(`duplicate JSON key: ${key}`); + keys.add(key); + } + } + return result; +} + +function verifyEvidence(binding, metrics, directory, identity) { + const path = require('node:path'); + const { spawnSync } = require('node:child_process'); + const sha = bytes => require('node:crypto').createHash('sha256').update(bytes).digest('hex'); + const read = name => fs.readFileSync(path.join(directory, name), 'utf8'); + const provenance = parseMetrics(read('binding-provenance.json')); + for (const field of ['coreSha', 'workflowSha', 'bindingSha']) { + if (provenance[field] !== identity[field] || + (!(binding === 'wasi' && field === 'bindingSha') && !/^[a-f0-9]{40}$/.test(provenance[field]))) { + throw new Error(`wrong provenance: ${field}`); + } + } + const artifact = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm'; + if (provenance.artifactPath !== artifact || sha(fs.readFileSync(path.join(directory, artifact))) !== provenance.artifactSha256) { + throw new Error('wrong executed artifact'); + } + const harness = provenance.harnessFiles; + const required = binding === 'jvm' ? ['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'] : ['nightly.py', 'wasi.py', 'validate.py']; + if (!harness || required.some(file => !harness[file])) throw new Error('missing harness hashes'); + // Python validation for both lanes belongs to the measured core, even when + // these reporting helpers come from main (which has no bindings839 harness). + // WASI provenance must hash the same validator that we execute below. + const measuredHarness = path.resolve('benchmarks/bindings839'); + const harnessRoot = binding === 'jvm' ? directory : measuredHarness; + for (const [file, digest] of Object.entries(harness)) { + if (path.isAbsolute(file) || file.split('/').includes('..') || + sha(fs.readFileSync(path.join(harnessRoot, file))) !== digest) throw new Error('wrong harness hash'); + } + if (sha(JSON.stringify(Object.fromEntries(Object.entries(harness).sort()))) !== provenance.harnessSha256) { + throw new Error('wrong combined harness hash'); + } + const runtime = provenance.runtime; + if (binding === 'jvm') { + if (runtime?.distribution !== 'temurin' || runtime.vendor !== 'Eclipse Adoptium' || runtime.version !== '17.0.17+10' || + JSON.stringify(runtime.flags) !== JSON.stringify(['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'])) throw new Error('wrong JVM runtime'); + } else if (runtime?.version !== 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)' || + runtime.sha256 !== 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635' || + JSON.stringify(runtime.flags) !== JSON.stringify(['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'])) { + throw new Error('wrong WASI runtime'); + } + const rawPath = path.join(directory, 'benchmark/samples.jsonl'); + // Keep #839's validator authoritative for sample IDs/counts/iterations and + // the complete/check markers, including rejection of smoke streams. + const validation = spawnSync('python3', ['-I', path.join(measuredHarness, 'validate.py'), binding, rawPath], { encoding: 'utf8' }); + if (validation.status !== 0) throw new Error(`invalid raw samples: ${validation.stderr || validation.error}`); + const records = read('benchmark/samples.jsonl').trim().split('\n').map(parseMetrics); + const environment = records.find(r => r.type === 'environment'); + if (binding === 'jvm') { + if (environment.java_vendor !== runtime.vendor || environment.java_version !== '17.0.17' || + environment.vm_version !== runtime.version || JSON.stringify(environment.jvm_arguments) !== JSON.stringify(runtime.flags) || + environment.warmup_memory !== 100 || environment.warmup_storage !== 5 || + environment.build?.core_revision !== identity.coreSha || environment.build?.binding_revision !== identity.bindingSha) { + throw new Error('wrong JVM sample environment'); + } + } else if (environment.core_revision !== identity.coreSha || environment.wasm_sha256 !== provenance.artifactSha256 || + environment.runtime_version !== runtime.version || environment.runtime_sha256 !== runtime.sha256 || + JSON.stringify(environment.runtime_flags) !== JSON.stringify(runtime.flags) || environment.runtime_cache !== false || + environment.storage !== 'in-memory; no preopened directories' || environment.warmups !== 3 || environment.samples !== 10) { + throw new Error('wrong WASI sample environment'); + } + for (const id of expected(binding)) { + const means = records.filter(r => r.type === 'sample' && r.id === id).map(r => r.total_ns / r.iterations).sort((a, b) => a - b); + const median = (means[4] + means[5]) / 2; + if (Math.abs(metrics[id].latency.value - median) > Math.abs(median) * 1e-12) throw new Error(`wrong raw-sample median: ${id}`); + } + return provenance; +} + function main(command) { const read = p => fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : ''; const binding = process.env.BINDING; const ids = expected(binding); if (command === 'validate') { - validate(binding, JSON.parse(read('binding/binding-results.json'))); + const metrics = parseMetrics(read('binding/binding-results.json')); + validate(binding, metrics); + if (['jvm', 'wasi'].includes(binding)) verifyEvidence(binding, metrics, 'binding', { + coreSha: process.env.BENCH_SHA, workflowSha: process.env.WORKFLOW_SHA, + bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(), + }); const executed = read('binding-execution.txt').split('\n').filter(s => s.startsWith('ok: ')).map(s => s.slice(4)); if (executed.length !== ids.length || new Set(executed).size !== ids.length || executed.some(id => !ids.includes(id))) throw new Error('executed IDs do not match shortlist'); return; @@ -32,9 +130,10 @@ function main(command) { const execution = read('binding-execution.txt').replace(/^ok: (.+)$/gm, 'Benchmarking $1: Analyzing'); const result = audit(ids, execution, read('binding-bencher.txt'), process.env.BENCHER_OUTCOME); const report = { - binding, coreSha: process.env.BENCH_SHA, bindingSha: read('binding-sha.txt').trim(), + binding, coreSha: process.env.BENCH_SHA, bindingSha: binding === 'wasi' ? null : read('binding-sha.txt').trim(), workflowSha: process.env.WORKFLOW_SHA, ref: process.env.BENCH_REF, historyBranch: process.env.BINDING_HISTORY, - testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result, + suite: `binding_${binding}`, testbed: `binding-${binding}-ubuntu24-v1`, selected: ids, ...result, + provenance: read('binding/binding-provenance.json') ? parseMetrics(read('binding/binding-provenance.json')) : null, }; fs.writeFileSync('binding-report.json', JSON.stringify(report, null, 2) + '\n'); const summary = `### Binding benchmark: ${binding}\n\nCore: ${report.coreSha}; binding: ${report.bindingSha}; workflow: ${report.workflowSha}\n\nHistory branch: ${report.historyBranch}; testbed: ${report.testbed}; executed ${result.executed.length}/${ids.length}; uploaded ${result.uploaded.length}/${ids.length}; alerts ${result.alerts.length}; upload ${result.uploadStatus}\n\n${result.errors.join('\n')}\n`; @@ -42,7 +141,7 @@ function main(command) { if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); if (result.errors.length) process.exitCode = 1; } -module.exports = { expected, validate }; +module.exports = { expected, validate, parseMetrics, verifyEvidence }; if (require.main === module) { try { main(process.argv[2]); } catch (e) { console.error(e.message); process.exitCode = 1; } } diff --git a/.github/bench/bindings-report.test.cjs b/.github/bench/bindings-report.test.cjs index cdf921d40a..c24a645a43 100644 --- a/.github/bench/bindings-report.test.cjs +++ b/.github/bench/bindings-report.test.cjs @@ -3,8 +3,8 @@ const assert = require('node:assert/strict'); const { expected, validate } = require('./bindings-report.cjs'); const data = binding => Object.fromEntries(expected(binding).map(id => [id, { latency: { value: 100 } }])); test('exact dedicated routes and finite positive latency only', () => { - for (const binding of ['python', 'browser', 'node', 'c']) { - assert.equal(validate(binding, data(binding)).length, binding === 'browser' ? 8 : 6); + for (const binding of ['python', 'browser', 'node', 'c', 'jvm', 'wasi']) { + assert.equal(validate(binding, data(binding)).length, ({ browser: 8, jvm: 14, wasi: 3 }[binding] || 6)); const missing = data(binding); delete missing[expected(binding)[0]]; assert.throws(() => validate(binding, missing), /IDs/); const extra = data(binding); extra['native/query'] = { latency: { value: 1 } }; @@ -44,3 +44,136 @@ test('CLI verifies execution and records provenance even when upload is partial' assert.ok(report.errors.length); } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); + +// New lanes must keep distinct inventories, not alias the six P16 cases. +test('JVM lifecycle/storage and WASI REPL shortlists are exact', () => { + assert.deepEqual(expected('wasi'), [ + 'binding/wasi/repl_startup_empty', 'binding/wasi/repl_script/1k', + 'binding/wasi/repl_query_roundtrip/1k', + ]); + assert.deepEqual(expected('jvm').slice(6), [ + 'prepare/point', 'register_predicate/first', 'register_aggregate/first', + 'fact_log_batch_256/1k', 'options_error/negative_cache', + 'options_readonly_open/1k', 'file_transact/1k', 'log_writer_append_finish/1k', + ].map(id => `binding/jvm/${id}`)); +}); + +const { parseMetrics } = require('./bindings-report.cjs'); +const crypto = require('node:crypto'); +const sha = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +test('duplicate JSON metric keys fail instead of being overwritten', () => { + const id = expected('jvm')[0]; + assert.throws(() => parseMetrics(`{"${id}":{"latency":{"value":1}},"${id}":{"latency":{"value":2}}}`), /duplicate/); + assert.throws(() => parseMetrics('{"a":{"value":1,"value":2}}'), /duplicate/); + assert.deepEqual(parseMetrics('{"a":{"value":1},"b":{"value":2}}'), {a: {value: 1}, b: {value: 2}}); +}); + +for (const binding of ['jvm', 'wasi']) test(`${binding} checks raw medians, provenance and artifact bytes before upload`, () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-evidence-')); + const originalCwd = process.cwd(); + try { + // Model the default-branch workflow separately from the measured v3 tree. + // This fixture tests validator selection; Python's own suite tests #839's + // sample contract. It also keeps these shared Node tests runnable on main. + const measuredHarness = path.join(dir, 'benchmarks/bindings839'); + const workflowHarness = path.join(dir, '.bench-workflow/benchmarks/bindings839'); + const workflowHelpers = path.join(dir, '.bench-workflow/.github/bench'); + fs.mkdirSync(measuredHarness, {recursive: true}); + fs.mkdirSync(workflowHarness, {recursive: true}); + fs.mkdirSync(workflowHelpers, {recursive: true}); + for (const file of ['bindings-report.cjs', 'report.cjs']) { + fs.copyFileSync(path.join(__dirname, file), path.join(workflowHelpers, file)); + } + fs.writeFileSync(path.join(measuredHarness, 'validate.py'), `import json, pathlib, sys +pathlib.Path('measured-validator-ran').write_text(__file__) +records = [json.loads(line) for line in pathlib.Path(sys.argv[2]).read_text().splitlines()] +if records[-1]['type'] != 'complete': + sys.exit('missing complete marker') +`); + for (const file of ['nightly.py', 'wasi.py']) fs.writeFileSync(path.join(measuredHarness, file), file); + fs.writeFileSync(path.join(workflowHarness, 'validate.py'), "raise SystemExit('tampered workflow validator ran')\n"); + const { verifyEvidence } = require(path.join(workflowHelpers, 'bindings-report.cjs')); + process.chdir(dir); + fs.mkdirSync(path.join(dir, 'benchmark')); + const coreSha = 'a'.repeat(40), workflowSha = 'b'.repeat(40), bindingSha = binding === 'jvm' ? 'c'.repeat(40) : null; + const ids = expected(binding); + const environment = binding === 'jvm' ? { + java_vendor: 'Eclipse Adoptium', java_version: '17.0.17', vm_version: '17.0.17+10', + jvm_arguments: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'], warmup_memory: 100, warmup_storage: 5, + build: {core_revision: coreSha, binding_revision: bindingSha}, + } : { + core_revision: coreSha, wasm_sha256: sha('actual artifact'), + runtime_version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', + runtime_sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', + runtime_flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'], + runtime_cache: false, storage: 'in-memory; no preopened directories', warmups: 3, samples: 10, + }; + const records = [{ type: 'environment', mode: 'benchmark', ...environment }]; + const metrics = {}; + for (const id of ids) { + const iterations = id.endsWith('repl_query_roundtrip/1k') ? 20 : binding === 'wasi' || /(?:file_transact|log_writer_append_finish)\/1k$/.test(id) ? 1 : 20; + for (let sample = 0; sample < 10; sample++) records.push({type: 'sample', id, sample, iterations, total_ns: (sample + 1) * iterations, ns_per_op: sample + 1}); + records.push({type: 'checked', id}); + metrics[id] = { latency: { value: 5.5 } }; + } + records.push({type: 'complete', mode: 'benchmark', count: ids.length, ids}); + const raw = path.join(dir, 'benchmark/samples.jsonl'); + fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n')); + const artifactPath = binding === 'jvm' ? 'benchmark/libminigraf_ffi.so' : 'minigraf.wasm'; + fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact'); + const harnessFiles = binding === 'jvm' + ? Object.fromEntries(['bench/nightly.sh', 'bench/manual/build.sh', 'bench/manual/build.gradle.kts', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/Benchmark.kt', + 'bench/manual/src/main/kotlin/io/github/project_minigraf/benchmark/ResultChecks.kt'].map(p => [p, sha(p)])) + : Object.fromEntries(['nightly.py', 'wasi.py', 'validate.py'].map(p => [p, sha(fs.readFileSync(path.join(measuredHarness, p)))])); + if (binding === 'jvm') for (const file of Object.keys(harnessFiles)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), {recursive: true}); + fs.writeFileSync(path.join(dir, file), file); + } + const provenance = { coreSha, workflowSha, bindingSha, artifactPath, artifactSha256: sha('actual artifact'), harnessFiles, + harnessSha256: sha(JSON.stringify(Object.fromEntries(Object.entries(harnessFiles).sort()))), + runtime: binding === 'jvm' + ? { distribution: 'temurin', vendor: 'Eclipse Adoptium', version: '17.0.17+10', flags: ['-Xms512m', '-Xmx512m', '-XX:+UseG1GC'] } + : { version: 'wasmtime 49.0.2 (3c8a3e79a 2026-10-02)', sha256: 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635', flags: ['run', '-C', 'cache=n', '-C', 'parallel-compilation=n'] }, + }; + const file = path.join(dir, 'binding-provenance.json'); + fs.writeFileSync(file, JSON.stringify(provenance)); + const identity = { coreSha, workflowSha, bindingSha }; + assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha); + assert.equal(fs.readFileSync('measured-validator-ran', 'utf8'), path.join(measuredHarness, 'validate.py')); + // main has no workflow harness at all; validation still uses measured v3. + fs.rmSync(workflowHarness, {recursive: true}); + assert.equal(verifyEvidence(binding, metrics, dir, identity).coreSha, coreSha); + metrics[ids[0]].latency.value = 6; + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /median/); + metrics[ids[0]].latency.value = 5.5; + for (const field of ['coreSha', 'workflowSha', 'bindingSha']) { + if (field === 'bindingSha' && binding === 'wasi') continue; + assert.throws(() => verifyEvidence(binding, metrics, dir, {...identity, [field]: 'd'.repeat(40)}), /provenance/); + } + fs.writeFileSync(file, JSON.stringify({...provenance, runtime: {...provenance.runtime, flags: []}})); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /runtime/); + fs.writeFileSync(file, JSON.stringify(provenance)); + fs.writeFileSync(path.join(dir, artifactPath), 'different artifact'); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /artifact/); + fs.writeFileSync(path.join(dir, artifactPath), 'actual artifact'); + const wrongEnvironment = {...records[0], ...(binding === 'jvm' ? {warmup_memory: 3} : {runtime_cache: true})}; + fs.writeFileSync(raw, [wrongEnvironment, ...records.slice(1)].map(r => JSON.stringify(r)).join('\n')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /environment/); + fs.writeFileSync(raw, records.slice(0, -1).map(r => JSON.stringify(r)).join('\n')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /samples/); + fs.writeFileSync(raw, records.map(r => JSON.stringify(r)).join('\n')); + // A workflow copy cannot rescue a missing measured validator on v3. + fs.mkdirSync(workflowHarness, {recursive: true}); + fs.writeFileSync(path.join(workflowHarness, 'validate.py'), '# accepts everything\n'); + if (binding === 'wasi') { + fs.appendFileSync(path.join(measuredHarness, 'validate.py'), '# changed since measurement\n'); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /harness hash/); + } + fs.unlinkSync(path.join(measuredHarness, 'validate.py')); + assert.throws(() => verifyEvidence(binding, metrics, dir, identity), /validate.py|raw samples/); + } finally { + process.chdir(originalCwd); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/.github/bench/bindings-workflow.test.cjs b/.github/bench/bindings-workflow.test.cjs new file mode 100644 index 0000000000..fc6f6081c8 --- /dev/null +++ b/.github/bench/bindings-workflow.test.cjs @@ -0,0 +1,235 @@ +// Run workflow shell/notification bodies locally; no network or GitHub writes. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/bench.yml'), 'utf8'); +const bindingJob = workflow.split('\n bindings:\n')[1]; +function body(name, kind = 'run') { + const step = bindingJob.split(` - name: ${name}\n`)[1]; + assert.ok(step, `missing step ${name}`); + return step.split(` ${kind}: |\n`)[1]?.split(/\n - /)[0].replace(/^ /gm, '').trimEnd() + || step.split(` ${kind}: |\n`)[1].split(/\n - /)[0].replace(/^ /gm, '').trimEnd(); +} +function shell(name, env, setup = () => {}, prefix = '') { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-workflow-')); + try { + setup(cwd); + const result = spawnSync('bash', ['-e', '-c', prefix + body(name).replace(/\$\{\{ secrets\.BENCHER_API_TOKEN \}\}/g, 'test-token')], { + cwd, encoding: 'utf8', env: {...process.env, GITHUB_OUTPUT: path.join(cwd, 'out'), GITHUB_ENV: path.join(cwd, 'env'), GITHUB_STEP_SUMMARY: path.join(cwd, 'summary'), ...env}, + }); + const read = file => fs.existsSync(path.join(cwd, file)) ? fs.readFileSync(path.join(cwd, file), 'utf8') : ''; + return {...result, output: read('out'), vars: read('env'), summary: read('summary')}; + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } +} +test('new rows use the actual binding suite job names', () => { + assert.match(bindingJob, /name: bench \/ binding_\$\{\{ matrix.binding \}\}/); + assert.match(bindingJob, /binding: jvm, repo: minigraf-java/); + assert.match(bindingJob, /binding: wasi/); + assert.match(workflow, /jvm-ref:/); +}); +test('main skips; either v3 lane fails when its measured validator is absent', () => { + for (const binding of ['jvm', 'wasi']) { + const main = shell('Check v3 binding suite availability', {BENCH_REF: 'main', BINDING: binding}); + assert.equal(main.status, 0, main.stderr); + assert.match(main.summary, /Skipping/); + assert.doesNotMatch(main.output, /enabled=true/); + for (const missing of [true, false]) { + const result = shell('Check v3 binding suite availability', {BENCH_REF: 'v3', BINDING: binding}, cwd => { + for (const file of ['benches/minigraf_bench/prepared.rs', 'benchmarks/bindings839/nightly.py', + 'benchmarks/bindings839/wasi.py', ...(missing ? [] : ['benchmarks/bindings839/validate.py'])]) { + fs.mkdirSync(path.dirname(path.join(cwd, file)), {recursive: true}); + fs.writeFileSync(path.join(cwd, file), 'fixture'); + } + }); + assert.equal(result.status === 0, !missing, result.stderr); + assert.equal(result.output.includes('enabled=true'), !missing); + } + } +}); +test('workflow checkout needs only shared helpers, never the measured Python harness', () => { + const checkout = bindingJob.split('ref: ${{ github.workflow_sha }}')[1].split('persist-credentials: false')[0]; + assert.match(checkout, /\/\.github\/bench\//); + assert.doesNotMatch(checkout, /bindings839/); +}); +test('reporting CI skips absent Python tests on main and runs them on v3', () => { + const reporting = fs.readFileSync(path.join(__dirname, '../workflows/bench-report.yml'), 'utf8'); + assert.equal((reporting.match(/'benchmarks\/bindings839\/\*\*'/g) || []).length, 2, 'keep both path triggers'); + const step = reporting.split(' - ').find(s => s.includes('python3 -I -m unittest')); + assert.match(step, /if:.*hashFiles\('benchmarks\/bindings839\/test_\*\.py'\) != ''/); + const expression = step.match(/if: \$\{\{ (.+) \}\}/)[1]; + for (const [hash, shouldRun] of [['', false], ['test-files-hash', true]]) { + const hashFiles = pattern => { + assert.equal(pattern, 'benchmarks/bindings839/test_*.py'); + return hash; + }; + assert.equal(new Function('hashFiles', `return ${expression}`)(hashFiles), shouldRun); + } +}); +for (const binding of ['jvm', 'wasi']) test(`${binding} uploads only to its own testbed without inherited history`, () => { + for (const history of ['v3', 'binding-experiment/experiment/v3']) { + const result = shell('Upload foreign runtime metrics', {BINDING: binding, BINDING_HISTORY: history, BENCH_SHA: 'a'.repeat(40), BENCH_THRESHOLD: '0.999'}, undefined, + 'bencher() { printf "%s\\0" "$@"; }\n'); + assert.equal(result.status, 0, result.stderr); + const args = result.stdout.split('\0'); + assert.equal(args[args.indexOf('--testbed') + 1], `binding-${binding}-ubuntu24-v1`); + assert.equal(args[args.indexOf('--branch') + 1], history); + assert.equal(args[args.indexOf('--threshold-max-sample-size') + 1], '30'); + assert.equal(args[args.indexOf('--threshold-upper-boundary') + 1], '0.999'); + assert.ok(!args.some(a => a.startsWith('--start-point'))); + } +}); + +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; +for (const binding of ['jvm', 'wasi']) test(`${binding} repeated exact IDs, immediate upload failure and failed lookup`, async () => { + const suite = `binding_${binding}`, id = `binding/${binding}/example`; + const alert = name => `- ${name} (Latency): https://bencher.dev/console/projects/minigraf/alerts/123`; + const original = {...process.env}; + try { + Object.assign(process.env, {SUITE: suite, BENCH_REF: 'v3', BENCH_SHA: 'a'.repeat(40)}); + for (const scenario of ['repeat', 'different-id', 'upload-failure', 'lookup-failure', 'existing-issue', 'experiment-prior']) { + const calls = [], warnings = []; + const github = {rest: {actions: { + listWorkflowRuns: async () => ({data: {workflow_runs: [{id: 1, display_title: 'Nightly Benchmarks (main)'}, {id: 2, display_title: 'Nightly Benchmarks (v3)', html_url: 'prior'}]}}), + listJobsForWorkflowRun: async args => { assert.equal(args.run_id, 2); return {data: {jobs: [{id: 3, name: `bench / ${suite}`}, {id: 4, name: 'bench / binding_python'}]}}; }, + downloadJobLogsForWorkflowRun: async args => { + assert.equal(args.job_id, 3); + if (scenario === 'lookup-failure') throw new Error('unavailable logs'); + return {data: `History branch: ${scenario === 'experiment-prior' ? 'binding-experiment/v3/java-only' : 'v3'}; testbed: binding-${binding}-ubuntu24-v1;\n` + alert(scenario === 'different-id' ? `${id}/other` : id)}; + }, + }, issues: { + listForRepo: async () => ({data: scenario === 'existing-issue' ? [{number: 8, title: `Benchmark regression: ${suite} (v3)`}] : []}), + create: async args => { calls.push(args); return {data: {number: 9}}; }, + createComment: async args => { calls.push(args); }, + }}}; + const summary = {addHeading() {return this;}, addRaw() {return this;}, addList() {return this;}, addBreak() {return this;}, async write() {}}; + const mockRequire = name => { assert.equal(name, 'fs'); return {readFileSync: () => scenario === 'upload-failure' ? 'Unauthorized' : alert(id)}; }; + await new AsyncFunction('require', 'github', 'context', 'core', 'console', body('Report regression', 'script'))( + mockRequire, github, {repo: {owner: 'test', repo: 'test'}, runId: 10, serverUrl: 'https://example.com'}, {warning: text => warnings.push(text), summary}, {log() {}}); + assert.equal(calls.length, ['repeat', 'upload-failure', 'existing-issue'].includes(scenario) ? 1 : 0); + if (scenario === 'lookup-failure') assert.ok(warnings.some(w => w.includes('Could not read'))); + if (scenario === 'upload-failure') assert.match(calls[0].body, /API or upload error/); + if (scenario === 'repeat') assert.match(calls[0].body, /two runs in a row/); + if (scenario === 'existing-issue') assert.equal(calls[0].issue_number, 8); + } + } finally { + for (const key of ['SUITE', 'BENCH_REF', 'BENCH_SHA']) { + if (original[key] === undefined) delete process.env[key]; else process.env[key] = original[key]; + } + } +}); + +test('experimental binding and core refs are isolated, including WASI without a repo', () => { + for (const [binding, ref, bindingRef, history] of [ + ['jvm', 'v3', 'v3', 'v3'], + ['jvm', 'v3', 'java-only', 'binding-experiment/v3/java-only'], + ['jvm', 'candidate', 'v3', 'binding-experiment/candidate/v3'], + ['wasi', 'candidate', 'v3', 'binding-experiment/candidate/v3'], + ]) { + const result = shell('Pin binding source', {BINDING: binding, BENCH_REF: ref, BINDING_REF: bindingRef}, undefined, + 'git() { printf "%s\\n" binding-sha; }\n'); + assert.equal(result.status, 0, result.stderr); + assert.match(result.vars, new RegExp(`BINDING_HISTORY=${history}\\n`)); + } +}); + +test('JVM manual builder owns the patch and wrong Cargo resolutions fail', () => { + const helper = path.join(__dirname, 'bindings.sh'); + for (const resolution of ['correct', 'release-pin', 'other-checkout', 'duplicate']) { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-build-')); + try { + fs.mkdirSync(path.join(cwd, 'binding/bench/manual'), {recursive: true}); + fs.mkdirSync(path.join(cwd, 'bin')); + fs.writeFileSync(path.join(cwd, 'binding/bench/nightly.sh'), '# contract placeholder\n'); + fs.writeFileSync(path.join(cwd, 'binding/bench/manual/build.sh'), `set -euo pipefail + test ! -e .cargo/config.toml + test "$CARGO_BUILD_JOBS" = 4 + test "$CARGO_TARGET_DIR" = "$PWD/target" + test "$CORE_REVISION" = "$BENCH_SHA" + mkdir .cargo + echo 'owned by manual build' > .cargo/config.toml + `); + fs.writeFileSync(path.join(cwd, 'bin/git'), '#!/bin/sh\nprintf "%s\\n" "$BENCH_SHA"\n', {mode: 0o755}); + const pkg = {name: 'minigraf', source: resolution === 'release-pin' ? 'git+release' : null, + manifest_path: path.join(resolution === 'other-checkout' ? '/wrong' : cwd, 'Cargo.toml')}; + const metadata = {packages: resolution === 'duplicate' ? [pkg, pkg] : [pkg]}; + fs.writeFileSync(path.join(cwd, 'metadata.json'), JSON.stringify(metadata)); + fs.writeFileSync(path.join(cwd, 'bin/cargo'), '#!/bin/sh\ncat ../metadata.json\n', {mode: 0o755}); + const result = spawnSync('bash', [helper, 'build', 'jvm'], {cwd, encoding: 'utf8', + env: {...process.env, PATH: `${cwd}/bin:${process.env.PATH}`, BENCH_SHA: 'a'.repeat(40)}}); + assert.equal(result.status === 0, resolution === 'correct', result.stderr); + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } + } +}); + +test('JVM run refuses stale output and smoke that emits metrics', () => { + for (const scenario of ['stale', 'smoke-metrics', 'failed-exit']) { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'binding-run-')); + try { + fs.mkdirSync(path.join(cwd, 'binding/bench'), {recursive: true}); + fs.writeFileSync(path.join(cwd, 'binding/bench/nightly.sh'), scenario === 'failed-exit' ? 'exit 7\n' : 'echo "{}" > binding-results.json\n'); + if (scenario === 'stale') fs.writeFileSync(path.join(cwd, 'binding/binding-results.json'), '{}'); + const args = [path.join(__dirname, 'bindings.sh'), 'run', 'jvm']; + if (scenario === 'smoke-metrics') args.push('--test'); + const result = spawnSync('bash', args, {cwd, encoding: 'utf8'}); + assert.notEqual(result.status, 0, scenario); + } finally { fs.rmSync(cwd, {recursive: true, force: true}); } + } +}); + +for (const binding of ['jvm', 'wasi']) test(`${binding} finds the most recent production history within a bounded search`, async () => { + const original = {...process.env}; + const suite = `binding_${binding}`, id = `binding/${binding}/example`; + const alert = `- ${id} (Latency): https://bencher.dev/console/projects/minigraf/alerts/123`; + const production = `History branch: v3; testbed: binding-${binding}-ubuntu24-v1;`; + try { + Object.assign(process.env, {SUITE: suite, BENCH_REF: 'v3', BENCH_SHA: 'a'.repeat(40)}); + for (const scenario of ['repeat-after-experiment', 'clean-production', 'wrong-testbed', 'no-suite', 'lookup-failure', 'exhausted']) { + const calls = [], warnings = [], visited = []; + const github = {rest: {actions: { + listWorkflowRuns: async args => { + assert.equal(args.status, 'completed'); + assert.equal(args.per_page, 20); + return {data: {workflow_runs: [ + {id: 100, display_title: 'Nightly Benchmarks (v3)'}, // current + {id: 99, display_title: 'Nightly Benchmarks (main)'}, + ...Array.from({length: 18}, (_, i) => ({id: 98 - i, display_title: 'Nightly Benchmarks (v3)', html_url: `prior-${98 - i}`})), + ]}}; + }, + listJobsForWorkflowRun: async args => { + visited.push(args.run_id); + return {data: {jobs: scenario === 'no-suite' && args.run_id === 98 ? [] : [{id: args.run_id, name: `bench / ${suite}`}]}}; + }, + downloadJobLogsForWorkflowRun: async ({job_id}) => { + if (scenario === 'lookup-failure' && job_id === 97) throw new Error('unavailable logs'); + if (job_id === 98 || scenario === 'exhausted') { + const history = scenario === 'wrong-testbed' ? 'History branch: v3; testbed: binding-python-ubuntu24-v1;' : + `History branch: binding-experiment/v3/java-only; testbed: binding-${binding}-ubuntu24-v1;`; + return {data: `${history}\n${alert}`}; + } + return {data: `${production}\n${scenario === 'clean-production' && job_id === 97 ? '' : alert}`}; + }, + }, issues: { + listForRepo: async () => ({data: []}), + create: async args => { calls.push(args); return {data: {number: 1}}; }, + }}}; + const summary = {addHeading() {return this;}, addRaw() {return this;}, addList() {return this;}, addBreak() {return this;}, async write() {}}; + await new AsyncFunction('require', 'github', 'context', 'core', 'console', body('Report regression', 'script'))( + () => ({readFileSync: () => alert}), github, + {repo: {owner: 'test', repo: 'test'}, runId: 100, serverUrl: 'https://example.com'}, + {warning: message => warnings.push(message), summary}, {log() {}}); + const reports = ['repeat-after-experiment', 'wrong-testbed', 'no-suite'].includes(scenario); + assert.equal(calls.length, reports ? 1 : 0, scenario); + if (reports) assert.match(calls[0].body, /Previous run:\*\* prior-97/); + assert.deepEqual(visited, scenario === 'exhausted' ? Array.from({length: 18}, (_, i) => 98 - i) : [98, 97]); + if (scenario === 'lookup-failure') assert.ok(warnings.some(w => w.includes('Could not read'))); + if (scenario === 'exhausted') assert.ok(warnings.some(w => w.includes('No matching production history'))); + } + } finally { + for (const key of ['SUITE', 'BENCH_REF', 'BENCH_SHA']) { + if (original[key] === undefined) delete process.env[key]; else process.env[key] = original[key]; + } + } +}); diff --git a/.github/bench/bindings.sh b/.github/bench/bindings.sh index e0170e257b..fc9c0245e2 100644 --- a/.github/bench/bindings.sh +++ b/.github/bench/bindings.sh @@ -5,7 +5,67 @@ mode=$1 binding=$2 core_dir=$PWD export CARGO_TARGET_DIR="$PWD/binding/target/p16" -export CARGO_BUILD_JOBS=2 +export CARGO_BUILD_JOBS=4 +[[ "$mode" = build || "$mode" = run ]] +if [[ "$binding" = wasi ]]; then + if [[ "$mode" = build ]]; then + test "$(git rev-parse HEAD)" = "$BENCH_SHA" + test -f benchmarks/bindings839/nightly.py + test -f benchmarks/bindings839/wasi.py + test -f benchmarks/bindings839/validate.py + # No binding repo. Build the CLI from this once-resolved core checkout. + export CARGO_TARGET_DIR="$core_dir/target" + cargo build --locked --release --target wasm32-wasip1 --bin minigraf + mkdir binding + cp target/wasm32-wasip1/release/minigraf.wasm binding/minigraf.wasm + curl -sSfL 'https://github.com/bytecodealliance/wasmtime/releases/download/v49.0.2/wasmtime-v49.0.2-x86_64-linux.tar.xz' \ + -o binding/wasmtime.tar.xz + tar -xf binding/wasmtime.tar.xz -C binding + echo 'd0a014e0d5b0cf48dd3549c38e2e6ecd78ff09fb3e3f10d751b87b72bdfd8635 binding/wasmtime-v49.0.2-x86_64-linux/wasmtime' | sha256sum --check --strict + else + shift 2 + python3 -I benchmarks/bindings839/nightly.py \ + --runtime binding/wasmtime-v49.0.2-x86_64-linux/wasmtime \ + --wasm binding/minigraf.wasm --core-dir "$core_dir" --output binding "$@" + fi + exit +fi +if [[ "$binding" = jvm ]]; then + # manual/build.sh and Gradle use target/release explicitly. It alone owns + # the patch, so do not create .cargo/config.toml before calling it. + export CARGO_TARGET_DIR="$core_dir/binding/target" + cd binding + test -f bench/nightly.sh + if [[ "$mode" = build ]]; then + test "$(git -C "$core_dir" rev-parse HEAD)" = "$BENCH_SHA" + CORE_REVISION="$BENCH_SHA" BINDING_REVISION="$(git rev-parse HEAD)" \ + bash bench/manual/build.sh "$core_dir" + cargo metadata --format-version 1 --locked > target-metadata.json + python3 -I - "$core_dir" <<'PY_METADATA' +import json, pathlib, sys +packages = json.load(open('target-metadata.json'))['packages'] +core = [p for p in packages if p['name'] == 'minigraf'] +assert len(core) == 1 and core[0]['source'] is None +assert pathlib.Path(core[0]['manifest_path']).parent.resolve() == pathlib.Path(sys.argv[1]).resolve() +PY_METADATA + else + shift 2 + # Contract in docs/BINDING_BENCHMARKS.md; failures must propagate unchanged. + test ! -e binding-results.json + test ! -e binding-provenance.json + if [[ "${1:-}" = --test ]]; then + test ! -e smoke + else + test ! -e benchmark + fi + bash bench/nightly.sh "$@" + if [[ "${1:-}" = --test ]]; then + test ! -e binding-results.json + test ! -e binding-provenance.json + fi + fi + exit +fi cd binding if [[ "$mode" = build ]]; then mkdir -p .cargo diff --git a/.github/bench/report.test.cjs b/.github/bench/report.test.cjs index bef897b2ee..290aadf08f 100644 --- a/.github/bench/report.test.cjs +++ b/.github/bench/report.test.cjs @@ -96,3 +96,17 @@ test('CLI retains provenance, IDs and summary on success and early failure', () assert.equal(JSON.parse(fs.readFileSync(path.join(cwd, 'bench-report.json'))).sha, env.BENCH_SHA); } finally { fs.rmSync(cwd, { recursive: true, force: true }); } }); + +for (const runtime of ['jvm', 'wasi']) test(`${runtime} requires selected = executed = uploaded`, () => { + const { expected } = require('./bindings-report.cjs'); + const selected = expected(runtime); + const execution = selected.map(id => `Benchmarking ${id}: Analyzing`).join('\n'); + const uploaded = selected.map(id => `- ${id} (Latency): https://bencher.dev/console/projects/minigraf/perf?x=1`).join('\n'); + assert.deepEqual(audit(selected, execution, uploaded, 'success').errors, []); + for (const bad of ['', execution + '\n' + execution, execution.split('\n').slice(1).join('\n')]) { + assert.ok(audit(selected, bad, uploaded, 'success').errors.length); + } + for (const bad of ['', uploaded + '\n' + uploaded, uploaded.split('\n').slice(1).join('\n')]) { + assert.ok(audit(selected, execution, bad, 'success').errors.length); + } +}); diff --git a/.github/workflows/bench-report.yml b/.github/workflows/bench-report.yml index ccdd53aa63..b0eafb2046 100644 --- a/.github/workflows/bench-report.yml +++ b/.github/workflows/bench-report.yml @@ -3,10 +3,10 @@ name: Benchmark reporting checks on: push: branches: [main, 'v[0-9]*'] - paths: ['.github/bench/**', '.github/workflows/bench*.yml'] + paths: ['.github/bench/**', '.github/workflows/bench*.yml', 'benchmarks/bindings839/**'] pull_request: branches: [main, 'v[0-9]*'] - paths: ['.github/bench/**', '.github/workflows/bench*.yml'] + paths: ['.github/bench/**', '.github/workflows/bench*.yml', 'benchmarks/bindings839/**'] permissions: contents: read @@ -20,3 +20,5 @@ jobs: with: node-version: '24' - run: node --test .github/bench/*.test.cjs + - run: python3 -I -m unittest discover -s benchmarks/bindings839 -p 'test_*.py' + if: ${{ hashFiles('benchmarks/bindings839/test_*.py') != '' }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 558a63d6c9..e9c0841b43 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -21,16 +21,22 @@ on: default: main binding-ref: - description: Binding ref for all P16 repos (phase B may select the PR branch) + description: Common binding ref (use jvm-ref for a JVM-only experiment) required: false default: v3 + jvm-ref: + description: Override only minigraf-java (other bindings keep binding-ref) + required: false + default: '' + permissions: contents: read issues: write actions: read # read the previous run's job logs to confirm an alert repeats env: + CARGO_BUILD_JOBS: "4" # One threshold per branch/testbed/measure; never race per-suite models. # 0.999 retains the noisy file/concurrency setting and existing testbed history. BENCH_THRESHOLD: "0.999" @@ -568,9 +574,11 @@ jobs: - { binding: browser, repo: minigraf-wasm } - { binding: node, repo: minigraf-node } - { binding: c, repo: minigraf-c } + - { binding: jvm, repo: minigraf-java } + - { binding: wasi } env: BINDING: ${{ matrix.binding }} - BINDING_REF: ${{ inputs.binding-ref || 'v3' }} + BINDING_REF: ${{ matrix.binding == 'wasi' && 'v3' || (matrix.binding == 'jvm' && inputs.jvm-ref) || inputs.binding-ref || 'v3' }} BENCH_SHA: ${{ needs.compile.outputs.sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} steps: @@ -580,10 +588,19 @@ jobs: - name: Check v3 binding suite availability id: source run: | - if [ -f benches/minigraf_bench/prepared.rs ]; then + # A missing harness on v3 is an error, never a successful empty run. + if [ "$BENCH_REF" = v3 ] || [ -f src/query/engine/mod.rs ]; then + test -f benches/minigraf_bench/prepared.rs + if [ "$BINDING" = jvm ] || [ "$BINDING" = wasi ]; then + test -f benchmarks/bindings839/validate.py + fi + if [ "$BINDING" = wasi ]; then + test -f benchmarks/bindings839/nightly.py + test -f benchmarks/bindings839/wasi.py + fi echo 'enabled=true' >> "$GITHUB_OUTPUT" else - echo 'Skipping v3-only binding jobs for this measured checkout.' >> "$GITHUB_STEP_SUMMARY" + echo 'Skipping v3-only binding jobs for this measured main/v2 checkout.' >> "$GITHUB_STEP_SUMMARY" fi - uses: actions/checkout@v7 if: steps.source.outputs.enabled == 'true' @@ -591,29 +608,32 @@ jobs: ref: ${{ github.workflow_sha }} path: .bench-workflow sparse-checkout-cone-mode: false - sparse-checkout: /.github/bench/ + sparse-checkout: | + /.github/bench/ persist-credentials: false - uses: actions/checkout@v7 - if: steps.source.outputs.enabled == 'true' + if: steps.source.outputs.enabled == 'true' && matrix.binding != 'wasi' with: repository: project-minigraf/${{ matrix.repo }} - ref: ${{ inputs.binding-ref || 'v3' }} + ref: ${{ env.BINDING_REF }} path: binding persist-credentials: false - name: Pin binding source if: steps.source.outputs.enabled == 'true' run: | set -o pipefail - git -C binding rev-parse HEAD | tee binding-sha.txt + if [ "$BINDING" != wasi ]; then + git -C binding rev-parse HEAD | tee binding-sha.txt + fi history="$BENCH_REF" - if [ "$BINDING_REF" != v3 ]; then + if [ "$BINDING_REF" != v3 ] || [ "$BENCH_REF" != v3 ]; then history="binding-experiment/$BENCH_REF/$BINDING_REF" fi echo "BINDING_HISTORY=$history" >> "$GITHUB_ENV" - uses: dtolnay/rust-toolchain@stable if: steps.source.outputs.enabled == 'true' with: - targets: wasm32-unknown-unknown + targets: wasm32-unknown-unknown, wasm32-wasip1 - uses: actions/setup-python@v6 if: steps.source.outputs.enabled == 'true' with: @@ -622,6 +642,11 @@ jobs: if: steps.source.outputs.enabled == 'true' with: node-version: '20' + - uses: actions/setup-java@v5 + if: steps.source.outputs.enabled == 'true' && matrix.binding == 'jvm' + with: + distribution: temurin + java-version: '17.0.17+10' - name: Build binding with measured core if: steps.source.outputs.enabled == 'true' run: bash .bench-workflow/.github/bench/bindings.sh build "$BINDING" @@ -652,7 +677,7 @@ jobs: --threshold-max-sample-size 30 --threshold-upper-boundary "$BENCH_THRESHOLD" \ --err --adapter json --file binding/binding-results.json 2>&1 | tee binding-bencher.txt - name: Report regression - if: steps.bencher.outcome == 'failure' && env.BINDING_REF == 'v3' + if: steps.bencher.outcome == 'failure' && env.BINDING_REF == 'v3' && env.BENCH_REF == 'v3' uses: actions/github-script@v9 env: SUITE: binding_${{ matrix.binding }} @@ -685,9 +710,9 @@ jobs: `**Run:** ${runUrl}`, ].join('\n'); } else { - // Find this suite's job in the previous completed run of the same - // branch (by run name, see run-name) and read its alerts. Runs - // from before the branch was in the run name count as main. + // Search at most 20 completed runs for this suite's most recent + // production history. A JVM-only experiment shares the v3 run + // title but must not hide an earlier production repetition. let previous = new Set(); let previousUrl = null; try { @@ -702,10 +727,8 @@ jobs: const m = /\(([^)]+)\)$/.exec(r.display_title || ''); return m ? m[1] : 'main'; }; - const prevRun = runs.workflow_runs.find( - r => r.id !== context.runId && runRef(r) === ref); - if (prevRun) { - previousUrl = prevRun.html_url; + for (const prevRun of runs.workflow_runs) { + if (prevRun.id >= context.runId || runRef(prevRun) !== ref) continue; const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({ owner, repo, run_id: prevRun.id, per_page: 100, }); @@ -714,9 +737,19 @@ jobs: const { data: log } = await github.rest.actions.downloadJobLogsForWorkflowRun({ owner, repo, job_id: prevJob.id, }); - previous = parseAlerts(typeof log === 'string' ? log : Buffer.from(log).toString('utf8')); + const text = typeof log === 'string' ? log : Buffer.from(log).toString('utf8'); + // A JVM-only experiment can share this run title. Its + // alerts must never count as a production repetition. + const testbed = `binding-${suite.slice('binding_'.length)}-ubuntu24-v1`; + if (text.includes(`History branch: ${ref}; testbed: ${testbed};`)) { + previous = parseAlerts(text); + previousUrl = prevRun.html_url; + // A clean production run breaks the streak too. + break; + } } } + if (!previousUrl) core.warning('No matching production history in the last 20 completed runs; using an empty prior alert set.'); } catch (err) { core.warning(`Could not read the previous run's alerts: ${err.message}`); } @@ -808,3 +841,8 @@ jobs: binding-report.json binding/binding-results.json binding/target-metadata.json + binding/binding-provenance.json + binding/benchmark/ + binding/smoke/ + binding/minigraf.wasm + binding/bench/manual/build/evidence/