From 7b5ad191028533151e0ca89a6953cb73005961dc Mon Sep 17 00:00:00 2001 From: bryan Date: Sun, 4 Oct 2026 15:04:56 -0700 Subject: [PATCH 01/16] node: rebuild undici's llhttp wasm from source against a new wasi-libc wasi-libc (wasi-sdk-21) is built with our clang; node's build recompiles the llhttp blobs in deps/undici and npm's undici, splices them in, and fails on any wasm it did not rebuild. Co-Authored-By: Claude Opus 5.5 --- packages/node/build.ncl | 12 + packages/node/build.sh | 34 +- packages/node/rebuild-llhttp-wasm.py | 898 +++++++++++++++++++++++++++ packages/wasi-libc/build.ncl | 69 ++ packages/wasi-libc/build.sh | 38 ++ 5 files changed, 1050 insertions(+), 1 deletion(-) create mode 100755 packages/node/rebuild-llhttp-wasm.py create mode 100644 packages/wasi-libc/build.ncl create mode 100755 packages/wasi-libc/build.sh diff --git a/packages/node/build.ncl b/packages/node/build.ncl index 8fd0230f9..3a384ac81 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -20,7 +20,11 @@ let zstd = import "../zstd/build.ncl" in let gtest = import "../gtest/build.ncl" in let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +# undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. +let npm-undici = "6.28.0" in let version = "26.10.0" in { name = "node", @@ -33,7 +37,15 @@ let version = "26.10.0" in # Adapts the Mach-O SEA injection to the LIEF 1.x Section API — see # the patch header. Build-time only: it rewrites the source tree. { file = "lief-1.x-macho-section-ctor.patch" } | Local, + # Rebuilds and splices undici's llhttp wasm blobs. + { file = "rebuild-llhttp-wasm.py" } | Local, + { + url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", + sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", + } | Source, base, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, make, pkgconf, patch, diff --git a/packages/node/build.sh b/packages/node/build.sh index 5ab00ba5e..c0ee75a05 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -17,6 +17,29 @@ grep -q "LIEF::MachO::Section::create(section_name, data)" src/node_sea_bin.cc | exit 1 } +# llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped +# base64 blobs. Every pin fails the build on a node bump until re-derived. +LH="python3 ../rebuild-llhttp-wasm.py" +$LH selftest +NPM_UNDICI=6.28.0 +NPM_LH=deps/npm/node_modules/undici/lib/llhttp +$LH check-tree --undici 8.10.2 --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version 8.10.2 --llhttp 9.3.1 --sysroot /usr/lib/wasi +tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +$LH splice deps/undici/undici.js ab4573a06c43574936dc98c3943b10a4ee32798e9797329e9f327e5f64115f54 ../wasm/llhttp.wasm +$LH splice deps/undici/undici.js 18403ee56fab07f89a845028bad358bb3ebcda213f8e2ec0989a991e9c33da69 ../wasm/llhttp_simd.wasm +$LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm +$LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm +# Never installed; amaro (swc wasm) is configured out below. +rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ + deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js +rm -r deps/amaro +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm" +$LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" \ + deps lib src + case $(uname -m) in x86_64) MARCH="-march=x86-64-v3" ;; aarch64) MARCH="-march=armv8-a" ;; @@ -28,8 +51,11 @@ export CXXFLAGS="${CFLAGS}" ./configure --prefix=/usr \ --with-intl=system-icu --shared-openssl --shared-zlib --shared-zstd --shared-sqlite --shared-libuv \ - --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-lief --shared-cares + --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-lief --shared-cares \ + --without-amaro make -j$(nproc) +$LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ + ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install # npm's compiled-in global prefix is /usr, which is the read-only package @@ -38,3 +64,9 @@ make DESTDIR=$OUTPUT_DIR install # user config) pointing globals at ~/.local, whose bin/ is already on the # session PATH. See gominimal/inbox#559. printf 'prefix=~/.local\n' > "$OUTPUT_DIR/usr/lib/node_modules/npm/npmrc" + +# Installed wasm is exactly the rebuilt blobs: js2c'd into node, verbatim in npm's undici. +$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js=1 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp_simd-wasm.js=1 \ + usr diff --git a/packages/node/rebuild-llhttp-wasm.py b/packages/node/rebuild-llhttp-wasm.py new file mode 100755 index 000000000..202d3eb73 --- /dev/null +++ b/packages/node/rebuild-llhttp-wasm.py @@ -0,0 +1,898 @@ +#!/usr/bin/env python3 +"""Regenerate undici's llhttp wasm from in-tree C and replace the shipped blobs. + + check-tree node tree tripwires: pinned undici / npm-undici versions, no cjs lexer wasm + build pin build/wasm.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes + splice decode the pinned shipped blob in a carrier, gate tables+features, replace it once + census every wasm carrier under the roots is an allowed sha or on an allowed path + smoke instantiate blobs under a built node and parse a POST; optional loopback fetch + selftest synthetic tree, proves every gate both passes and fails + +stdlib only. Exit 1 on any mismatch. +""" +import argparse +import base64 +import binascii +import collections +import fnmatch +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +MAGIC = b"\0asm\x01\0\0\0" + +# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v8.10.2. +WASM_JS_PINNED = ( + "let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi'", + "WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor'", + "WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all'", + "WASM_LDFLAGS += ' -Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table'", + "WASM_LDFLAGS += ' -Wl,--export=malloc -Wl,--export=free -Wl,--no-entry'", + "const WASM_OPT_FLAGS = '-O4 --converge --strip-debug --strip-dwarf --strip-producers'", +) +CFLAGS = "-Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor".split() +LDFLAGS = ("-Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all " + "-Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table " + "-Wl,--export=malloc -Wl,--export=free -Wl,--no-entry").split() +# clang 17's "generic" cpu; clang 21's generic adds bulk-memory, multivalue, reference-types, +# nontrapping-fptoint and call-indirect-overlong, none of which the shipped blobs use. +CPU = ["-mcpu=mvp", "-mmutable-globals"] +LLHTTP_SRCS = ("api.c", "http.c", "llhttp.c") +GATE_ALLOWED = {"mutable-globals"} + +B64_ALIGNED = re.compile(rb"AGFzbQ[A-Za-z0-9+/]*={0,2}") + + +class Fail(Exception): + pass + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +def log(msg): + print(msg, flush=True) + + +# ---------------------------------------------------------------- wasm reader + +def uleb(b, i, maxbytes=5): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated leb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + return r, i + if n >= maxbytes: + raise Fail("leb128 too long") + + +def uleb_len(b, i): + v, j = uleb(b, i) + return v, j, j - i + + +def sleb(b, i, maxbytes=10): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated sleb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + if x & 0x40: + r -= 1 << s + return r, i + if n >= maxbytes: + raise Fail("sleb128 too long") + + +def name(b, i): + n, i = uleb(b, i) + return b[i:i + n].decode("utf-8", "replace"), i + n + + +VALTYPES = {0x7F: "i32", 0x7E: "i64", 0x7D: "f32", 0x7C: "f64", 0x7B: "v128", 0x70: "funcref", 0x6F: "externref"} + + +def valtype(b, i, feats): + t = b[i] + if t not in VALTYPES: + raise Fail("unknown valtype 0x%02x" % t) + if t == 0x7B: + feats["simd128"] += 1 + elif t in (0x70, 0x6F): + feats["reference-types"] += 1 + return VALTYPES[t], i + 1 + + +def limits(b, i, feats): + flag = b[i] + i += 1 + if flag & ~1: + feats["limits-flag-%d" % flag] += 1 + _, i = uleb(b, i, 10) + if flag & 1: + _, i = uleb(b, i, 10) + return i + + +def walk_ops(b, i, end, feats, const_expr=False): + """Walk instructions in b[i:end]; count non-MVP features; unknown opcodes fail closed.""" + while i < end: + op = b[i] + i += 1 + if op == 0x0B and const_expr: + return i + if op in (0x00, 0x01, 0x05, 0x0B, 0x0F, 0x1A, 0x1B, 0xD1) or 0x45 <= op <= 0xBF: + if op == 0xD1: + feats["reference-types"] += 1 + elif op in (0x02, 0x03, 0x04): + bt = b[i] + if bt == 0x40 or bt in VALTYPES: + if bt == 0x7B: + feats["simd128"] += 1 + i += 1 + else: + _, i = sleb(b, i, 5) + feats["multivalue"] += 1 + elif op in (0x0C, 0x0D, 0x10, 0x20, 0x21, 0x22, 0x23, 0x24): + _, i = uleb(b, i) + elif op in (0x25, 0x26, 0xD2): + _, i = uleb(b, i) + feats["reference-types"] += 1 + elif op == 0x0E: + n, i = uleb(b, i) + for _ in range(n + 1): + _, i = uleb(b, i) + elif op == 0x11: + _, i = uleb(b, i) + tbl, i, ln = uleb_len(b, i) + if ln > 1: + feats["call-indirect-overlong"] += 1 + if tbl != 0: + feats["reference-types"] += 1 + elif op == 0x1C: + n, i = uleb(b, i) + for _ in range(n): + _, i = valtype(b, i, feats) + feats["reference-types"] += 1 + elif 0x28 <= op <= 0x3E: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif op in (0x3F, 0x40): + if b[i] != 0: + feats["multi-memory"] += 1 + _, i = uleb(b, i) + elif op == 0x41: + _, i = sleb(b, i, 5) + elif op == 0x42: + _, i = sleb(b, i, 10) + elif op == 0x43: + i += 4 + elif op == 0x44: + i += 8 + elif 0xC0 <= op <= 0xC4: + feats["sign-ext"] += 1 + elif op == 0xD0: + i += 1 + feats["reference-types"] += 1 + elif op == 0xFC: + sub, i = uleb(b, i) + if sub <= 7: + feats["nontrapping-fptoint"] += 1 + elif sub <= 14: + feats["bulk-memory"] += 1 + nimm = {8: 2, 9: 1, 10: 2, 11: 1, 12: 2, 13: 1, 14: 2}[sub] + for _ in range(nimm): + _, i = uleb(b, i) + elif sub <= 17: + feats["reference-types"] += 1 + _, i = uleb(b, i) + else: + raise Fail("unknown opcode 0xfc %d" % sub) + elif op == 0xFD: + sub, i = uleb(b, i) + feats["simd128"] += 1 + if sub <= 11 or sub in (92, 93): + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif sub in (12, 13): + i += 16 + elif 21 <= sub <= 34: + i += 1 + elif 84 <= sub <= 91: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + i += 1 + elif sub >= 0x100: + raise Fail("relaxed-simd opcode 0xfd %d" % sub) + else: + raise Fail("unknown opcode 0x%02x" % op) + if const_expr: + raise Fail("const expr without end") + return i + + +def wasm_info(b): + if b[:8] != MAGIC: + raise Fail("not a wasm v1 module") + feats = collections.Counter() + types, funcs, imports, exports, ids = [], [], set(), set(), [] + import_funcs = [] + i = 8 + while i < len(b): + sid = b[i] + sz, i = uleb(b, i + 1) + end = i + sz + if end > len(b): + raise Fail("section %d overruns module" % sid) + s, j = b, i + ids.append(sid) + if sid == 0: + nm, _ = name(s, j) + feats["custom-section:" + nm] += 1 + elif sid == 1: + n, j = uleb(s, j) + for _ in range(n): + if s[j] != 0x60: + raise Fail("unknown type form 0x%02x" % s[j]) + j += 1 + sig = [] + for _part in range(2): + k, j = uleb(s, j) + vs = [] + for _ in range(k): + v, j = valtype(s, j, feats) + vs.append(v) + sig.append(vs) + if len(sig[1]) > 1: + feats["multivalue"] += 1 + types.append("(%s)->(%s)" % (",".join(sig[0]), ",".join(sig[1]))) + elif sid == 2: + n, j = uleb(s, j) + for _ in range(n): + mod, j = name(s, j) + fld, j = name(s, j) + kind = s[j] + j += 1 + if kind == 0: + t, j = uleb(s, j) + import_funcs.append(t) + imports.add((mod, fld, "func", types[t])) + elif kind == 1: + _, j = valtype(s, j, feats) + j = limits(s, j, feats) + imports.add((mod, fld, "table", "")) + elif kind == 2: + j = limits(s, j, feats) + imports.add((mod, fld, "memory", "")) + elif kind == 3: + v, j = valtype(s, j, feats) + if s[j]: + feats["mutable-globals"] += 1 + j += 1 + imports.add((mod, fld, "global", v)) + else: + raise Fail("unknown import kind %d" % kind) + elif sid == 3: + n, j = uleb(s, j) + for _ in range(n): + t, j = uleb(s, j) + funcs.append(t) + elif sid == 6: + n, j = uleb(s, j) + for _ in range(n): + _, j = valtype(s, j, feats) + j += 1 + j = walk_ops(s, j, end, feats, const_expr=True) + elif sid == 7: + n, j = uleb(s, j) + for _ in range(n): + nm, j = name(s, j) + kind = s[j] + idx, j = uleb(s, j + 1) + kname = {0: "func", 1: "table", 2: "memory", 3: "global"}.get(kind) + if kname is None: + raise Fail("unknown export kind %d" % kind) + desc = "" + if kind == 0: + ni = len(import_funcs) + desc = types[import_funcs[idx] if idx < ni else funcs[idx - ni]] + exports.add((nm, kname, desc)) + elif sid == 9: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["elem-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + for _ in range(k): + _, j = uleb(s, j) + elif sid == 10: + n, j = uleb(s, j) + for _ in range(n): + bsz, j = uleb(s, j) + bend = j + bsz + ng, j = uleb(s, j) + for _ in range(ng): + _, j = uleb(s, j) + _, j = valtype(s, j, feats) + j = walk_ops(s, j, bend, feats) + if j != bend or s[bend - 1] != 0x0B: + raise Fail("function body did not parse to its end") + elif sid == 11: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["data-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + j += k + elif sid == 12: + feats["bulk-memory"] += 1 + elif sid in (4, 5, 8): + pass + else: + feats["section-%d" % sid] += 1 + i = end + return {"imports": imports, "exports": exports, "feats": feats, "ids": set(ids)} + + +def gate(shipped, rebuilt, label): + """Rebuilt blob must present the shipped blob's tables and no extra wasm features.""" + a, r = wasm_info(shipped), wasm_info(rebuilt) + errs = [] + for k in ("imports", "exports"): + if a[k] != r[k]: + errs.append("%s differ: missing %s, extra %s" % (k, sorted(a[k] - r[k]), sorted(r[k] - a[k]))) + simd = "simd128" in a["feats"] + if simd != ("simd128" in r["feats"]): + errs.append("simd128 use differs: shipped %s, rebuilt %s" % (simd, not simd)) + allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) + extra = sorted(set(r["feats"]) - allowed) + if extra: + errs.append("features not allowed: %s" % ", ".join("%s x%d" % (f, r["feats"][f]) for f in extra)) + if r["ids"] - a["ids"]: + errs.append("sections not in shipped blob: %s" % sorted(r["ids"] - a["ids"])) + if errs: + raise Fail("%s: gate failed\n " % label + "\n ".join(errs)) + log("@@GATE ok %s imports=%d exports=%d feats=%s" % ( + label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}")) + + +# ---------------------------------------------------------------- carriers + +def b64_decode(run): + try: + return base64.b64decode(run, validate=True) + except (binascii.Error, ValueError): + return None + + +def _misaligned_patterns(): + out = [] + for k in (1, 2): + enc = base64.b64encode(b"\0" * k + MAGIC) + lo, hi = 8 * k, 8 * k + 8 * len(MAGIC) + first, last = -(-lo // 6), hi // 6 - 1 + out.append(enc[first:last + 1]) + return out + + +MISALIGNED = _misaligned_patterns() +OTHER_FORMS = ( + ("js-bytes", re.compile(rb"(? (p, at, len) => { seen[k] = (seen[k] || '') + Buffer.from(mem.buffer, at, len).toString(); return 0; }; + let mem; + const env = { + wasm_on_message_begin: () => 0, wasm_on_status: span('status'), + wasm_on_url: span('url'), wasm_on_header_field: span('hf'), wasm_on_header_value: span('hv'), + wasm_on_headers_complete: () => 0, wasm_on_body: span('body'), + wasm_on_message_complete: () => { seen.done = true; return 0; }, + }; + const { exports: e } = new WebAssembly.Instance(new WebAssembly.Module(fs.readFileSync(f)), { env }); + mem = e.memory; + e._initialize(); + const p = e.llhttp_alloc(1); + const buf = e.malloc(req.length); + new Uint8Array(mem.buffer, buf, req.length).set(req); + const rc = e.llhttp_execute(p, buf, req.length); + if (rc !== 0 || seen.url !== '/x?y=1' || seen.body !== 'hello' || !seen.done) { + console.error('@@SMOKE FAIL', f, rc, JSON.stringify(seen)); process.exit(1); + } + e.free(buf); e.llhttp_free(p); + console.log('@@SMOKE ok llhttp_execute', f); +} +""" + +FETCH_JS = r""" +const http = require('node:http'); +const s = http.createServer((q, r) => { let b = ''; q.on('data', (d) => b += d); q.on('end', () => r.end('echo:' + b)); }); +s.on('error', (e) => { + if (['EPERM', 'EACCES', 'EADDRNOTAVAIL'].includes(e.code)) { console.log('@@SMOKE fetch SKIP no loopback', e.code); process.exit(0); } + console.error('@@SMOKE FAIL listen', e); process.exit(1); +}); +s.listen(0, '127.0.0.1', async () => { + try { + const r = await fetch(`http://127.0.0.1:${s.address().port}/x`, { method: 'POST', body: 'hello' }); + const t = await r.text(); + if (t !== 'echo:hello') throw new Error('body ' + t); + console.log('@@SMOKE ok fetch'); s.close(); + } catch (e) { console.error('@@SMOKE FAIL fetch', e); process.exit(1); } +}); +""" + + +def smoke(node, wasms, fetch, requires): + subprocess.run([node, "-e", SMOKE_JS] + list(wasms), check=True) + for r in requires: + subprocess.run([node, "-e", "require(%s); console.log('@@SMOKE ok require', %s)" % ( + json.dumps(os.path.abspath(r)), json.dumps(r))], check=True) + if fetch: + subprocess.run([node, "-e", FETCH_JS], check=True) + + +# ---------------------------------------------------------------- selftest + +def _sec(sid, body): + return bytes([sid]) + _leb(len(body)) + body + + +def _leb(n): + out = bytearray() + while True: + x = n & 0x7F + n >>= 7 + if n: + out.append(x | 0x80) + else: + out.append(x) + return bytes(out) + + +def _str(s): + return _leb(len(s)) + s.encode() + + +def _vec(items): + return _leb(len(items)) + b"".join(items) + + +def mkwasm(imports=("wasm_on_url",), exports=("malloc", "llhttp_execute"), body=b"\x41\x00\x0b", simd=False): + """Tiny module: imports env. (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" + t3 = b"\x60\x03\x7f\x7f\x7f\x01\x7f" + types = _sec(1, _vec([t3])) + imp = _sec(2, _vec([_str("env") + _str(n) + b"\x00\x00" for n in imports])) + nf = len(exports) + fn = _sec(3, _vec([b"\x00"] * nf)) + mem = _sec(5, _vec([b"\x00\x01"])) + exp = _sec(7, _vec([_str("memory") + b"\x02\x00"] + + [_str(n) + b"\x00" + _leb(len(imports) + k) for k, n in enumerate(exports)])) + if simd: + body = b"\xfd\x0c" + b"\x00" * 16 + b"\x1a" + body + code1 = _leb(0) + body + code = _sec(10, _vec([_leb(len(code1)) + code1] * nf)) + return MAGIC + types + imp + fn + mem + exp + code + + +def _expect_fail(fn, *a, contains=""): + try: + fn(*a) + except Fail as e: + if contains and contains not in str(e): + raise AssertionError("failed for the wrong reason: %s" % e) + log(" (expected failure: %s)" % str(e).splitlines()[0]) + return + raise AssertionError("%s%r should have failed" % (fn.__name__, a)) + + +def selftest(): + shipped = mkwasm() + shipped_simd = mkwasm(simd=True) + good = mkwasm(body=b"\x41\x01\x0b") + good_simd = mkwasm(body=b"\x41\x01\x0b", simd=True) + bad_import = mkwasm(imports=("wasm_on_url", "__multi3"), body=b"\x41\x02\x0b") + bad_export = mkwasm(exports=("malloc",), body=b"\x41\x03\x0b") + bad_bulk = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\xfc\x0b\x00\x41\x00\x0b") + bad_overlong = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\x41\x00\x11\x00\x80\x80\x80\x80\x00\x0b") + bad_unknown = mkwasm(body=b"\x06\x40\x0b\x41\x00\x0b") + bad_signext = mkwasm(body=b"\x41\x00\xc0\x0b") + pins = {k: sha(v) for k, v in (("plain", shipped), ("simd", shipped_simd))} + + log("selftest: gate") + gate(shipped, good, "good") + _expect_fail(gate, shipped, bad_signext, "sign-ext", contains="sign-ext") + gate(shipped_simd, good_simd, "good simd") + _expect_fail(gate, shipped, bad_import, "bad import", contains="imports differ") + _expect_fail(gate, shipped, bad_export, "bad export", contains="exports differ") + _expect_fail(gate, shipped, bad_bulk, "bulk-memory", contains="bulk-memory") + _expect_fail(gate, shipped, bad_overlong, "overlong", contains="call-indirect-overlong") + _expect_fail(gate, shipped, good_simd, "simd in plain", contains="simd128") + _expect_fail(gate, shipped_simd, good, "no simd in simd", contains="simd128") + _expect_fail(wasm_info, bad_unknown, contains="unknown opcode") + _expect_fail(wasm_info, b"\0asm\x02\0\0\0", contains="not a wasm v1") + + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as td: + def tree(): + root = os.path.join(td, "t%d" % len(os.listdir(td))) + u = os.path.join(root, "deps/undici") + n = os.path.join(root, "deps/npm/node_modules/undici/lib/llhttp") + os.makedirs(n) + os.makedirs(u, exist_ok=True) + os.makedirs(os.path.join(root, "deps/v8/third_party/wasm-api/example")) + with open(os.path.join(u, "undici.js"), "w") as f: + f.write('var wasmBase64 = "%s";\n// next\nvar wasmBase64 = "%s";\n' % ( + base64.b64encode(shipped).decode(), base64.b64encode(shipped_simd).decode())) + with open(os.path.join(n, "llhttp-wasm.js"), "w") as f: + f.write("module.exports = Buffer.from('%s', 'base64')\n" % base64.b64encode(shipped).decode()) + with open(os.path.join(root, "deps/v8/third_party/wasm-api/example/hello.wasm"), "wb") as f: + f.write(mkwasm(exports=("run",))) + for name, b in (("good.wasm", good), ("good_simd.wasm", good_simd), ("bad_bulk.wasm", bad_bulk)): + with open(os.path.join(root, name), "wb") as f: + f.write(b) + return root + + def full_pass(root): + os.chdir(root) + splice("deps/undici/undici.js", pins["plain"], "good.wasm") + splice("deps/undici/undici.js", pins["simd"], "good_simd.wasm") + splice("deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js", pins["plain"], "good.wasm") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}) + + try: + log("selftest: splice + census pass") + full_pass(tree()) + + log("selftest: N+ expectations") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (1, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (3, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}, + contains="expected 3+") + + log("selftest: splice rejects") + root = tree() + os.chdir(root) + _expect_fail(splice, "deps/undici/undici.js", "0" * 64, "good.wasm", contains="found 0 times") + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "bad_bulk.wasm", contains="bulk-memory") + before = open("deps/undici/undici.js", "rb").read() + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good_simd.wasm", contains="simd128") + assert open("deps/undici/undici.js", "rb").read() == before, "failed splice modified the carrier" + with open("deps/undici/undici.js", "a") as f: + f.write('var again = "%s";\n' % base64.b64encode(shipped).decode()) + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good.wasm", contains="found 2 times") + + log("selftest: census rejects") + for label, mutate, why in ( + ("unspliced shipped blob", lambda: None, "base64 deps/undici/undici.js"), + ("rogue base64", lambda: open("deps/rogue.js", "w").write( + "x='%s'" % base64.b64encode(mkwasm(exports=("evil",))).decode()), "deps/rogue.js"), + ("raw wasm", lambda: open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))), "raw deps/x.wasm"), + ("embedded raw", lambda: open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"..."), "raw deps/x.bin"), + ("misaligned base64", lambda: open("deps/m.js", "w").write( + "y='%s'" % base64.b64encode(b"Q" + mkwasm()).decode()), "base64-misaligned deps/m.js"), + ("js byte array", lambda: open("deps/a.js", "w").write("[0, 97, 115, 109, 1, 0, 0, 0]"), "js-bytes"), + ("truncated base64", lambda: open("deps/t.js", "w").write("z='AGFzbQEAAAABJw'"), "undecodable"), + ("expect count", lambda: open("deps/undici/undici.js", "a").write( + "var dup = '%s';\n" % base64.b64encode(good).decode()), "expected 2 allowed blob(s)"), + ("unexpected carrier", lambda: open("deps/copy.js", "w").write( + "c='%s'" % base64.b64encode(good).decode()), "unexpected file deps/copy.js"), + ): + log(" case: " + label) + root = tree() + os.chdir(root) + if label != "unspliced shipped blob": + full_pass(root) + mutate() + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], + ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), + "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}, + contains=why) + + log("selftest: source pins") + src = os.path.join(td, "undici-src") + os.makedirs(os.path.join(src, "deps/llhttp/src")) + os.makedirs(os.path.join(src, "deps/llhttp/include")) + os.makedirs(os.path.join(src, "build")) + json.dump({"version": "8.10.2"}, open(os.path.join(src, "package.json"), "w")) + open(os.path.join(src, "deps/llhttp/include/llhttp.h"), "w").write( + "#define LLHTTP_VERSION_MAJOR 9\n#define LLHTTP_VERSION_MINOR 3\n#define LLHTTP_VERSION_PATCH 1\n") + for s in LLHTTP_SRCS: + open(os.path.join(src, "deps/llhttp/src", s), "w").write("") + open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + build(src, os.path.join(td, "out"), "8.10.2", "9.3.1", "clang", "/usr/share/wasi-sysroot", None, True) + _expect_fail(check_undici_src, src, "8.10.3", "9.3.1", contains="recipe pins 8.10.3") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.0", contains="recipe pins 9.3.0") + open(os.path.join(src, "deps/llhttp/src/extra.c"), "w").write("") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="src/*.c") + os.unlink(os.path.join(src, "deps/llhttp/src/extra.c")) + open(os.path.join(src, "build/wasm.js"), "w").write( + "\n".join(WASM_JS_PINNED).replace("-Ofast", "-O3") + "\n") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="flags drifted") + finally: + os.chdir(cwd) + + cmd = compile_cmd("clang", "/s", None, "o.wasm", True) + for f in CFLAGS + LDFLAGS + CPU + ["-msimd128", "-mexec-model=reactor", "-lc", "--no-wasm-opt"]: + assert f in cmd, f + log("@@SELFTEST PASS") + + +# ---------------------------------------------------------------- cli + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sp = ap.add_subparsers(dest="cmd", required=True) + p = sp.add_parser("check-tree") + p.add_argument("--undici", required=True) + p.add_argument("--npm-undici", required=True) + p = sp.add_parser("build") + p.add_argument("src") + p.add_argument("outdir") + p.add_argument("--version", required=True) + p.add_argument("--llhttp", required=True) + p.add_argument("--cc", default="clang") + p.add_argument("--sysroot", default="/usr/lib/wasi", help="packages/wasi-libc install root") + p.add_argument("--rt", help="explicit wasm32 builtins archive, if llhttp needs one") + p.add_argument("--dry-run", action="store_true") + p = sp.add_parser("splice") + p.add_argument("carrier") + p.add_argument("shipped_sha") + p.add_argument("rebuilt") + p = sp.add_parser("census") + p.add_argument("roots", nargs="+") + p.add_argument("-C", dest="chdir") + p.add_argument("--wasm", action="append", default=[], help="rebuilt blob whose sha is allowed") + p.add_argument("--allow-path", action="append", default=[], help="fnmatch glob, relative to -C") + p.add_argument("--expect", action="append", default=[], help="PATH=N (exact) or PATH=N+ allowed blobs in PATH") + p = sp.add_parser("smoke") + p.add_argument("node") + p.add_argument("wasm", nargs="+") + p.add_argument("--fetch", action="store_true") + p.add_argument("--require", action="append", default=[]) + sp.add_parser("selftest") + a = ap.parse_args(argv) + try: + if a.cmd == "check-tree": + check_tree(a.undici, a.npm_undici) + elif a.cmd == "build": + build(a.src, a.outdir, a.version, a.llhttp, a.cc, a.sysroot, a.rt, a.dry_run) + elif a.cmd == "splice": + splice(a.carrier, a.shipped_sha, a.rebuilt) + elif a.cmd == "census": + shas = [sha(open(w, "rb").read()) for w in a.wasm] + if a.chdir: + os.chdir(a.chdir) + expect = {} + for e in a.expect: + k, _, v = e.rpartition("=") + expect[os.path.normpath(k)] = (int(v.rstrip("+")), v.endswith("+")) + census(a.roots, shas, a.allow_path, expect) + elif a.cmd == "smoke": + smoke(a.node, a.wasm, a.fetch, a.require) + elif a.cmd == "selftest": + selftest() + except Fail as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/packages/wasi-libc/build.ncl b/packages/wasi-libc/build.ncl new file mode 100644 index 000000000..d9a147287 --- /dev/null +++ b/packages/wasi-libc/build.ncl @@ -0,0 +1,69 @@ +let { Attrs, BuildSpec, Local, OutputData, OutputLib, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let make = import "../make/build.ncl" in + +# wasi-libc has no releases of its own; it is tagged with the wasi-sdk release +# that ships it. wasi-sdk-21 is commit c5264e2b, packaged by Alpine as 0.20231215. +let tag = "wasi-sdk-21" in +{ + name = "wasi-libc", + + build_deps = [ + { file = "build.sh" } | Local, + { + url = "gs://minimal-staging-archives/wasi-libc-%{tag}.tar.gz", + sha256 = "4a2a3e3b120ba1163c57f34ac79c3de720a8355ee3a753d81f1f0c58c4cf6017", + extract = true, + strip_prefix = "wasi-libc-%{tag}", + } | Source, + base, + llvm, # clang, llvm-ar, llvm-nm + make, + ], + + cmd = "./build.sh", + outputs = { + # wasm32 objects and archives, not ELF. + libs = { glob = "usr/lib/wasi/lib/wasm32-wasi/*.{a,o}", allow_data = true } | OutputLib, + includes = { glob = "usr/lib/wasi/include/**" } | OutputData, + licenses = { glob = "usr/share/licenses/wasi-libc/*" } | OutputData, + }, + + attrs = + { + upstream_version = "0.20231215", + license_spdx = "(Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT) AND CC0-1.0 AND BSD-2-Clause AND MIT", + repology_project = "wasi-libc", + source_provenance = { + category = 'GithubRepo, + owner = "WebAssembly", + repo = "wasi-libc", + }, + } | Attrs, + + tests = { + # memcpy/memset lowered to memory.copy/memory.fill means the build picked up + # clang 21's default CPU instead of the pinned feature set. + no_bulk_memory = + { + class = 'Standalone, + test_deps = [base, llvm], + cmds = [ + ["/bin/bash", "-c", "set -eu; L=/usr/lib/wasi/lib/wasm32-wasi; llvm-objdump -d $L/libc.a $L/crt1-reactor.o > dis.txt; test -s dis.txt; ! grep -Eq 'memory\\.(copy|fill)|i(32|64)\\.trunc_sat' dis.txt"], + ], + } | Test, + + # A reactor module exporting malloc/free, the shape llhttp is linked in. + # -nodefaultlibs skips the compiler-rt builtins clang otherwise adds. + reactor_link = + { + class = 'Standalone, + test_deps = [base, llvm], + cmds = [ + ["/bin/bash", "-c", "printf '#include \\nint f(char *d, const char *s, int n) { memcpy(d, s, n); return (int)strlen(d); }\\n' > t.c"], + ["/bin/bash", "-c", "set -eu; clang --target=wasm32-wasi --sysroot=/usr/lib/wasi -mcpu=mvp -mmutable-globals -O2 -mexec-model=reactor -nodefaultlibs -Wl,--export=malloc -Wl,--export=free -Wl,--export=f -Wl,--no-entry -Wl,--strip-all -o t.wasm t.c -lc; test \"$(head -c 4 t.wasm | od -An -tx1 | tr -d ' ')\" = 0061736d"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/wasi-libc/build.sh b/packages/wasi-libc/build.sh new file mode 100755 index 000000000..1d5acb2ab --- /dev/null +++ b/packages/wasi-libc/build.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -ex + +# clang 21's default wasm CPU turns on bulk-memory, multivalue, reference-types, +# nontrapping-fptoint and call-indirect-overlong; clang 17's enabled only +# sign-ext and mutable-globals. This pins the clang 17 feature set. +WASM_FEATURES="-mcpu=mvp -mmutable-globals" + +# clang 21 puts -Wunterminated-string-initialization in -Wextra; musl's +# vfprintf.c xdigits[16] trips it under the Makefile's -Werror. +NOWERR="-Wno-error=unterminated-string-initialization" + +# Empty BULK_MEMORY_SOURCES compiles musl's memcpy/memmove/memset without the +# per-file -mbulk-memory the Makefile adds. +# +# `finish` is not built: it depends on check-symbols, whose predefined-macro +# list stops at clang 18, so the steps it runs after `libc` are done below. +make -j"$(nproc)" \ + CC=clang \ + AR=llvm-ar \ + NM=llvm-nm \ + SYSROOT="$PWD/sysroot" \ + THREAD_MODEL=single \ + BULK_MEMORY_SOURCES= \ + EXTRA_CFLAGS="-O2 -DNDEBUG $WASM_FEATURES $NOWERR -ffile-prefix-map=$PWD=/builddir" \ + startup_files libc + +# The placeholder archives `finish` creates. +for name in m rt pthread crypt util xnet resolv; do + llvm-ar crs "sysroot/lib/wasm32-wasi/lib${name}.a" +done + +mkdir -p "$OUTPUT_DIR/usr/lib/wasi" +cp -r sysroot/include sysroot/lib "$OUTPUT_DIR/usr/lib/wasi/" + +mkdir -p "$OUTPUT_DIR/usr/share/licenses/wasi-libc" +cp LICENSE LICENSE-APACHE LICENSE-APACHE-LLVM LICENSE-MIT \ + "$OUTPUT_DIR/usr/share/licenses/wasi-libc/" From f08b49385819e8d5b0412ab780177c689616899f Mon Sep 17 00:00:00 2001 From: bryan Date: Sun, 4 Oct 2026 16:25:23 -0700 Subject: [PATCH 02/16] node: allow V8's two type-section-only test modules in the install census rebuild-llhttp-wasm.py census gains --allow-raw PATH=N: exactly N bare wasm headers that do not decode as a module from offset 0, each logged with hex. Co-Authored-By: Claude Opus 5.5 --- packages/node/build.sh | 3 ++- packages/node/rebuild-llhttp-wasm.py | 32 +++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/packages/node/build.sh b/packages/node/build.sh index c0ee75a05..4d506a5d2 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -66,7 +66,8 @@ make DESTDIR=$OUTPUT_DIR install printf 'prefix=~/.local\n' > "$OUTPUT_DIR/usr/lib/node_modules/npm/npmrc" # Installed wasm is exactly the rebuilt blobs: js2c'd into node, verbatim in npm's undici. -$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ \ +# V8 test runtime (runtime-test-wasm.cc) compiles in two type-section-only modules. +$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ --allow-raw usr/bin/node=2 \ --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js=1 \ --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp_simd-wasm.js=1 \ usr diff --git a/packages/node/rebuild-llhttp-wasm.py b/packages/node/rebuild-llhttp-wasm.py index 202d3eb73..8bc679ac2 100755 --- a/packages/node/rebuild-llhttp-wasm.py +++ b/packages/node/rebuild-llhttp-wasm.py @@ -460,9 +460,10 @@ def splice(path, pin, rebuilt_path): path, pin[:16], sha(new)[:16], " (identical to upstream)" if pin == sha(new) else "")) -def census(roots, allowed_shas, allow_paths, expect): +def census(roots, allowed_shas, allow_paths, expect, allow_raw=None): allowed_shas = set(allowed_shas) - fails, found = [], collections.Counter() + allow_raw = allow_raw or {} + fails, found, raws = [], collections.Counter(), collections.Counter() nfiles = 0 for root in roots: if not os.path.exists(root): @@ -484,6 +485,10 @@ def census(roots, allowed_shas, allow_paths, expect): if kind == "base64": found[p] += 1 log("@@CENSUS ok %s %s:%d %s" % (kind, p, off, h[:16])) + elif kind == "raw" and blob is None and p in allow_raw: + # a bare header inside a binary (V8 byte constants), not an extractable module + raws[p] += 1 + log("@@CENSUS raw-allowed %s:%d %s" % (p, off, data[max(0, off - 8):off + 24].hex())) elif path_ok: log("@@CENSUS allowed-path %s %s:%d %s" % (kind, p, off, (h or "-")[:16])) else: @@ -491,6 +496,9 @@ def census(roots, allowed_shas, allow_paths, expect): for p, (n, at_least) in sorted(expect.items()): if found[p] < n if at_least else found[p] != n: fails.append("expected %d%s allowed blob(s) in %s, found %d" % (n, "+" if at_least else "", p, found[p])) + for p, n in sorted(allow_raw.items()): + if raws[p] != n: + fails.append("expected %d raw header(s) in %s, found %d" % (n, p, raws[p])) if expect: for p in sorted(set(found) - set(expect)): if not any(fnmatch.fnmatch(p, g) for g in allow_paths): @@ -802,6 +810,19 @@ def full_pass(root): "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}, contains=why) + log("selftest: census allow-raw is an exact count") + root = tree() + os.chdir(root) + full_pass(root) + open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"........" + MAGIC + b"...") + exp = {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)} + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, {"deps/x.bin": 2}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.bin": 1}, contains="expected 1 raw header(s)") + open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.wasm": 1}, contains="raw deps/x.wasm") + log("selftest: source pins") src = os.path.join(td, "undici-src") os.makedirs(os.path.join(src, "deps/llhttp/src")) @@ -858,6 +879,7 @@ def main(argv): p.add_argument("--wasm", action="append", default=[], help="rebuilt blob whose sha is allowed") p.add_argument("--allow-path", action="append", default=[], help="fnmatch glob, relative to -C") p.add_argument("--expect", action="append", default=[], help="PATH=N (exact) or PATH=N+ allowed blobs in PATH") + p.add_argument("--allow-raw", action="append", default=[], help="PATH=N: exactly N undecodable raw wasm headers in PATH") p = sp.add_parser("smoke") p.add_argument("node") p.add_argument("wasm", nargs="+") @@ -880,7 +902,11 @@ def main(argv): for e in a.expect: k, _, v = e.rpartition("=") expect[os.path.normpath(k)] = (int(v.rstrip("+")), v.endswith("+")) - census(a.roots, shas, a.allow_path, expect) + raw = {} + for e in a.allow_raw: + k, _, v = e.rpartition("=") + raw[os.path.normpath(k)] = int(v) + census(a.roots, shas, a.allow_path, expect, raw) elif a.cmd == "smoke": smoke(a.node, a.wasm, a.fetch, a.require) elif a.cmd == "selftest": From 029f3df8783aa178150a6f10428962939e797f99 Mon Sep 17 00:00:00 2001 From: bryan Date: Sun, 4 Oct 2026 20:50:27 -0700 Subject: [PATCH 03/16] node-lts: rebuild undici's llhttp wasm from source against wasi-libc Same recipe change as node; undici 7.29.0 / npm undici 6.26.0. The node 24.19.0 Source moves to the staging mirror. Co-Authored-By: Claude Opus 5.5 --- packages/node-lts/build.ncl | 14 +- packages/node-lts/build.sh | 35 +- packages/node-lts/rebuild-llhttp-wasm.py | 924 +++++++++++++++++++++++ 3 files changed, 971 insertions(+), 2 deletions(-) create mode 100755 packages/node-lts/rebuild-llhttp-wasm.py diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index c7591387b..cd3058328 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -18,17 +18,29 @@ let zstd = import "../zstd/build.ncl" in let gtest = import "../gtest/build.ncl" in let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +# undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. +let npm-undici = "6.26.0" in let version = "24.19.0" in { name = "node-lts", build_deps = [ { file = "build.sh" } | Local, { - url = "https://nodejs.org/dist/v%{version}/node-v%{version}.tar.gz", + url = "gs://minimal-staging-archives/node-v%{version}.tar.gz", sha256 = "16fe258006a6e86844fbe05b3b5e1e5623ca8d3da54e32d98d9e83234bf25b01", } | Source, + # Rebuilds and splices undici's llhttp wasm blobs. + { file = "rebuild-llhttp-wasm.py" } | Local, + { + url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", + sha256 = "fc371d680e507ddf32f4944cf6b355a520f16ef5fc4987e0f25dc668372475c7", + } | Source, base, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, make, pkgconf, python, diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 0b7c81203..183a3b17b 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -6,6 +6,29 @@ export CC=gcc tar -xof "node-v${MINIMAL_ARG_VERSION}.tar.gz" cd "node-v${MINIMAL_ARG_VERSION}" +# llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped +# base64 blobs. Every pin fails the build on a node bump until re-derived. +LH="python3 ../rebuild-llhttp-wasm.py" +$LH selftest +NPM_UNDICI=6.26.0 +NPM_LH=deps/npm/node_modules/undici/lib/llhttp +$LH check-tree --undici 7.29.0 --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version 7.29.0 --llhttp 9.3.0 --sysroot /usr/lib/wasi +tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +$LH splice deps/undici/undici.js 57184a16144b5bc880e29105fa35d75c93dd318563e1b743e110de845e2ab661 ../wasm/llhttp.wasm +$LH splice deps/undici/undici.js 2434215e1f7fa0163f22ce895ddbeb83984fa0aa51e123164e053213e5aca028 ../wasm/llhttp_simd.wasm +$LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm +$LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm +# Never installed; amaro (swc wasm) is configured out below. +rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ + deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js +rm -r deps/amaro +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm" +$LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" \ + deps lib src + case $(uname -m) in x86_64) MARCH="-march=x86-64-v3" ;; aarch64) MARCH="-march=armv8-a" ;; @@ -17,9 +40,12 @@ export CXXFLAGS="${CFLAGS}" ./configure --prefix=/usr \ --with-intl=system-icu --shared-openssl --shared-zlib --shared-zstd --shared-sqlite --shared-libuv \ - --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-cares + --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-cares \ + --without-amaro # Note: --shared-lief is omitted; that configure option was not added until Node.js v25. make -j$(nproc) +$LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ + ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install # npm's compiled-in global prefix is /usr, which is the read-only package @@ -28,3 +54,10 @@ make DESTDIR=$OUTPUT_DIR install # user config) pointing globals at ~/.local, whose bin/ is already on the # session PATH. See gominimal/inbox#559. printf 'prefix=~/.local\n' > "$OUTPUT_DIR/usr/lib/node_modules/npm/npmrc" + +# Installed wasm is exactly the rebuilt blobs: js2c'd into node, verbatim in npm's undici. +# V8 test runtime (runtime-test-wasm.cc) compiles in two type-section-only modules. +$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ --allow-raw usr/bin/node=2 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js=1 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp_simd-wasm.js=1 \ + usr diff --git a/packages/node-lts/rebuild-llhttp-wasm.py b/packages/node-lts/rebuild-llhttp-wasm.py new file mode 100755 index 000000000..8bc679ac2 --- /dev/null +++ b/packages/node-lts/rebuild-llhttp-wasm.py @@ -0,0 +1,924 @@ +#!/usr/bin/env python3 +"""Regenerate undici's llhttp wasm from in-tree C and replace the shipped blobs. + + check-tree node tree tripwires: pinned undici / npm-undici versions, no cjs lexer wasm + build pin build/wasm.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes + splice decode the pinned shipped blob in a carrier, gate tables+features, replace it once + census every wasm carrier under the roots is an allowed sha or on an allowed path + smoke instantiate blobs under a built node and parse a POST; optional loopback fetch + selftest synthetic tree, proves every gate both passes and fails + +stdlib only. Exit 1 on any mismatch. +""" +import argparse +import base64 +import binascii +import collections +import fnmatch +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +MAGIC = b"\0asm\x01\0\0\0" + +# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v8.10.2. +WASM_JS_PINNED = ( + "let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi'", + "WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor'", + "WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all'", + "WASM_LDFLAGS += ' -Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table'", + "WASM_LDFLAGS += ' -Wl,--export=malloc -Wl,--export=free -Wl,--no-entry'", + "const WASM_OPT_FLAGS = '-O4 --converge --strip-debug --strip-dwarf --strip-producers'", +) +CFLAGS = "-Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor".split() +LDFLAGS = ("-Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all " + "-Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table " + "-Wl,--export=malloc -Wl,--export=free -Wl,--no-entry").split() +# clang 17's "generic" cpu; clang 21's generic adds bulk-memory, multivalue, reference-types, +# nontrapping-fptoint and call-indirect-overlong, none of which the shipped blobs use. +CPU = ["-mcpu=mvp", "-mmutable-globals"] +LLHTTP_SRCS = ("api.c", "http.c", "llhttp.c") +GATE_ALLOWED = {"mutable-globals"} + +B64_ALIGNED = re.compile(rb"AGFzbQ[A-Za-z0-9+/]*={0,2}") + + +class Fail(Exception): + pass + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +def log(msg): + print(msg, flush=True) + + +# ---------------------------------------------------------------- wasm reader + +def uleb(b, i, maxbytes=5): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated leb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + return r, i + if n >= maxbytes: + raise Fail("leb128 too long") + + +def uleb_len(b, i): + v, j = uleb(b, i) + return v, j, j - i + + +def sleb(b, i, maxbytes=10): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated sleb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + if x & 0x40: + r -= 1 << s + return r, i + if n >= maxbytes: + raise Fail("sleb128 too long") + + +def name(b, i): + n, i = uleb(b, i) + return b[i:i + n].decode("utf-8", "replace"), i + n + + +VALTYPES = {0x7F: "i32", 0x7E: "i64", 0x7D: "f32", 0x7C: "f64", 0x7B: "v128", 0x70: "funcref", 0x6F: "externref"} + + +def valtype(b, i, feats): + t = b[i] + if t not in VALTYPES: + raise Fail("unknown valtype 0x%02x" % t) + if t == 0x7B: + feats["simd128"] += 1 + elif t in (0x70, 0x6F): + feats["reference-types"] += 1 + return VALTYPES[t], i + 1 + + +def limits(b, i, feats): + flag = b[i] + i += 1 + if flag & ~1: + feats["limits-flag-%d" % flag] += 1 + _, i = uleb(b, i, 10) + if flag & 1: + _, i = uleb(b, i, 10) + return i + + +def walk_ops(b, i, end, feats, const_expr=False): + """Walk instructions in b[i:end]; count non-MVP features; unknown opcodes fail closed.""" + while i < end: + op = b[i] + i += 1 + if op == 0x0B and const_expr: + return i + if op in (0x00, 0x01, 0x05, 0x0B, 0x0F, 0x1A, 0x1B, 0xD1) or 0x45 <= op <= 0xBF: + if op == 0xD1: + feats["reference-types"] += 1 + elif op in (0x02, 0x03, 0x04): + bt = b[i] + if bt == 0x40 or bt in VALTYPES: + if bt == 0x7B: + feats["simd128"] += 1 + i += 1 + else: + _, i = sleb(b, i, 5) + feats["multivalue"] += 1 + elif op in (0x0C, 0x0D, 0x10, 0x20, 0x21, 0x22, 0x23, 0x24): + _, i = uleb(b, i) + elif op in (0x25, 0x26, 0xD2): + _, i = uleb(b, i) + feats["reference-types"] += 1 + elif op == 0x0E: + n, i = uleb(b, i) + for _ in range(n + 1): + _, i = uleb(b, i) + elif op == 0x11: + _, i = uleb(b, i) + tbl, i, ln = uleb_len(b, i) + if ln > 1: + feats["call-indirect-overlong"] += 1 + if tbl != 0: + feats["reference-types"] += 1 + elif op == 0x1C: + n, i = uleb(b, i) + for _ in range(n): + _, i = valtype(b, i, feats) + feats["reference-types"] += 1 + elif 0x28 <= op <= 0x3E: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif op in (0x3F, 0x40): + if b[i] != 0: + feats["multi-memory"] += 1 + _, i = uleb(b, i) + elif op == 0x41: + _, i = sleb(b, i, 5) + elif op == 0x42: + _, i = sleb(b, i, 10) + elif op == 0x43: + i += 4 + elif op == 0x44: + i += 8 + elif 0xC0 <= op <= 0xC4: + feats["sign-ext"] += 1 + elif op == 0xD0: + i += 1 + feats["reference-types"] += 1 + elif op == 0xFC: + sub, i = uleb(b, i) + if sub <= 7: + feats["nontrapping-fptoint"] += 1 + elif sub <= 14: + feats["bulk-memory"] += 1 + nimm = {8: 2, 9: 1, 10: 2, 11: 1, 12: 2, 13: 1, 14: 2}[sub] + for _ in range(nimm): + _, i = uleb(b, i) + elif sub <= 17: + feats["reference-types"] += 1 + _, i = uleb(b, i) + else: + raise Fail("unknown opcode 0xfc %d" % sub) + elif op == 0xFD: + sub, i = uleb(b, i) + feats["simd128"] += 1 + if sub <= 11 or sub in (92, 93): + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif sub in (12, 13): + i += 16 + elif 21 <= sub <= 34: + i += 1 + elif 84 <= sub <= 91: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + i += 1 + elif sub >= 0x100: + raise Fail("relaxed-simd opcode 0xfd %d" % sub) + else: + raise Fail("unknown opcode 0x%02x" % op) + if const_expr: + raise Fail("const expr without end") + return i + + +def wasm_info(b): + if b[:8] != MAGIC: + raise Fail("not a wasm v1 module") + feats = collections.Counter() + types, funcs, imports, exports, ids = [], [], set(), set(), [] + import_funcs = [] + i = 8 + while i < len(b): + sid = b[i] + sz, i = uleb(b, i + 1) + end = i + sz + if end > len(b): + raise Fail("section %d overruns module" % sid) + s, j = b, i + ids.append(sid) + if sid == 0: + nm, _ = name(s, j) + feats["custom-section:" + nm] += 1 + elif sid == 1: + n, j = uleb(s, j) + for _ in range(n): + if s[j] != 0x60: + raise Fail("unknown type form 0x%02x" % s[j]) + j += 1 + sig = [] + for _part in range(2): + k, j = uleb(s, j) + vs = [] + for _ in range(k): + v, j = valtype(s, j, feats) + vs.append(v) + sig.append(vs) + if len(sig[1]) > 1: + feats["multivalue"] += 1 + types.append("(%s)->(%s)" % (",".join(sig[0]), ",".join(sig[1]))) + elif sid == 2: + n, j = uleb(s, j) + for _ in range(n): + mod, j = name(s, j) + fld, j = name(s, j) + kind = s[j] + j += 1 + if kind == 0: + t, j = uleb(s, j) + import_funcs.append(t) + imports.add((mod, fld, "func", types[t])) + elif kind == 1: + _, j = valtype(s, j, feats) + j = limits(s, j, feats) + imports.add((mod, fld, "table", "")) + elif kind == 2: + j = limits(s, j, feats) + imports.add((mod, fld, "memory", "")) + elif kind == 3: + v, j = valtype(s, j, feats) + if s[j]: + feats["mutable-globals"] += 1 + j += 1 + imports.add((mod, fld, "global", v)) + else: + raise Fail("unknown import kind %d" % kind) + elif sid == 3: + n, j = uleb(s, j) + for _ in range(n): + t, j = uleb(s, j) + funcs.append(t) + elif sid == 6: + n, j = uleb(s, j) + for _ in range(n): + _, j = valtype(s, j, feats) + j += 1 + j = walk_ops(s, j, end, feats, const_expr=True) + elif sid == 7: + n, j = uleb(s, j) + for _ in range(n): + nm, j = name(s, j) + kind = s[j] + idx, j = uleb(s, j + 1) + kname = {0: "func", 1: "table", 2: "memory", 3: "global"}.get(kind) + if kname is None: + raise Fail("unknown export kind %d" % kind) + desc = "" + if kind == 0: + ni = len(import_funcs) + desc = types[import_funcs[idx] if idx < ni else funcs[idx - ni]] + exports.add((nm, kname, desc)) + elif sid == 9: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["elem-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + for _ in range(k): + _, j = uleb(s, j) + elif sid == 10: + n, j = uleb(s, j) + for _ in range(n): + bsz, j = uleb(s, j) + bend = j + bsz + ng, j = uleb(s, j) + for _ in range(ng): + _, j = uleb(s, j) + _, j = valtype(s, j, feats) + j = walk_ops(s, j, bend, feats) + if j != bend or s[bend - 1] != 0x0B: + raise Fail("function body did not parse to its end") + elif sid == 11: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["data-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + j += k + elif sid == 12: + feats["bulk-memory"] += 1 + elif sid in (4, 5, 8): + pass + else: + feats["section-%d" % sid] += 1 + i = end + return {"imports": imports, "exports": exports, "feats": feats, "ids": set(ids)} + + +def gate(shipped, rebuilt, label): + """Rebuilt blob must present the shipped blob's tables and no extra wasm features.""" + a, r = wasm_info(shipped), wasm_info(rebuilt) + errs = [] + for k in ("imports", "exports"): + if a[k] != r[k]: + errs.append("%s differ: missing %s, extra %s" % (k, sorted(a[k] - r[k]), sorted(r[k] - a[k]))) + simd = "simd128" in a["feats"] + if simd != ("simd128" in r["feats"]): + errs.append("simd128 use differs: shipped %s, rebuilt %s" % (simd, not simd)) + allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) + extra = sorted(set(r["feats"]) - allowed) + if extra: + errs.append("features not allowed: %s" % ", ".join("%s x%d" % (f, r["feats"][f]) for f in extra)) + if r["ids"] - a["ids"]: + errs.append("sections not in shipped blob: %s" % sorted(r["ids"] - a["ids"])) + if errs: + raise Fail("%s: gate failed\n " % label + "\n ".join(errs)) + log("@@GATE ok %s imports=%d exports=%d feats=%s" % ( + label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}")) + + +# ---------------------------------------------------------------- carriers + +def b64_decode(run): + try: + return base64.b64decode(run, validate=True) + except (binascii.Error, ValueError): + return None + + +def _misaligned_patterns(): + out = [] + for k in (1, 2): + enc = base64.b64encode(b"\0" * k + MAGIC) + lo, hi = 8 * k, 8 * k + 8 * len(MAGIC) + first, last = -(-lo // 6), hi // 6 - 1 + out.append(enc[first:last + 1]) + return out + + +MISALIGNED = _misaligned_patterns() +OTHER_FORMS = ( + ("js-bytes", re.compile(rb"(? (p, at, len) => { seen[k] = (seen[k] || '') + Buffer.from(mem.buffer, at, len).toString(); return 0; }; + let mem; + const env = { + wasm_on_message_begin: () => 0, wasm_on_status: span('status'), + wasm_on_url: span('url'), wasm_on_header_field: span('hf'), wasm_on_header_value: span('hv'), + wasm_on_headers_complete: () => 0, wasm_on_body: span('body'), + wasm_on_message_complete: () => { seen.done = true; return 0; }, + }; + const { exports: e } = new WebAssembly.Instance(new WebAssembly.Module(fs.readFileSync(f)), { env }); + mem = e.memory; + e._initialize(); + const p = e.llhttp_alloc(1); + const buf = e.malloc(req.length); + new Uint8Array(mem.buffer, buf, req.length).set(req); + const rc = e.llhttp_execute(p, buf, req.length); + if (rc !== 0 || seen.url !== '/x?y=1' || seen.body !== 'hello' || !seen.done) { + console.error('@@SMOKE FAIL', f, rc, JSON.stringify(seen)); process.exit(1); + } + e.free(buf); e.llhttp_free(p); + console.log('@@SMOKE ok llhttp_execute', f); +} +""" + +FETCH_JS = r""" +const http = require('node:http'); +const s = http.createServer((q, r) => { let b = ''; q.on('data', (d) => b += d); q.on('end', () => r.end('echo:' + b)); }); +s.on('error', (e) => { + if (['EPERM', 'EACCES', 'EADDRNOTAVAIL'].includes(e.code)) { console.log('@@SMOKE fetch SKIP no loopback', e.code); process.exit(0); } + console.error('@@SMOKE FAIL listen', e); process.exit(1); +}); +s.listen(0, '127.0.0.1', async () => { + try { + const r = await fetch(`http://127.0.0.1:${s.address().port}/x`, { method: 'POST', body: 'hello' }); + const t = await r.text(); + if (t !== 'echo:hello') throw new Error('body ' + t); + console.log('@@SMOKE ok fetch'); s.close(); + } catch (e) { console.error('@@SMOKE FAIL fetch', e); process.exit(1); } +}); +""" + + +def smoke(node, wasms, fetch, requires): + subprocess.run([node, "-e", SMOKE_JS] + list(wasms), check=True) + for r in requires: + subprocess.run([node, "-e", "require(%s); console.log('@@SMOKE ok require', %s)" % ( + json.dumps(os.path.abspath(r)), json.dumps(r))], check=True) + if fetch: + subprocess.run([node, "-e", FETCH_JS], check=True) + + +# ---------------------------------------------------------------- selftest + +def _sec(sid, body): + return bytes([sid]) + _leb(len(body)) + body + + +def _leb(n): + out = bytearray() + while True: + x = n & 0x7F + n >>= 7 + if n: + out.append(x | 0x80) + else: + out.append(x) + return bytes(out) + + +def _str(s): + return _leb(len(s)) + s.encode() + + +def _vec(items): + return _leb(len(items)) + b"".join(items) + + +def mkwasm(imports=("wasm_on_url",), exports=("malloc", "llhttp_execute"), body=b"\x41\x00\x0b", simd=False): + """Tiny module: imports env. (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" + t3 = b"\x60\x03\x7f\x7f\x7f\x01\x7f" + types = _sec(1, _vec([t3])) + imp = _sec(2, _vec([_str("env") + _str(n) + b"\x00\x00" for n in imports])) + nf = len(exports) + fn = _sec(3, _vec([b"\x00"] * nf)) + mem = _sec(5, _vec([b"\x00\x01"])) + exp = _sec(7, _vec([_str("memory") + b"\x02\x00"] + + [_str(n) + b"\x00" + _leb(len(imports) + k) for k, n in enumerate(exports)])) + if simd: + body = b"\xfd\x0c" + b"\x00" * 16 + b"\x1a" + body + code1 = _leb(0) + body + code = _sec(10, _vec([_leb(len(code1)) + code1] * nf)) + return MAGIC + types + imp + fn + mem + exp + code + + +def _expect_fail(fn, *a, contains=""): + try: + fn(*a) + except Fail as e: + if contains and contains not in str(e): + raise AssertionError("failed for the wrong reason: %s" % e) + log(" (expected failure: %s)" % str(e).splitlines()[0]) + return + raise AssertionError("%s%r should have failed" % (fn.__name__, a)) + + +def selftest(): + shipped = mkwasm() + shipped_simd = mkwasm(simd=True) + good = mkwasm(body=b"\x41\x01\x0b") + good_simd = mkwasm(body=b"\x41\x01\x0b", simd=True) + bad_import = mkwasm(imports=("wasm_on_url", "__multi3"), body=b"\x41\x02\x0b") + bad_export = mkwasm(exports=("malloc",), body=b"\x41\x03\x0b") + bad_bulk = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\xfc\x0b\x00\x41\x00\x0b") + bad_overlong = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\x41\x00\x11\x00\x80\x80\x80\x80\x00\x0b") + bad_unknown = mkwasm(body=b"\x06\x40\x0b\x41\x00\x0b") + bad_signext = mkwasm(body=b"\x41\x00\xc0\x0b") + pins = {k: sha(v) for k, v in (("plain", shipped), ("simd", shipped_simd))} + + log("selftest: gate") + gate(shipped, good, "good") + _expect_fail(gate, shipped, bad_signext, "sign-ext", contains="sign-ext") + gate(shipped_simd, good_simd, "good simd") + _expect_fail(gate, shipped, bad_import, "bad import", contains="imports differ") + _expect_fail(gate, shipped, bad_export, "bad export", contains="exports differ") + _expect_fail(gate, shipped, bad_bulk, "bulk-memory", contains="bulk-memory") + _expect_fail(gate, shipped, bad_overlong, "overlong", contains="call-indirect-overlong") + _expect_fail(gate, shipped, good_simd, "simd in plain", contains="simd128") + _expect_fail(gate, shipped_simd, good, "no simd in simd", contains="simd128") + _expect_fail(wasm_info, bad_unknown, contains="unknown opcode") + _expect_fail(wasm_info, b"\0asm\x02\0\0\0", contains="not a wasm v1") + + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as td: + def tree(): + root = os.path.join(td, "t%d" % len(os.listdir(td))) + u = os.path.join(root, "deps/undici") + n = os.path.join(root, "deps/npm/node_modules/undici/lib/llhttp") + os.makedirs(n) + os.makedirs(u, exist_ok=True) + os.makedirs(os.path.join(root, "deps/v8/third_party/wasm-api/example")) + with open(os.path.join(u, "undici.js"), "w") as f: + f.write('var wasmBase64 = "%s";\n// next\nvar wasmBase64 = "%s";\n' % ( + base64.b64encode(shipped).decode(), base64.b64encode(shipped_simd).decode())) + with open(os.path.join(n, "llhttp-wasm.js"), "w") as f: + f.write("module.exports = Buffer.from('%s', 'base64')\n" % base64.b64encode(shipped).decode()) + with open(os.path.join(root, "deps/v8/third_party/wasm-api/example/hello.wasm"), "wb") as f: + f.write(mkwasm(exports=("run",))) + for name, b in (("good.wasm", good), ("good_simd.wasm", good_simd), ("bad_bulk.wasm", bad_bulk)): + with open(os.path.join(root, name), "wb") as f: + f.write(b) + return root + + def full_pass(root): + os.chdir(root) + splice("deps/undici/undici.js", pins["plain"], "good.wasm") + splice("deps/undici/undici.js", pins["simd"], "good_simd.wasm") + splice("deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js", pins["plain"], "good.wasm") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}) + + try: + log("selftest: splice + census pass") + full_pass(tree()) + + log("selftest: N+ expectations") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (1, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (3, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}, + contains="expected 3+") + + log("selftest: splice rejects") + root = tree() + os.chdir(root) + _expect_fail(splice, "deps/undici/undici.js", "0" * 64, "good.wasm", contains="found 0 times") + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "bad_bulk.wasm", contains="bulk-memory") + before = open("deps/undici/undici.js", "rb").read() + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good_simd.wasm", contains="simd128") + assert open("deps/undici/undici.js", "rb").read() == before, "failed splice modified the carrier" + with open("deps/undici/undici.js", "a") as f: + f.write('var again = "%s";\n' % base64.b64encode(shipped).decode()) + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good.wasm", contains="found 2 times") + + log("selftest: census rejects") + for label, mutate, why in ( + ("unspliced shipped blob", lambda: None, "base64 deps/undici/undici.js"), + ("rogue base64", lambda: open("deps/rogue.js", "w").write( + "x='%s'" % base64.b64encode(mkwasm(exports=("evil",))).decode()), "deps/rogue.js"), + ("raw wasm", lambda: open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))), "raw deps/x.wasm"), + ("embedded raw", lambda: open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"..."), "raw deps/x.bin"), + ("misaligned base64", lambda: open("deps/m.js", "w").write( + "y='%s'" % base64.b64encode(b"Q" + mkwasm()).decode()), "base64-misaligned deps/m.js"), + ("js byte array", lambda: open("deps/a.js", "w").write("[0, 97, 115, 109, 1, 0, 0, 0]"), "js-bytes"), + ("truncated base64", lambda: open("deps/t.js", "w").write("z='AGFzbQEAAAABJw'"), "undecodable"), + ("expect count", lambda: open("deps/undici/undici.js", "a").write( + "var dup = '%s';\n" % base64.b64encode(good).decode()), "expected 2 allowed blob(s)"), + ("unexpected carrier", lambda: open("deps/copy.js", "w").write( + "c='%s'" % base64.b64encode(good).decode()), "unexpected file deps/copy.js"), + ): + log(" case: " + label) + root = tree() + os.chdir(root) + if label != "unspliced shipped blob": + full_pass(root) + mutate() + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], + ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), + "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}, + contains=why) + + log("selftest: census allow-raw is an exact count") + root = tree() + os.chdir(root) + full_pass(root) + open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"........" + MAGIC + b"...") + exp = {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)} + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, {"deps/x.bin": 2}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.bin": 1}, contains="expected 1 raw header(s)") + open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.wasm": 1}, contains="raw deps/x.wasm") + + log("selftest: source pins") + src = os.path.join(td, "undici-src") + os.makedirs(os.path.join(src, "deps/llhttp/src")) + os.makedirs(os.path.join(src, "deps/llhttp/include")) + os.makedirs(os.path.join(src, "build")) + json.dump({"version": "8.10.2"}, open(os.path.join(src, "package.json"), "w")) + open(os.path.join(src, "deps/llhttp/include/llhttp.h"), "w").write( + "#define LLHTTP_VERSION_MAJOR 9\n#define LLHTTP_VERSION_MINOR 3\n#define LLHTTP_VERSION_PATCH 1\n") + for s in LLHTTP_SRCS: + open(os.path.join(src, "deps/llhttp/src", s), "w").write("") + open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + build(src, os.path.join(td, "out"), "8.10.2", "9.3.1", "clang", "/usr/share/wasi-sysroot", None, True) + _expect_fail(check_undici_src, src, "8.10.3", "9.3.1", contains="recipe pins 8.10.3") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.0", contains="recipe pins 9.3.0") + open(os.path.join(src, "deps/llhttp/src/extra.c"), "w").write("") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="src/*.c") + os.unlink(os.path.join(src, "deps/llhttp/src/extra.c")) + open(os.path.join(src, "build/wasm.js"), "w").write( + "\n".join(WASM_JS_PINNED).replace("-Ofast", "-O3") + "\n") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="flags drifted") + finally: + os.chdir(cwd) + + cmd = compile_cmd("clang", "/s", None, "o.wasm", True) + for f in CFLAGS + LDFLAGS + CPU + ["-msimd128", "-mexec-model=reactor", "-lc", "--no-wasm-opt"]: + assert f in cmd, f + log("@@SELFTEST PASS") + + +# ---------------------------------------------------------------- cli + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sp = ap.add_subparsers(dest="cmd", required=True) + p = sp.add_parser("check-tree") + p.add_argument("--undici", required=True) + p.add_argument("--npm-undici", required=True) + p = sp.add_parser("build") + p.add_argument("src") + p.add_argument("outdir") + p.add_argument("--version", required=True) + p.add_argument("--llhttp", required=True) + p.add_argument("--cc", default="clang") + p.add_argument("--sysroot", default="/usr/lib/wasi", help="packages/wasi-libc install root") + p.add_argument("--rt", help="explicit wasm32 builtins archive, if llhttp needs one") + p.add_argument("--dry-run", action="store_true") + p = sp.add_parser("splice") + p.add_argument("carrier") + p.add_argument("shipped_sha") + p.add_argument("rebuilt") + p = sp.add_parser("census") + p.add_argument("roots", nargs="+") + p.add_argument("-C", dest="chdir") + p.add_argument("--wasm", action="append", default=[], help="rebuilt blob whose sha is allowed") + p.add_argument("--allow-path", action="append", default=[], help="fnmatch glob, relative to -C") + p.add_argument("--expect", action="append", default=[], help="PATH=N (exact) or PATH=N+ allowed blobs in PATH") + p.add_argument("--allow-raw", action="append", default=[], help="PATH=N: exactly N undecodable raw wasm headers in PATH") + p = sp.add_parser("smoke") + p.add_argument("node") + p.add_argument("wasm", nargs="+") + p.add_argument("--fetch", action="store_true") + p.add_argument("--require", action="append", default=[]) + sp.add_parser("selftest") + a = ap.parse_args(argv) + try: + if a.cmd == "check-tree": + check_tree(a.undici, a.npm_undici) + elif a.cmd == "build": + build(a.src, a.outdir, a.version, a.llhttp, a.cc, a.sysroot, a.rt, a.dry_run) + elif a.cmd == "splice": + splice(a.carrier, a.shipped_sha, a.rebuilt) + elif a.cmd == "census": + shas = [sha(open(w, "rb").read()) for w in a.wasm] + if a.chdir: + os.chdir(a.chdir) + expect = {} + for e in a.expect: + k, _, v = e.rpartition("=") + expect[os.path.normpath(k)] = (int(v.rstrip("+")), v.endswith("+")) + raw = {} + for e in a.allow_raw: + k, _, v = e.rpartition("=") + raw[os.path.normpath(k)] = int(v) + census(a.roots, shas, a.allow_path, expect, raw) + elif a.cmd == "smoke": + smoke(a.node, a.wasm, a.fetch, a.require) + elif a.cmd == "selftest": + selftest() + except Fail as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 021df5ad3c46a4c0f486184b72b394b2673a4faf Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 15 Oct 2026 09:13:57 -0700 Subject: [PATCH 04/16] llhttp-wasm-rebuild: one shared copy of the wasm rebuild script node and node-lts each carried an identical rebuild-llhttp-wasm.py (Local files cannot reach outside the package directory). Ship it once as a build-time package, run its selftest there, and depend on it from both. Co-Authored-By: Claude Opus 5.5 --- packages/llhttp-wasm-rebuild/build.ncl | 27 + packages/llhttp-wasm-rebuild/build.sh | 5 + .../rebuild-llhttp-wasm.py | 0 packages/node-lts/build.ncl | 4 +- packages/node-lts/build.sh | 3 +- packages/node/build.ncl | 4 +- packages/node/build.sh | 3 +- packages/node/rebuild-llhttp-wasm.py | 924 ------------------ 8 files changed, 38 insertions(+), 932 deletions(-) create mode 100644 packages/llhttp-wasm-rebuild/build.ncl create mode 100755 packages/llhttp-wasm-rebuild/build.sh rename packages/{node-lts => llhttp-wasm-rebuild}/rebuild-llhttp-wasm.py (100%) delete mode 100755 packages/node/rebuild-llhttp-wasm.py diff --git a/packages/llhttp-wasm-rebuild/build.ncl b/packages/llhttp-wasm-rebuild/build.ncl new file mode 100644 index 000000000..4fe3925b8 --- /dev/null +++ b/packages/llhttp-wasm-rebuild/build.ncl @@ -0,0 +1,27 @@ +let { BuildSpec, Local, OutputData, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let python = import "../python/build.ncl" in + +# Build-time helper for node and node-lts: rebuilds undici's llhttp wasm from its +# in-tree C and splices it over the shipped base64 blobs. See the script's docstring. +{ + name = "llhttp-wasm-rebuild", + build_deps = [ + { file = "build.sh" } | Local, + { file = "rebuild-llhttp-wasm.py" } | Local, + base, + python, + ], + cmd = "./build.sh", + outputs = { + script = { glob = "usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" } | OutputData, + }, + tests = { + selftest = + { + class = 'Standalone, + test_deps = [python], + cmds = [["python3", "/usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py", "selftest"]], + } | Test, + }, +} | BuildSpec diff --git a/packages/llhttp-wasm-rebuild/build.sh b/packages/llhttp-wasm-rebuild/build.sh new file mode 100755 index 000000000..adc2e05db --- /dev/null +++ b/packages/llhttp-wasm-rebuild/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex + +python3 rebuild-llhttp-wasm.py selftest +install -Dm 0644 rebuild-llhttp-wasm.py "$OUTPUT_DIR/usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" diff --git a/packages/node-lts/rebuild-llhttp-wasm.py b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py similarity index 100% rename from packages/node-lts/rebuild-llhttp-wasm.py rename to packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index cd3058328..25eee6489 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -20,6 +20,7 @@ let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in let llvm = import "../llvm/build.ncl" in let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. let npm-undici = "6.26.0" in @@ -32,8 +33,6 @@ let version = "24.19.0" in url = "gs://minimal-staging-archives/node-v%{version}.tar.gz", sha256 = "16fe258006a6e86844fbe05b3b5e1e5623ca8d3da54e32d98d9e83234bf25b01", } | Source, - # Rebuilds and splices undici's llhttp wasm blobs. - { file = "rebuild-llhttp-wasm.py" } | Local, { url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", sha256 = "fc371d680e507ddf32f4944cf6b355a520f16ef5fc4987e0f25dc668372475c7", @@ -41,6 +40,7 @@ let version = "24.19.0" in base, llvm, # clang + wasm-ld for the llhttp wasm wasi-libc, + llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs make, pkgconf, python, diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 183a3b17b..971416c40 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -8,8 +8,7 @@ cd "node-v${MINIMAL_ARG_VERSION}" # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. -LH="python3 ../rebuild-llhttp-wasm.py" -$LH selftest +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" NPM_UNDICI=6.26.0 NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici 7.29.0 --npm-undici "$NPM_UNDICI" diff --git a/packages/node/build.ncl b/packages/node/build.ncl index 3a384ac81..0f4015b56 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -22,6 +22,7 @@ let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in let llvm = import "../llvm/build.ncl" in let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. let npm-undici = "6.28.0" in @@ -37,8 +38,6 @@ let version = "26.10.0" in # Adapts the Mach-O SEA injection to the LIEF 1.x Section API — see # the patch header. Build-time only: it rewrites the source tree. { file = "lief-1.x-macho-section-ctor.patch" } | Local, - # Rebuilds and splices undici's llhttp wasm blobs. - { file = "rebuild-llhttp-wasm.py" } | Local, { url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", @@ -46,6 +45,7 @@ let version = "26.10.0" in base, llvm, # clang + wasm-ld for the llhttp wasm wasi-libc, + llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs make, pkgconf, patch, diff --git a/packages/node/build.sh b/packages/node/build.sh index 4d506a5d2..ed17ba3a4 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -19,8 +19,7 @@ grep -q "LIEF::MachO::Section::create(section_name, data)" src/node_sea_bin.cc | # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. -LH="python3 ../rebuild-llhttp-wasm.py" -$LH selftest +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" NPM_UNDICI=6.28.0 NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici 8.10.2 --npm-undici "$NPM_UNDICI" diff --git a/packages/node/rebuild-llhttp-wasm.py b/packages/node/rebuild-llhttp-wasm.py deleted file mode 100755 index 8bc679ac2..000000000 --- a/packages/node/rebuild-llhttp-wasm.py +++ /dev/null @@ -1,924 +0,0 @@ -#!/usr/bin/env python3 -"""Regenerate undici's llhttp wasm from in-tree C and replace the shipped blobs. - - check-tree node tree tripwires: pinned undici / npm-undici versions, no cjs lexer wasm - build pin build/wasm.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes - splice decode the pinned shipped blob in a carrier, gate tables+features, replace it once - census every wasm carrier under the roots is an allowed sha or on an allowed path - smoke instantiate blobs under a built node and parse a POST; optional loopback fetch - selftest synthetic tree, proves every gate both passes and fails - -stdlib only. Exit 1 on any mismatch. -""" -import argparse -import base64 -import binascii -import collections -import fnmatch -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile - -MAGIC = b"\0asm\x01\0\0\0" - -# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v8.10.2. -WASM_JS_PINNED = ( - "let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi'", - "WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor'", - "WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all'", - "WASM_LDFLAGS += ' -Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table'", - "WASM_LDFLAGS += ' -Wl,--export=malloc -Wl,--export=free -Wl,--no-entry'", - "const WASM_OPT_FLAGS = '-O4 --converge --strip-debug --strip-dwarf --strip-producers'", -) -CFLAGS = "-Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor".split() -LDFLAGS = ("-Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all " - "-Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table " - "-Wl,--export=malloc -Wl,--export=free -Wl,--no-entry").split() -# clang 17's "generic" cpu; clang 21's generic adds bulk-memory, multivalue, reference-types, -# nontrapping-fptoint and call-indirect-overlong, none of which the shipped blobs use. -CPU = ["-mcpu=mvp", "-mmutable-globals"] -LLHTTP_SRCS = ("api.c", "http.c", "llhttp.c") -GATE_ALLOWED = {"mutable-globals"} - -B64_ALIGNED = re.compile(rb"AGFzbQ[A-Za-z0-9+/]*={0,2}") - - -class Fail(Exception): - pass - - -def sha(b): - return hashlib.sha256(b).hexdigest() - - -def log(msg): - print(msg, flush=True) - - -# ---------------------------------------------------------------- wasm reader - -def uleb(b, i, maxbytes=5): - r = s = n = 0 - while True: - if i >= len(b): - raise Fail("truncated leb128") - x = b[i] - i += 1 - n += 1 - r |= (x & 0x7F) << s - s += 7 - if x < 0x80: - return r, i - if n >= maxbytes: - raise Fail("leb128 too long") - - -def uleb_len(b, i): - v, j = uleb(b, i) - return v, j, j - i - - -def sleb(b, i, maxbytes=10): - r = s = n = 0 - while True: - if i >= len(b): - raise Fail("truncated sleb128") - x = b[i] - i += 1 - n += 1 - r |= (x & 0x7F) << s - s += 7 - if x < 0x80: - if x & 0x40: - r -= 1 << s - return r, i - if n >= maxbytes: - raise Fail("sleb128 too long") - - -def name(b, i): - n, i = uleb(b, i) - return b[i:i + n].decode("utf-8", "replace"), i + n - - -VALTYPES = {0x7F: "i32", 0x7E: "i64", 0x7D: "f32", 0x7C: "f64", 0x7B: "v128", 0x70: "funcref", 0x6F: "externref"} - - -def valtype(b, i, feats): - t = b[i] - if t not in VALTYPES: - raise Fail("unknown valtype 0x%02x" % t) - if t == 0x7B: - feats["simd128"] += 1 - elif t in (0x70, 0x6F): - feats["reference-types"] += 1 - return VALTYPES[t], i + 1 - - -def limits(b, i, feats): - flag = b[i] - i += 1 - if flag & ~1: - feats["limits-flag-%d" % flag] += 1 - _, i = uleb(b, i, 10) - if flag & 1: - _, i = uleb(b, i, 10) - return i - - -def walk_ops(b, i, end, feats, const_expr=False): - """Walk instructions in b[i:end]; count non-MVP features; unknown opcodes fail closed.""" - while i < end: - op = b[i] - i += 1 - if op == 0x0B and const_expr: - return i - if op in (0x00, 0x01, 0x05, 0x0B, 0x0F, 0x1A, 0x1B, 0xD1) or 0x45 <= op <= 0xBF: - if op == 0xD1: - feats["reference-types"] += 1 - elif op in (0x02, 0x03, 0x04): - bt = b[i] - if bt == 0x40 or bt in VALTYPES: - if bt == 0x7B: - feats["simd128"] += 1 - i += 1 - else: - _, i = sleb(b, i, 5) - feats["multivalue"] += 1 - elif op in (0x0C, 0x0D, 0x10, 0x20, 0x21, 0x22, 0x23, 0x24): - _, i = uleb(b, i) - elif op in (0x25, 0x26, 0xD2): - _, i = uleb(b, i) - feats["reference-types"] += 1 - elif op == 0x0E: - n, i = uleb(b, i) - for _ in range(n + 1): - _, i = uleb(b, i) - elif op == 0x11: - _, i = uleb(b, i) - tbl, i, ln = uleb_len(b, i) - if ln > 1: - feats["call-indirect-overlong"] += 1 - if tbl != 0: - feats["reference-types"] += 1 - elif op == 0x1C: - n, i = uleb(b, i) - for _ in range(n): - _, i = valtype(b, i, feats) - feats["reference-types"] += 1 - elif 0x28 <= op <= 0x3E: - _, i = uleb(b, i) - _, i = uleb(b, i, 10) - elif op in (0x3F, 0x40): - if b[i] != 0: - feats["multi-memory"] += 1 - _, i = uleb(b, i) - elif op == 0x41: - _, i = sleb(b, i, 5) - elif op == 0x42: - _, i = sleb(b, i, 10) - elif op == 0x43: - i += 4 - elif op == 0x44: - i += 8 - elif 0xC0 <= op <= 0xC4: - feats["sign-ext"] += 1 - elif op == 0xD0: - i += 1 - feats["reference-types"] += 1 - elif op == 0xFC: - sub, i = uleb(b, i) - if sub <= 7: - feats["nontrapping-fptoint"] += 1 - elif sub <= 14: - feats["bulk-memory"] += 1 - nimm = {8: 2, 9: 1, 10: 2, 11: 1, 12: 2, 13: 1, 14: 2}[sub] - for _ in range(nimm): - _, i = uleb(b, i) - elif sub <= 17: - feats["reference-types"] += 1 - _, i = uleb(b, i) - else: - raise Fail("unknown opcode 0xfc %d" % sub) - elif op == 0xFD: - sub, i = uleb(b, i) - feats["simd128"] += 1 - if sub <= 11 or sub in (92, 93): - _, i = uleb(b, i) - _, i = uleb(b, i, 10) - elif sub in (12, 13): - i += 16 - elif 21 <= sub <= 34: - i += 1 - elif 84 <= sub <= 91: - _, i = uleb(b, i) - _, i = uleb(b, i, 10) - i += 1 - elif sub >= 0x100: - raise Fail("relaxed-simd opcode 0xfd %d" % sub) - else: - raise Fail("unknown opcode 0x%02x" % op) - if const_expr: - raise Fail("const expr without end") - return i - - -def wasm_info(b): - if b[:8] != MAGIC: - raise Fail("not a wasm v1 module") - feats = collections.Counter() - types, funcs, imports, exports, ids = [], [], set(), set(), [] - import_funcs = [] - i = 8 - while i < len(b): - sid = b[i] - sz, i = uleb(b, i + 1) - end = i + sz - if end > len(b): - raise Fail("section %d overruns module" % sid) - s, j = b, i - ids.append(sid) - if sid == 0: - nm, _ = name(s, j) - feats["custom-section:" + nm] += 1 - elif sid == 1: - n, j = uleb(s, j) - for _ in range(n): - if s[j] != 0x60: - raise Fail("unknown type form 0x%02x" % s[j]) - j += 1 - sig = [] - for _part in range(2): - k, j = uleb(s, j) - vs = [] - for _ in range(k): - v, j = valtype(s, j, feats) - vs.append(v) - sig.append(vs) - if len(sig[1]) > 1: - feats["multivalue"] += 1 - types.append("(%s)->(%s)" % (",".join(sig[0]), ",".join(sig[1]))) - elif sid == 2: - n, j = uleb(s, j) - for _ in range(n): - mod, j = name(s, j) - fld, j = name(s, j) - kind = s[j] - j += 1 - if kind == 0: - t, j = uleb(s, j) - import_funcs.append(t) - imports.add((mod, fld, "func", types[t])) - elif kind == 1: - _, j = valtype(s, j, feats) - j = limits(s, j, feats) - imports.add((mod, fld, "table", "")) - elif kind == 2: - j = limits(s, j, feats) - imports.add((mod, fld, "memory", "")) - elif kind == 3: - v, j = valtype(s, j, feats) - if s[j]: - feats["mutable-globals"] += 1 - j += 1 - imports.add((mod, fld, "global", v)) - else: - raise Fail("unknown import kind %d" % kind) - elif sid == 3: - n, j = uleb(s, j) - for _ in range(n): - t, j = uleb(s, j) - funcs.append(t) - elif sid == 6: - n, j = uleb(s, j) - for _ in range(n): - _, j = valtype(s, j, feats) - j += 1 - j = walk_ops(s, j, end, feats, const_expr=True) - elif sid == 7: - n, j = uleb(s, j) - for _ in range(n): - nm, j = name(s, j) - kind = s[j] - idx, j = uleb(s, j + 1) - kname = {0: "func", 1: "table", 2: "memory", 3: "global"}.get(kind) - if kname is None: - raise Fail("unknown export kind %d" % kind) - desc = "" - if kind == 0: - ni = len(import_funcs) - desc = types[import_funcs[idx] if idx < ni else funcs[idx - ni]] - exports.add((nm, kname, desc)) - elif sid == 9: - n, j = uleb(s, j) - for _ in range(n): - flag, j = uleb(s, j) - if flag != 0: - feats["elem-flags-%d" % flag] += 1 - break - j = walk_ops(s, j, end, feats, const_expr=True) - k, j = uleb(s, j) - for _ in range(k): - _, j = uleb(s, j) - elif sid == 10: - n, j = uleb(s, j) - for _ in range(n): - bsz, j = uleb(s, j) - bend = j + bsz - ng, j = uleb(s, j) - for _ in range(ng): - _, j = uleb(s, j) - _, j = valtype(s, j, feats) - j = walk_ops(s, j, bend, feats) - if j != bend or s[bend - 1] != 0x0B: - raise Fail("function body did not parse to its end") - elif sid == 11: - n, j = uleb(s, j) - for _ in range(n): - flag, j = uleb(s, j) - if flag != 0: - feats["data-flags-%d" % flag] += 1 - break - j = walk_ops(s, j, end, feats, const_expr=True) - k, j = uleb(s, j) - j += k - elif sid == 12: - feats["bulk-memory"] += 1 - elif sid in (4, 5, 8): - pass - else: - feats["section-%d" % sid] += 1 - i = end - return {"imports": imports, "exports": exports, "feats": feats, "ids": set(ids)} - - -def gate(shipped, rebuilt, label): - """Rebuilt blob must present the shipped blob's tables and no extra wasm features.""" - a, r = wasm_info(shipped), wasm_info(rebuilt) - errs = [] - for k in ("imports", "exports"): - if a[k] != r[k]: - errs.append("%s differ: missing %s, extra %s" % (k, sorted(a[k] - r[k]), sorted(r[k] - a[k]))) - simd = "simd128" in a["feats"] - if simd != ("simd128" in r["feats"]): - errs.append("simd128 use differs: shipped %s, rebuilt %s" % (simd, not simd)) - allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) - extra = sorted(set(r["feats"]) - allowed) - if extra: - errs.append("features not allowed: %s" % ", ".join("%s x%d" % (f, r["feats"][f]) for f in extra)) - if r["ids"] - a["ids"]: - errs.append("sections not in shipped blob: %s" % sorted(r["ids"] - a["ids"])) - if errs: - raise Fail("%s: gate failed\n " % label + "\n ".join(errs)) - log("@@GATE ok %s imports=%d exports=%d feats=%s" % ( - label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}")) - - -# ---------------------------------------------------------------- carriers - -def b64_decode(run): - try: - return base64.b64decode(run, validate=True) - except (binascii.Error, ValueError): - return None - - -def _misaligned_patterns(): - out = [] - for k in (1, 2): - enc = base64.b64encode(b"\0" * k + MAGIC) - lo, hi = 8 * k, 8 * k + 8 * len(MAGIC) - first, last = -(-lo // 6), hi // 6 - 1 - out.append(enc[first:last + 1]) - return out - - -MISALIGNED = _misaligned_patterns() -OTHER_FORMS = ( - ("js-bytes", re.compile(rb"(? (p, at, len) => { seen[k] = (seen[k] || '') + Buffer.from(mem.buffer, at, len).toString(); return 0; }; - let mem; - const env = { - wasm_on_message_begin: () => 0, wasm_on_status: span('status'), - wasm_on_url: span('url'), wasm_on_header_field: span('hf'), wasm_on_header_value: span('hv'), - wasm_on_headers_complete: () => 0, wasm_on_body: span('body'), - wasm_on_message_complete: () => { seen.done = true; return 0; }, - }; - const { exports: e } = new WebAssembly.Instance(new WebAssembly.Module(fs.readFileSync(f)), { env }); - mem = e.memory; - e._initialize(); - const p = e.llhttp_alloc(1); - const buf = e.malloc(req.length); - new Uint8Array(mem.buffer, buf, req.length).set(req); - const rc = e.llhttp_execute(p, buf, req.length); - if (rc !== 0 || seen.url !== '/x?y=1' || seen.body !== 'hello' || !seen.done) { - console.error('@@SMOKE FAIL', f, rc, JSON.stringify(seen)); process.exit(1); - } - e.free(buf); e.llhttp_free(p); - console.log('@@SMOKE ok llhttp_execute', f); -} -""" - -FETCH_JS = r""" -const http = require('node:http'); -const s = http.createServer((q, r) => { let b = ''; q.on('data', (d) => b += d); q.on('end', () => r.end('echo:' + b)); }); -s.on('error', (e) => { - if (['EPERM', 'EACCES', 'EADDRNOTAVAIL'].includes(e.code)) { console.log('@@SMOKE fetch SKIP no loopback', e.code); process.exit(0); } - console.error('@@SMOKE FAIL listen', e); process.exit(1); -}); -s.listen(0, '127.0.0.1', async () => { - try { - const r = await fetch(`http://127.0.0.1:${s.address().port}/x`, { method: 'POST', body: 'hello' }); - const t = await r.text(); - if (t !== 'echo:hello') throw new Error('body ' + t); - console.log('@@SMOKE ok fetch'); s.close(); - } catch (e) { console.error('@@SMOKE FAIL fetch', e); process.exit(1); } -}); -""" - - -def smoke(node, wasms, fetch, requires): - subprocess.run([node, "-e", SMOKE_JS] + list(wasms), check=True) - for r in requires: - subprocess.run([node, "-e", "require(%s); console.log('@@SMOKE ok require', %s)" % ( - json.dumps(os.path.abspath(r)), json.dumps(r))], check=True) - if fetch: - subprocess.run([node, "-e", FETCH_JS], check=True) - - -# ---------------------------------------------------------------- selftest - -def _sec(sid, body): - return bytes([sid]) + _leb(len(body)) + body - - -def _leb(n): - out = bytearray() - while True: - x = n & 0x7F - n >>= 7 - if n: - out.append(x | 0x80) - else: - out.append(x) - return bytes(out) - - -def _str(s): - return _leb(len(s)) + s.encode() - - -def _vec(items): - return _leb(len(items)) + b"".join(items) - - -def mkwasm(imports=("wasm_on_url",), exports=("malloc", "llhttp_execute"), body=b"\x41\x00\x0b", simd=False): - """Tiny module: imports env. (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" - t3 = b"\x60\x03\x7f\x7f\x7f\x01\x7f" - types = _sec(1, _vec([t3])) - imp = _sec(2, _vec([_str("env") + _str(n) + b"\x00\x00" for n in imports])) - nf = len(exports) - fn = _sec(3, _vec([b"\x00"] * nf)) - mem = _sec(5, _vec([b"\x00\x01"])) - exp = _sec(7, _vec([_str("memory") + b"\x02\x00"] + - [_str(n) + b"\x00" + _leb(len(imports) + k) for k, n in enumerate(exports)])) - if simd: - body = b"\xfd\x0c" + b"\x00" * 16 + b"\x1a" + body - code1 = _leb(0) + body - code = _sec(10, _vec([_leb(len(code1)) + code1] * nf)) - return MAGIC + types + imp + fn + mem + exp + code - - -def _expect_fail(fn, *a, contains=""): - try: - fn(*a) - except Fail as e: - if contains and contains not in str(e): - raise AssertionError("failed for the wrong reason: %s" % e) - log(" (expected failure: %s)" % str(e).splitlines()[0]) - return - raise AssertionError("%s%r should have failed" % (fn.__name__, a)) - - -def selftest(): - shipped = mkwasm() - shipped_simd = mkwasm(simd=True) - good = mkwasm(body=b"\x41\x01\x0b") - good_simd = mkwasm(body=b"\x41\x01\x0b", simd=True) - bad_import = mkwasm(imports=("wasm_on_url", "__multi3"), body=b"\x41\x02\x0b") - bad_export = mkwasm(exports=("malloc",), body=b"\x41\x03\x0b") - bad_bulk = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\xfc\x0b\x00\x41\x00\x0b") - bad_overlong = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\x41\x00\x11\x00\x80\x80\x80\x80\x00\x0b") - bad_unknown = mkwasm(body=b"\x06\x40\x0b\x41\x00\x0b") - bad_signext = mkwasm(body=b"\x41\x00\xc0\x0b") - pins = {k: sha(v) for k, v in (("plain", shipped), ("simd", shipped_simd))} - - log("selftest: gate") - gate(shipped, good, "good") - _expect_fail(gate, shipped, bad_signext, "sign-ext", contains="sign-ext") - gate(shipped_simd, good_simd, "good simd") - _expect_fail(gate, shipped, bad_import, "bad import", contains="imports differ") - _expect_fail(gate, shipped, bad_export, "bad export", contains="exports differ") - _expect_fail(gate, shipped, bad_bulk, "bulk-memory", contains="bulk-memory") - _expect_fail(gate, shipped, bad_overlong, "overlong", contains="call-indirect-overlong") - _expect_fail(gate, shipped, good_simd, "simd in plain", contains="simd128") - _expect_fail(gate, shipped_simd, good, "no simd in simd", contains="simd128") - _expect_fail(wasm_info, bad_unknown, contains="unknown opcode") - _expect_fail(wasm_info, b"\0asm\x02\0\0\0", contains="not a wasm v1") - - cwd = os.getcwd() - with tempfile.TemporaryDirectory() as td: - def tree(): - root = os.path.join(td, "t%d" % len(os.listdir(td))) - u = os.path.join(root, "deps/undici") - n = os.path.join(root, "deps/npm/node_modules/undici/lib/llhttp") - os.makedirs(n) - os.makedirs(u, exist_ok=True) - os.makedirs(os.path.join(root, "deps/v8/third_party/wasm-api/example")) - with open(os.path.join(u, "undici.js"), "w") as f: - f.write('var wasmBase64 = "%s";\n// next\nvar wasmBase64 = "%s";\n' % ( - base64.b64encode(shipped).decode(), base64.b64encode(shipped_simd).decode())) - with open(os.path.join(n, "llhttp-wasm.js"), "w") as f: - f.write("module.exports = Buffer.from('%s', 'base64')\n" % base64.b64encode(shipped).decode()) - with open(os.path.join(root, "deps/v8/third_party/wasm-api/example/hello.wasm"), "wb") as f: - f.write(mkwasm(exports=("run",))) - for name, b in (("good.wasm", good), ("good_simd.wasm", good_simd), ("bad_bulk.wasm", bad_bulk)): - with open(os.path.join(root, name), "wb") as f: - f.write(b) - return root - - def full_pass(root): - os.chdir(root) - splice("deps/undici/undici.js", pins["plain"], "good.wasm") - splice("deps/undici/undici.js", pins["simd"], "good_simd.wasm") - splice("deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js", pins["plain"], "good.wasm") - census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], - {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}) - - try: - log("selftest: splice + census pass") - full_pass(tree()) - - log("selftest: N+ expectations") - census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], - {"deps/undici/undici.js": (1, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}) - _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], - {"deps/undici/undici.js": (3, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}, - contains="expected 3+") - - log("selftest: splice rejects") - root = tree() - os.chdir(root) - _expect_fail(splice, "deps/undici/undici.js", "0" * 64, "good.wasm", contains="found 0 times") - _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "bad_bulk.wasm", contains="bulk-memory") - before = open("deps/undici/undici.js", "rb").read() - _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good_simd.wasm", contains="simd128") - assert open("deps/undici/undici.js", "rb").read() == before, "failed splice modified the carrier" - with open("deps/undici/undici.js", "a") as f: - f.write('var again = "%s";\n' % base64.b64encode(shipped).decode()) - _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good.wasm", contains="found 2 times") - - log("selftest: census rejects") - for label, mutate, why in ( - ("unspliced shipped blob", lambda: None, "base64 deps/undici/undici.js"), - ("rogue base64", lambda: open("deps/rogue.js", "w").write( - "x='%s'" % base64.b64encode(mkwasm(exports=("evil",))).decode()), "deps/rogue.js"), - ("raw wasm", lambda: open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))), "raw deps/x.wasm"), - ("embedded raw", lambda: open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"..."), "raw deps/x.bin"), - ("misaligned base64", lambda: open("deps/m.js", "w").write( - "y='%s'" % base64.b64encode(b"Q" + mkwasm()).decode()), "base64-misaligned deps/m.js"), - ("js byte array", lambda: open("deps/a.js", "w").write("[0, 97, 115, 109, 1, 0, 0, 0]"), "js-bytes"), - ("truncated base64", lambda: open("deps/t.js", "w").write("z='AGFzbQEAAAABJw'"), "undecodable"), - ("expect count", lambda: open("deps/undici/undici.js", "a").write( - "var dup = '%s';\n" % base64.b64encode(good).decode()), "expected 2 allowed blob(s)"), - ("unexpected carrier", lambda: open("deps/copy.js", "w").write( - "c='%s'" % base64.b64encode(good).decode()), "unexpected file deps/copy.js"), - ): - log(" case: " + label) - root = tree() - os.chdir(root) - if label != "unspliced shipped blob": - full_pass(root) - mutate() - _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], - ["deps/v8/third_party/wasm-api/example/*"], - {"deps/undici/undici.js": (2, False), - "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}, - contains=why) - - log("selftest: census allow-raw is an exact count") - root = tree() - os.chdir(root) - full_pass(root) - open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"........" + MAGIC + b"...") - exp = {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)} - census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, {"deps/x.bin": 2}) - _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, - {"deps/x.bin": 1}, contains="expected 1 raw header(s)") - open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))) - _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, - {"deps/x.wasm": 1}, contains="raw deps/x.wasm") - - log("selftest: source pins") - src = os.path.join(td, "undici-src") - os.makedirs(os.path.join(src, "deps/llhttp/src")) - os.makedirs(os.path.join(src, "deps/llhttp/include")) - os.makedirs(os.path.join(src, "build")) - json.dump({"version": "8.10.2"}, open(os.path.join(src, "package.json"), "w")) - open(os.path.join(src, "deps/llhttp/include/llhttp.h"), "w").write( - "#define LLHTTP_VERSION_MAJOR 9\n#define LLHTTP_VERSION_MINOR 3\n#define LLHTTP_VERSION_PATCH 1\n") - for s in LLHTTP_SRCS: - open(os.path.join(src, "deps/llhttp/src", s), "w").write("") - open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") - build(src, os.path.join(td, "out"), "8.10.2", "9.3.1", "clang", "/usr/share/wasi-sysroot", None, True) - _expect_fail(check_undici_src, src, "8.10.3", "9.3.1", contains="recipe pins 8.10.3") - _expect_fail(check_undici_src, src, "8.10.2", "9.3.0", contains="recipe pins 9.3.0") - open(os.path.join(src, "deps/llhttp/src/extra.c"), "w").write("") - _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="src/*.c") - os.unlink(os.path.join(src, "deps/llhttp/src/extra.c")) - open(os.path.join(src, "build/wasm.js"), "w").write( - "\n".join(WASM_JS_PINNED).replace("-Ofast", "-O3") + "\n") - _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="flags drifted") - finally: - os.chdir(cwd) - - cmd = compile_cmd("clang", "/s", None, "o.wasm", True) - for f in CFLAGS + LDFLAGS + CPU + ["-msimd128", "-mexec-model=reactor", "-lc", "--no-wasm-opt"]: - assert f in cmd, f - log("@@SELFTEST PASS") - - -# ---------------------------------------------------------------- cli - -def main(argv): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - sp = ap.add_subparsers(dest="cmd", required=True) - p = sp.add_parser("check-tree") - p.add_argument("--undici", required=True) - p.add_argument("--npm-undici", required=True) - p = sp.add_parser("build") - p.add_argument("src") - p.add_argument("outdir") - p.add_argument("--version", required=True) - p.add_argument("--llhttp", required=True) - p.add_argument("--cc", default="clang") - p.add_argument("--sysroot", default="/usr/lib/wasi", help="packages/wasi-libc install root") - p.add_argument("--rt", help="explicit wasm32 builtins archive, if llhttp needs one") - p.add_argument("--dry-run", action="store_true") - p = sp.add_parser("splice") - p.add_argument("carrier") - p.add_argument("shipped_sha") - p.add_argument("rebuilt") - p = sp.add_parser("census") - p.add_argument("roots", nargs="+") - p.add_argument("-C", dest="chdir") - p.add_argument("--wasm", action="append", default=[], help="rebuilt blob whose sha is allowed") - p.add_argument("--allow-path", action="append", default=[], help="fnmatch glob, relative to -C") - p.add_argument("--expect", action="append", default=[], help="PATH=N (exact) or PATH=N+ allowed blobs in PATH") - p.add_argument("--allow-raw", action="append", default=[], help="PATH=N: exactly N undecodable raw wasm headers in PATH") - p = sp.add_parser("smoke") - p.add_argument("node") - p.add_argument("wasm", nargs="+") - p.add_argument("--fetch", action="store_true") - p.add_argument("--require", action="append", default=[]) - sp.add_parser("selftest") - a = ap.parse_args(argv) - try: - if a.cmd == "check-tree": - check_tree(a.undici, a.npm_undici) - elif a.cmd == "build": - build(a.src, a.outdir, a.version, a.llhttp, a.cc, a.sysroot, a.rt, a.dry_run) - elif a.cmd == "splice": - splice(a.carrier, a.shipped_sha, a.rebuilt) - elif a.cmd == "census": - shas = [sha(open(w, "rb").read()) for w in a.wasm] - if a.chdir: - os.chdir(a.chdir) - expect = {} - for e in a.expect: - k, _, v = e.rpartition("=") - expect[os.path.normpath(k)] = (int(v.rstrip("+")), v.endswith("+")) - raw = {} - for e in a.allow_raw: - k, _, v = e.rpartition("=") - raw[os.path.normpath(k)] = int(v) - census(a.roots, shas, a.allow_path, expect, raw) - elif a.cmd == "smoke": - smoke(a.node, a.wasm, a.fetch, a.require) - elif a.cmd == "selftest": - selftest() - except Fail as e: - print("ERROR: %s" % e, file=sys.stderr) - return 1 - except subprocess.CalledProcessError as e: - print("ERROR: %s" % e, file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) From 26aa7e3152f29154c3271f37c0d9e7184cc56454 Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 15 Oct 2026 09:40:47 -0700 Subject: [PATCH 05/16] next: rebuild the edge runtime's llhttp wasm from source The edge runtime bundles undici 6's llhttp pair as base64 in four files. Rebuild it from undici 6.28.0's C with our clang and wasi-libc, splice it over the sha-pinned shipped blobs, and census the installed tree so any wasm not on the exact-path allowlist fails the build. Co-Authored-By: Claude Opus 5.5 --- packages/next/build.ncl | 13 +++++++++++++ packages/next/build.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/next/build.ncl b/packages/next/build.ncl index 16b2ae662..189b05754 100644 --- a/packages/next/build.ncl +++ b/packages/next/build.ncl @@ -9,6 +9,12 @@ let pkgconf = import "../pkgconf/build.ncl" in let pnpm = import "../pnpm/build.ncl" in let python = import "../python/build.ncl" in let toolchain = import "../toolchain/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in + +# undici whose llhttp C rebuilds the wasm next bundles into its edge runtime. +let undici = "6.28.0" in let version = "16.4.1" in { @@ -21,6 +27,13 @@ let version = "16.4.1" in extract = true, strip_prefix = "next.js-%{version}", } | Source, + { + url = "gs://minimal-staging-archives/undici-%{undici}.tar.gz", + sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", + } | Source, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, + llhttp-wasm-rebuild, base, glib, make, diff --git a/packages/next/build.sh b/packages/next/build.sh index 9715cf49c..5b1c723de 100755 --- a/packages/next/build.sh +++ b/packages/next/build.sh @@ -15,6 +15,8 @@ export CXX=g++ export PNPM_CONFIG_STRICT_DEP_BUILDS=false export PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN=false +SRC_ROOT=$PWD + # Install all dependencies using the repo's pnpm-lock.yaml for reproducibility. # The lockfile pins exact versions of every transitive dependency used during the build. pnpm install --frozen-lockfile @@ -69,6 +71,33 @@ done NEXT_DIR="$OUTPUT_DIR/usr/libexec/next/lib/node_modules/next" +# llhttp wasm: the edge runtime bundles undici 6's llhttp pair base64 in four files. +# Rebuilt from undici's C and spliced over the sha-pinned shipped blobs. +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI=6.28.0 +mkdir -p /tmp/llhttp +tar -xof "$SRC_ROOT/undici-$UNDICI.tar.gz" -C /tmp/llhttp +$LH build "/tmp/llhttp/undici-$UNDICI" /tmp/llhttp/wasm --version "$UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +LH_CARRIERS="dist/compiled/edge-runtime/index.js dist/compiled/@edge-runtime/primitives/fetch.js + dist/compiled/@edge-runtime/primitives/load.js dist/compiled/@vercel/blob/index.cjs" +LH_EXPECT= +for c in $LH_CARRIERS; do + $LH splice "$NEXT_DIR/$c" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 /tmp/llhttp/wasm/llhttp.wasm + $LH splice "$NEXT_DIR/$c" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d /tmp/llhttp/wasm/llhttp_simd.wasm + LH_EXPECT="$LH_EXPECT --expect $c=2" +done +# Every other wasm in the tree is still upstream's, by exact path: a new carrier fails here. +# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten), +# webpack's and loader-utils' xxhash64/md4 (AssemblyScript), source-map08 (Rust). +$LH census -C "$NEXT_DIR" --wasm /tmp/llhttp/wasm/llhttp.wasm --wasm /tmp/llhttp/wasm/llhttp_simd.wasm $LH_EXPECT \ + --allow-path 'dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/llhttp.wasm' \ + --allow-path 'dist/compiled/@vercel/og/*' \ + --allow-path 'dist/compiled/loader-utils2/index.js' --allow-path 'dist/compiled/loader-utils3/index.js' \ + --allow-path 'dist/compiled/webpack/bundle5.js' --allow-path 'dist/compiled/source-map08/mappings.wasm' \ + . +$LH smoke node /tmp/llhttp/wasm/llhttp.wasm /tmp/llhttp/wasm/llhttp_simd.wasm \ + --require "$NEXT_DIR/dist/compiled/edge-runtime" + # Build sharp from source against our system libvips SHARP_STAGING=$(mktemp -d) cd "$SHARP_STAGING" From 83f98318c0f4c54ceecc8eeee8ed376a7d6b6e59 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 13:45:24 -0700 Subject: [PATCH 06/16] wasi-libc, node, node-lts, next: address review - wasi-libc: install the bundled cloudlibc LICENSE (BSD-2-Clause) and musl COPYRIGHT (MIT) next to the top-level licenses, renamed to LICENSE-cloudlibc / COPYRIGHT-musl because cloudlibc's file is also called LICENSE. - wasi-libc: comment on the Source naming the upstream tag archive and how to audit the mirror (sha256, and git get-tar-commit-id = the tag commit). - node, node-lts: forward npm_undici through build_args (MINIMAL_ARG_NPM_UNDICI); next: forward undici (MINIMAL_ARG_UNDICI). The version is now written once, in build.ncl. Co-Authored-By: Claude Opus 5.5 --- packages/next/build.ncl | 2 ++ packages/next/build.sh | 2 +- packages/node-lts/build.ncl | 6 ++++-- packages/node-lts/build.sh | 2 +- packages/node/build.ncl | 6 ++++-- packages/node/build.sh | 2 +- packages/wasi-libc/build.ncl | 5 +++++ packages/wasi-libc/build.sh | 4 ++++ 8 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/next/build.ncl b/packages/next/build.ncl index 189b05754..5fc830477 100644 --- a/packages/next/build.ncl +++ b/packages/next/build.ncl @@ -14,6 +14,7 @@ let wasi-libc = import "../wasi-libc/build.ncl" in let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in # undici whose llhttp C rebuilds the wasm next bundles into its edge runtime. +# Forwarded to build.sh as MINIMAL_ARG_UNDICI. let undici = "6.28.0" in let version = "16.4.1" in @@ -58,6 +59,7 @@ let version = "16.4.1" in cmd = "./build.sh", build_args = { include version, + include undici, }, outputs = { diff --git a/packages/next/build.sh b/packages/next/build.sh index 5b1c723de..a26fc2997 100755 --- a/packages/next/build.sh +++ b/packages/next/build.sh @@ -74,7 +74,7 @@ NEXT_DIR="$OUTPUT_DIR/usr/libexec/next/lib/node_modules/next" # llhttp wasm: the edge runtime bundles undici 6's llhttp pair base64 in four files. # Rebuilt from undici's C and spliced over the sha-pinned shipped blobs. LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" -UNDICI=6.28.0 +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" mkdir -p /tmp/llhttp tar -xof "$SRC_ROOT/undici-$UNDICI.tar.gz" -C /tmp/llhttp $LH build "/tmp/llhttp/undici-$UNDICI" /tmp/llhttp/wasm --version "$UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index 470b7e8ec..3e5c470e2 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -24,7 +24,8 @@ let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in let patch = import "../patch/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. -let npm-undici = "6.26.0" in +# Forwarded to build.sh as MINIMAL_ARG_NPM_UNDICI. +let npm_undici = "6.26.0" in let version = "24.19.0" in { name = "node-lts", @@ -35,7 +36,7 @@ let version = "24.19.0" in sha256 = "16fe258006a6e86844fbe05b3b5e1e5623ca8d3da54e32d98d9e83234bf25b01", } | Source, { - url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", + url = "gs://minimal-staging-archives/undici-%{npm_undici}.tar.gz", sha256 = "fc371d680e507ddf32f4944cf6b355a520f16ef5fc4987e0f25dc668372475c7", } | Source, # CVE-2026-111943 (eMudhra key compromise): backports of nodejs/node#66871 @@ -77,6 +78,7 @@ let version = "24.19.0" in cmd = "./build.sh", build_args = { include version, + include npm_undici, }, outputs = { diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 8c7632792..4e2f1630d 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -37,7 +37,7 @@ fi # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" -NPM_UNDICI=6.26.0 +NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici 7.29.0 --npm-undici "$NPM_UNDICI" $LH build deps/undici/src ../wasm --version 7.29.0 --llhttp 9.3.0 --sysroot /usr/lib/wasi diff --git a/packages/node/build.ncl b/packages/node/build.ncl index 284fd5416..12dec87ce 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -25,7 +25,8 @@ let wasi-libc = import "../wasi-libc/build.ncl" in let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. -let npm-undici = "6.28.0" in +# Forwarded to build.sh as MINIMAL_ARG_NPM_UNDICI. +let npm_undici = "6.28.0" in let version = "26.10.0" in { name = "node", @@ -39,7 +40,7 @@ let version = "26.10.0" in # the patch header. Build-time only: it rewrites the source tree. { file = "lief-1.x-macho-section-ctor.patch" } | Local, { - url = "gs://minimal-staging-archives/undici-%{npm-undici}.tar.gz", + url = "gs://minimal-staging-archives/undici-%{npm_undici}.tar.gz", sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", } | Source, # CVE-2026-111943 (eMudhra key compromise): backports of nodejs/node#66871 @@ -80,6 +81,7 @@ let version = "26.10.0" in cmd = "./build.sh", build_args = { include version, + include npm_undici, }, outputs = { diff --git a/packages/node/build.sh b/packages/node/build.sh index f06956b9b..8049de531 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -46,7 +46,7 @@ fi # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" -NPM_UNDICI=6.28.0 +NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici 8.10.2 --npm-undici "$NPM_UNDICI" $LH build deps/undici/src ../wasm --version 8.10.2 --llhttp 9.3.1 --sysroot /usr/lib/wasi diff --git a/packages/wasi-libc/build.ncl b/packages/wasi-libc/build.ncl index d9a147287..feec65ff9 100644 --- a/packages/wasi-libc/build.ncl +++ b/packages/wasi-libc/build.ncl @@ -11,6 +11,11 @@ let tag = "wasi-sdk-21" in build_deps = [ { file = "build.sh" } | Local, + # Mirror of the upstream tag archive + # https://github.com/WebAssembly/wasi-libc/archive/refs/tags/%{tag}.tar.gz + # To audit it, fetch that URL and compare sha256. The archive is a + # `git archive`, so `gunzip -c | git get-tar-commit-id` must also + # print the commit the tag points at (c5264e2bbe532994d06b039005f2af91bedcc1a6). { url = "gs://minimal-staging-archives/wasi-libc-%{tag}.tar.gz", sha256 = "4a2a3e3b120ba1163c57f34ac79c3de720a8355ee3a753d81f1f0c58c4cf6017", diff --git a/packages/wasi-libc/build.sh b/packages/wasi-libc/build.sh index 1d5acb2ab..64b3c1338 100755 --- a/packages/wasi-libc/build.sh +++ b/packages/wasi-libc/build.sh @@ -36,3 +36,7 @@ cp -r sysroot/include sysroot/lib "$OUTPUT_DIR/usr/lib/wasi/" mkdir -p "$OUTPUT_DIR/usr/share/licenses/wasi-libc" cp LICENSE LICENSE-APACHE LICENSE-APACHE-LLVM LICENSE-MIT \ "$OUTPUT_DIR/usr/share/licenses/wasi-libc/" +# The bundled third-party notices: cloudlibc (BSD-2-Clause) and musl (MIT). +# Renamed on install because cloudlibc's file is also called LICENSE. +cp libc-bottom-half/cloudlibc/LICENSE "$OUTPUT_DIR/usr/share/licenses/wasi-libc/LICENSE-cloudlibc" +cp libc-top-half/musl/COPYRIGHT "$OUTPUT_DIR/usr/share/licenses/wasi-libc/COPYRIGHT-musl" From 8e293ebb9435c45eb1386ebc0a3003c3b93aea32 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 18:56:44 -0700 Subject: [PATCH 07/16] wasm-bindgen-cli 0.2.100, amaro-wasm: amaro's swc TypeScript stripper from source (WIP) Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.ncl | 87 +++++++++++++++++++++++++++++ packages/amaro-wasm/build.sh | 53 ++++++++++++++++++ packages/wasm-bindgen-cli/build.ncl | 67 ++++++++++++++++++++++ packages/wasm-bindgen-cli/build.sh | 11 ++++ 4 files changed, 218 insertions(+) create mode 100644 packages/amaro-wasm/build.ncl create mode 100755 packages/amaro-wasm/build.sh create mode 100644 packages/wasm-bindgen-cli/build.ncl create mode 100755 packages/wasm-bindgen-cli/build.sh diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl new file mode 100644 index 000000000..7def72e25 --- /dev/null +++ b/packages/amaro-wasm/build.ncl @@ -0,0 +1,87 @@ +let { Attrs, BuildSpec, Local, Needs, OutputData, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let rust = import "../rust/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in +let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in + +# The amaro versions node (1.1.11) and node-lts (1.1.10) bundle. Each tag vendors the swc tree its +# TypeScript stripper is built from (deps/swc, with Cargo.lock). +let amaro_versions = ["1.1.10", "1.1.11"] in +# the rust package's version; build.sh fails if rustc is not this version +let rust_version = "1.99.0" in +{ + name = "amaro-wasm", + # amaro is node's built-in TypeScript type stripper (`node x.ts`): swc's + # binding_typescript_wasm, compiled to wasm32-unknown-unknown and run through wasm-bindgen, + # base64-inlined into amaro's dist/index.js. This rebuilds that module from the vendored swc + # source, the way amaro's tools/build-wasm.js does in its container (cargo --release with fat LTO + # and opt-level z, wasm-bindgen --target nodejs, no wasm-opt), so node and node-lts can splice it + # over the shipped blob. + # + # The rust package ships no wasm32 std and no rust-lld: std is built with -Zbuild-std from the + # rustc source tarball of the same version, and llvm's wasm-ld links. + build_deps = [ + { file = "build.sh" } | Local, + { + url = "https://github.com/nodejs/amaro/archive/refs/tags/v1.1.10.tar.gz", + sha256 = "fdb7f889ab6a52f0477d40cecb7d2248c699793c36359ea8c3f4bed031da7fd6", + extract = false, + } | Source, + { + url = "https://github.com/nodejs/amaro/archive/refs/tags/v1.1.11.tar.gz", + sha256 = "f65b4e8ae9ab519e8db1824377eb621d7fce66e21ae1020621d8f54b6ed774fa", + extract = false, + } | Source, + { + # the rust package's own source + url = "gs://minimal-staging-archives/rustc-%{rust_version}-src.tar.xz", + sha256 = "cc41916a8c84f5d9ec4f55561b44e39f43b647b133f8a6f51be9ea13c83f7036", + extract = false, + } | Source, + base, + llvm, + rust, + toolchain, + wasm-bindgen-cli, + ], + + # cargo fetches the crates in swc's Cargo.lock and std's library/Cargo.lock (--locked). + needs = + { + dns = {}, + internet = {}, + } | Needs, + + cmd = "./build.sh", + build_args = { + amaro_versions = std.string.join " " amaro_versions, + rust_version = rust_version, + }, + + outputs = { + wasm = { glob = "usr/share/amaro-wasm/**" } | OutputData, + }, + + attrs = + { + upstream_version = "1.1.11", + license_spdx = "MIT AND Apache-2.0", + source_provenance = { + category = 'GithubRepo, + owner = "nodejs", + repo = "amaro", + }, + } | Attrs, + + tests = { + modules = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "for v in 1.1.10 1.1.11; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/amaro-wasm/build.sh b/packages/amaro-wasm/build.sh new file mode 100755 index 000000000..c35529dff --- /dev/null +++ b/packages/amaro-wasm/build.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -ex + +RV="${MINIMAL_ARG_RUST_VERSION:?build.ncl forwards rust_version}" +VERSIONS="${MINIMAL_ARG_AMARO_VERSIONS:?build.ncl forwards amaro_versions}" +TOP=$(pwd) + +# std for wasm32 comes from the rustc source of the exact rustc in use. +case "$(rustc --version)" in "rustc $RV "*) ;; *) echo "rustc is not $RV: update rust_version and its sha" >&2; exit 1 ;; esac +tar -xof "rustc-$RV-src.tar.xz" "rustc-$RV-src/library" +LIB="$TOP/rustc-$RV-src/library" +export __CARGO_TESTS_ONLY_SRC_ROOT="$LIB" +export RUSTC_BOOTSTRAP=1 + +# Host artifacts (build scripts, proc macros) link with gcc; swc's .cargo/config.toml names +# cross linkers for the linux targets. +export CC=gcc +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_LINKER=wasm-ld +# amaro's swc scripts/build.sh +export CARGO_PROFILE_RELEASE_LTO=fat +export CARGO_PROFILE_RELEASE_OPT_LEVEL=z + +# build : the module amaro's tools/build-wasm.js produces, as +# /pkg/wasm_bg.wasm (before patch.mjs inlines it). +build() { + v=$1; w=$2 + rm -rf "$w"; mkdir -p "$w" + tar -xof "$TOP/v$v.tar.gz" -C "$w" "amaro-$v/deps/swc" + cd "$w/amaro-$v/deps/swc" + # The env rustflags replace binding_typescript_wasm/.cargo/config.toml's, so its stack size is + # repeated here. + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C link-args=-z stack-size=2097152 --remap-path-prefix=$w=/builddir --remap-path-prefix=$LIB=/rustc/library --remap-path-prefix=$HOME/.cargo=/cargo" \ + cargo build --release --locked -Zbuild-std=std,panic_abort --target wasm32-unknown-unknown -p binding_typescript_wasm + wasm-bindgen --target nodejs --out-name wasm --out-dir pkg --typescript \ + target/wasm32-unknown-unknown/release/binding_typescript_wasm.wasm + cd "$TOP" +} + +for v in $VERSIONS; do + # Built twice, at different paths: the module must not depend on where it was built. + build "$v" "$TOP/a-$v" + build "$v" "$TOP/b-$v" + A="$TOP/a-$v/amaro-$v/deps/swc/pkg/wasm_bg.wasm" + B="$TOP/b-$v/amaro-$v/deps/swc/pkg/wasm_bg.wasm" + cmp "$A" "$B" || { echo "amaro $v: two builds of wasm_bg.wasm differ" >&2; exit 1; } + D="$OUTPUT_DIR/usr/share/amaro-wasm/$v" + mkdir -p "$D" + cp "$A" "$D/wasm_bg.wasm" + cp "$TOP/a-$v/amaro-$v/deps/swc/pkg/wasm.js" "$D/wasm.js" + sha256sum "$D/wasm_bg.wasm" +done diff --git a/packages/wasm-bindgen-cli/build.ncl b/packages/wasm-bindgen-cli/build.ncl new file mode 100644 index 000000000..19ac61e81 --- /dev/null +++ b/packages/wasm-bindgen-cli/build.ncl @@ -0,0 +1,67 @@ +let { subsetOf, Attrs, BuildSpec, Local, Needs, OutputBin, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let rust = import "../rust/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in + +let gcc = import "../gcc/build.ncl" in +let glibc = import "../glibc/build.ncl" in + +let version = "0.2.100" in +{ + name = "wasm-bindgen-cli", + # The wasm-bindgen post-processor: turns a rustc wasm32-unknown-unknown module plus its + # wasm-bindgen custom sections into the final module and its JS glue. Only the `wasm-bindgen` + # binary ships; its version must equal the wasm-bindgen crate locked by the code it processes. + build_deps = [ + { file = "build.sh" } | Local, + { + url = "https://static.crates.io/crates/wasm-bindgen-cli/wasm-bindgen-cli-%{version}.crate", + sha256 = "9e7f06f41580c87b6f4f84176d033f09c0838db1523a8d1899064b53cbb92c59", + extract = true, + strip_prefix = "wasm-bindgen-cli-%{version}", + } | Source, + base, + rust, + toolchain, + ], + + runtime_deps = [ + glibc, + subsetOf gcc ["libgcc"], + ], + + # cargo fetches the crates in the published Cargo.lock (--locked) during the build. + needs = + { + dns = {}, + internet = {}, + } | Needs, + + cmd = "./build.sh", + + outputs = { + bins = { glob = "usr/bin/wasm-bindgen" } | OutputBin, + }, + + attrs = + { + upstream_version = version, + license_spdx = "MIT OR Apache-2.0", + source_provenance = { + category = 'GithubRepo, + owner = "rustwasm", + repo = "wasm-bindgen", + }, + } | Attrs, + + tests = { + version = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "wasm-bindgen --version | grep -qx 'wasm-bindgen %{version}'"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/wasm-bindgen-cli/build.sh b/packages/wasm-bindgen-cli/build.sh new file mode 100755 index 000000000..3439b1a6e --- /dev/null +++ b/packages/wasm-bindgen-cli/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -ex + +export CC=gcc +export LD=gcc +export RUSTFLAGS="-C linker=gcc --remap-path-prefix=$(pwd)=/builddir --remap-path-prefix=$HOME/.cargo=/cargo -C codegen-units=1" + +cargo build --release --locked --bin wasm-bindgen + +mkdir -p "$OUTPUT_DIR/usr/bin" +cp target/release/wasm-bindgen "$OUTPUT_DIR/usr/bin/" From 242af2bb7b6bdbeadb62eae28398627cf3d3d645 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 18:58:22 -0700 Subject: [PATCH 08/16] llhttp-wasm-rebuild: python as a runtime dep; next: name og's three wasm carriers The installed script runs under python3 in node, node-lts and next, so python is a runtime dep (it stays a build dep for the build-time selftest). next's census exception for @vercel/og was a glob; it now names the three files that carry wasm (resvg.wasm, yoga.wasm, and yoga again as base64 in index.node.js), so a new carrier there fails the build. Co-Authored-By: Claude Opus 5.5 --- packages/llhttp-wasm-rebuild/build.ncl | 4 +++- packages/next/build.sh | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/llhttp-wasm-rebuild/build.ncl b/packages/llhttp-wasm-rebuild/build.ncl index 4fe3925b8..1526565c3 100644 --- a/packages/llhttp-wasm-rebuild/build.ncl +++ b/packages/llhttp-wasm-rebuild/build.ncl @@ -10,8 +10,10 @@ let python = import "../python/build.ncl" in { file = "build.sh" } | Local, { file = "rebuild-llhttp-wasm.py" } | Local, base, - python, + python, # build.sh runs the selftest ], + # node, node-lts and next run the installed script with python3 + runtime_deps = [python], cmd = "./build.sh", outputs = { script = { glob = "usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" } | OutputData, diff --git a/packages/next/build.sh b/packages/next/build.sh index a26fc2997..e265922a5 100755 --- a/packages/next/build.sh +++ b/packages/next/build.sh @@ -87,11 +87,13 @@ for c in $LH_CARRIERS; do LH_EXPECT="$LH_EXPECT --expect $c=2" done # Every other wasm in the tree is still upstream's, by exact path: a new carrier fails here. -# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten), +# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten; also +# base64 in index.node.js), # webpack's and loader-utils' xxhash64/md4 (AssemblyScript), source-map08 (Rust). $LH census -C "$NEXT_DIR" --wasm /tmp/llhttp/wasm/llhttp.wasm --wasm /tmp/llhttp/wasm/llhttp_simd.wasm $LH_EXPECT \ --allow-path 'dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/llhttp.wasm' \ - --allow-path 'dist/compiled/@vercel/og/*' \ + --allow-path 'dist/compiled/@vercel/og/resvg.wasm' --allow-path 'dist/compiled/@vercel/og/yoga.wasm' \ + --allow-path 'dist/compiled/@vercel/og/index.node.js' \ --allow-path 'dist/compiled/loader-utils2/index.js' --allow-path 'dist/compiled/loader-utils3/index.js' \ --allow-path 'dist/compiled/webpack/bundle5.js' --allow-path 'dist/compiled/source-map08/mappings.wasm' \ . From e76fb1e94b707b3c616b520a019094249a7827e3 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 18:59:47 -0700 Subject: [PATCH 09/16] wasm-bindgen-cli: name the test version_check (version shadowed the binding) Co-Authored-By: Claude Opus 5.5 --- packages/wasm-bindgen-cli/build.ncl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wasm-bindgen-cli/build.ncl b/packages/wasm-bindgen-cli/build.ncl index 19ac61e81..273a8081b 100644 --- a/packages/wasm-bindgen-cli/build.ncl +++ b/packages/wasm-bindgen-cli/build.ncl @@ -55,7 +55,7 @@ let version = "0.2.100" in } | Attrs, tests = { - version = + version_check = { class = 'Standalone, test_deps = [base], From f93022f541a665aa2129d2e89a41b4d7c0ed9763 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 19:00:22 -0700 Subject: [PATCH 10/16] amaro-wasm: let bindings named apart from build_args fields Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.ncl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl index 7def72e25..39e7a3259 100644 --- a/packages/amaro-wasm/build.ncl +++ b/packages/amaro-wasm/build.ncl @@ -7,9 +7,9 @@ let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in # The amaro versions node (1.1.11) and node-lts (1.1.10) bundle. Each tag vendors the swc tree its # TypeScript stripper is built from (deps/swc, with Cargo.lock). -let amaro_versions = ["1.1.10", "1.1.11"] in +let amaro = ["1.1.10", "1.1.11"] in # the rust package's version; build.sh fails if rustc is not this version -let rust_version = "1.99.0" in +let rustc = "1.99.0" in { name = "amaro-wasm", # amaro is node's built-in TypeScript type stripper (`node x.ts`): swc's @@ -35,7 +35,7 @@ let rust_version = "1.99.0" in } | Source, { # the rust package's own source - url = "gs://minimal-staging-archives/rustc-%{rust_version}-src.tar.xz", + url = "gs://minimal-staging-archives/rustc-%{rustc}-src.tar.xz", sha256 = "cc41916a8c84f5d9ec4f55561b44e39f43b647b133f8a6f51be9ea13c83f7036", extract = false, } | Source, @@ -55,8 +55,8 @@ let rust_version = "1.99.0" in cmd = "./build.sh", build_args = { - amaro_versions = std.string.join " " amaro_versions, - rust_version = rust_version, + amaro_versions = std.string.join " " amaro, + rust_version = rustc, }, outputs = { From d5975d8a847ddceac1c49fc70d7b071cbeab82d0 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 19:03:07 -0700 Subject: [PATCH 11/16] wasm-bindgen-cli: unpack the .crate in build.sh Co-Authored-By: Claude Opus 5.5 --- packages/wasm-bindgen-cli/build.ncl | 7 +++++-- packages/wasm-bindgen-cli/build.sh | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/wasm-bindgen-cli/build.ncl b/packages/wasm-bindgen-cli/build.ncl index 273a8081b..5a2289559 100644 --- a/packages/wasm-bindgen-cli/build.ncl +++ b/packages/wasm-bindgen-cli/build.ncl @@ -17,8 +17,8 @@ let version = "0.2.100" in { url = "https://static.crates.io/crates/wasm-bindgen-cli/wasm-bindgen-cli-%{version}.crate", sha256 = "9e7f06f41580c87b6f4f84176d033f09c0838db1523a8d1899064b53cbb92c59", - extract = true, - strip_prefix = "wasm-bindgen-cli-%{version}", + # a .crate is a gzipped tar; build.sh unpacks it (Source extraction keys on the extension) + extract = false, } | Source, base, rust, @@ -38,6 +38,9 @@ let version = "0.2.100" in } | Needs, cmd = "./build.sh", + build_args = { + include version, + }, outputs = { bins = { glob = "usr/bin/wasm-bindgen" } | OutputBin, diff --git a/packages/wasm-bindgen-cli/build.sh b/packages/wasm-bindgen-cli/build.sh index 3439b1a6e..a4de81451 100755 --- a/packages/wasm-bindgen-cli/build.sh +++ b/packages/wasm-bindgen-cli/build.sh @@ -1,6 +1,10 @@ #!/bin/sh set -ex +V="${MINIMAL_ARG_VERSION:?build.ncl forwards version}" +tar -xzof "wasm-bindgen-cli-$V.crate" +cd "wasm-bindgen-cli-$V" + export CC=gcc export LD=gcc export RUSTFLAGS="-C linker=gcc --remap-path-prefix=$(pwd)=/builddir --remap-path-prefix=$HOME/.cargo=/cargo -C codegen-units=1" From 0bc74cf878731fa31744f513a8d3d752d0c27bcb Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 19:06:09 -0700 Subject: [PATCH 12/16] amaro-wasm: build from the binding directory; stack size as one rustflag token Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/amaro-wasm/build.sh b/packages/amaro-wasm/build.sh index c35529dff..30490358a 100755 --- a/packages/amaro-wasm/build.sh +++ b/packages/amaro-wasm/build.sh @@ -28,11 +28,13 @@ build() { v=$1; w=$2 rm -rf "$w"; mkdir -p "$w" tar -xof "$TOP/v$v.tar.gz" -C "$w" "amaro-$v/deps/swc" + # From the binding's directory, as wasm-pack runs it, so its .cargo/config.toml (the 2 MiB + # stack) applies along with the workspace's; the stack size is repeated in case the env value + # replaces rather than extends it. + cd "$w/amaro-$v/deps/swc/bindings/binding_typescript_wasm" + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-Clink-arg=-zstack-size=2097152 --remap-path-prefix=$w=/builddir --remap-path-prefix=$LIB=/rustc/library --remap-path-prefix=$HOME/.cargo=/cargo" \ + cargo build --release --locked -Zbuild-std=std,panic_abort --target wasm32-unknown-unknown cd "$w/amaro-$v/deps/swc" - # The env rustflags replace binding_typescript_wasm/.cargo/config.toml's, so its stack size is - # repeated here. - CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-C link-args=-z stack-size=2097152 --remap-path-prefix=$w=/builddir --remap-path-prefix=$LIB=/rustc/library --remap-path-prefix=$HOME/.cargo=/cargo" \ - cargo build --release --locked -Zbuild-std=std,panic_abort --target wasm32-unknown-unknown -p binding_typescript_wasm wasm-bindgen --target nodejs --out-name wasm --out-dir pkg --typescript \ target/wasm32-unknown-unknown/release/binding_typescript_wasm.wasm cd "$TOP" From d96479561f114ddd7893b32faf68594df03aaa25 Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 21 Oct 2026 19:19:59 -0700 Subject: [PATCH 13/16] node, node-lts: keep amaro, with its wasm rebuilt from swc amaro is node's built-in TypeScript type stripper (node x.ts), on by default in 24 and 26. packages/amaro-wasm rebuilds its swc module from the source amaro vendors (rust with -Zbuild-std for wasm32, llvm's wasm-ld, wasm-bindgen-cli 0.2.100, each module built twice), and node and node-lts splice it over the sha-pinned shipped blob in deps/amaro/dist/index.js instead of configuring --without-amaro. The census admits it like the llhttp blobs, a build step and a test run a .ts file. rebuild-llhttp-wasm.py splice --wasm-bindgen: wasm-bindgen numbers its closure-wrapper shim after a function index, so the rebuilt module differs from upstream's in that one import name. The flag allows exactly that difference (same module, kind and signature), renames the identifier in the carrier's glue (which must name it exactly once), and admits the features the shipped blob already uses. Selftests cover both verdicts. Checked against the real carriers: 61 imports and 10 exports match for amaro 1.1.11 (node 26) and 1.1.10 (node 24). Co-Authored-By: Claude Opus 5.5 --- .../rebuild-llhttp-wasm.py | 79 +++++++++++++++---- packages/node-lts/build.ncl | 11 +++ packages/node-lts/build.sh | 18 +++-- packages/node/build.ncl | 11 +++ packages/node/build.sh | 18 +++-- 5 files changed, 111 insertions(+), 26 deletions(-) diff --git a/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py index 8bc679ac2..598003e4e 100755 --- a/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py +++ b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py @@ -357,17 +357,32 @@ def wasm_info(b): return {"imports": imports, "exports": exports, "feats": feats, "ids": set(ids)} -def gate(shipped, rebuilt, label): - """Rebuilt blob must present the shipped blob's tables and no extra wasm features.""" +WBG_CLOSURE = re.compile(r"__wbindgen_closure_wrapper[0-9]+") + + +def gate(shipped, rebuilt, label, wbg=False): + """Rebuilt blob must present the shipped blob's tables and no extra wasm features. + + wbg (wasm-bindgen modules): wasm-bindgen numbers its closure-wrapper shim after a function + index, so one import may differ, if both sides are such a shim with the same module, kind and + signature; it is returned as (shipped name, rebuilt name) for the carrier's glue to follow. + Features may be any the shipped blob already uses.""" a, r = wasm_info(shipped), wasm_info(rebuilt) - errs = [] + errs, renames = [], [] for k in ("imports", "exports"): - if a[k] != r[k]: - errs.append("%s differ: missing %s, extra %s" % (k, sorted(a[k] - r[k]), sorted(r[k] - a[k]))) + miss, extra = a[k] - r[k], r[k] - a[k] + if wbg and k == "imports": + m = [x for x in miss if WBG_CLOSURE.fullmatch(x[1])] + e = [x for x in extra if WBG_CLOSURE.fullmatch(x[1])] + if len(m) == 1 and len(e) == 1 and (m[0][0], m[0][2], m[0][3]) == (e[0][0], e[0][2], e[0][3]): + renames.append((m[0][1], e[0][1])) + miss, extra = miss - set(m), extra - set(e) + if miss or extra: + errs.append("%s differ: missing %s, extra %s" % (k, sorted(miss), sorted(extra))) simd = "simd128" in a["feats"] if simd != ("simd128" in r["feats"]): errs.append("simd128 use differs: shipped %s, rebuilt %s" % (simd, not simd)) - allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) + allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) | (set(a["feats"]) if wbg else set()) extra = sorted(set(r["feats"]) - allowed) if extra: errs.append("features not allowed: %s" % ", ".join("%s x%d" % (f, r["feats"][f]) for f in extra)) @@ -375,8 +390,10 @@ def gate(shipped, rebuilt, label): errs.append("sections not in shipped blob: %s" % sorted(r["ids"] - a["ids"])) if errs: raise Fail("%s: gate failed\n " % label + "\n ".join(errs)) - log("@@GATE ok %s imports=%d exports=%d feats=%s" % ( - label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}")) + log("@@GATE ok %s imports=%d exports=%d feats=%s%s" % ( + label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}", + "".join(" rename=%s->%s" % rn for rn in renames))) + return renames # ---------------------------------------------------------------- carriers @@ -433,7 +450,7 @@ def carriers(data): yield (kind, m.start(), None) -def splice(path, pin, rebuilt_path): +def splice(path, pin, rebuilt_path, wbg=False): data = open(path, "rb").read() new = open(rebuilt_path, "rb").read() hits = [] @@ -444,8 +461,14 @@ def splice(path, pin, rebuilt_path): if len(hits) != 1: raise Fail("%s: shipped blob %s found %d times, want exactly 1" % (path, pin[:16], len(hits))) m, shipped = hits[0] - gate(shipped, new, "%s <- %s" % (path, os.path.basename(rebuilt_path))) + renames = gate(shipped, new, "%s <- %s" % (path, os.path.basename(rebuilt_path)), wbg) out = data[:m.start()] + base64.b64encode(new) + data[m.end():] + for old, nw in renames: + o, n = old.encode(), nw.encode() + word = lambda s: re.compile(rb"(? (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" +def mkwasm(imports=("wasm_on_url",), exports=("malloc", "llhttp_execute"), body=b"\x41\x00\x0b", simd=False, module="env"): + """Tiny module: imports . (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" t3 = b"\x60\x03\x7f\x7f\x7f\x01\x7f" types = _sec(1, _vec([t3])) - imp = _sec(2, _vec([_str("env") + _str(n) + b"\x00\x00" for n in imports])) + imp = _sec(2, _vec([_str(module) + _str(n) + b"\x00\x00" for n in imports])) nf = len(exports) fn = _sec(3, _vec([b"\x00"] * nf)) mem = _sec(5, _vec([b"\x00\x01"])) @@ -730,6 +753,17 @@ def selftest(): _expect_fail(wasm_info, bad_unknown, contains="unknown opcode") _expect_fail(wasm_info, b"\0asm\x02\0\0\0", contains="not a wasm v1") + log("selftest: wasm-bindgen gate") + P = "__wbindgen_placeholder__" + wb_shipped = mkwasm(imports=("__wbg_log_1", "__wbindgen_closure_wrapper10"), module=P) + wb_good = mkwasm(imports=("__wbg_log_1", "__wbindgen_closure_wrapper12"), module=P, body=b"\x41\x01\x0b") + wb_two = mkwasm(imports=("__wbg_log_2", "__wbindgen_closure_wrapper12"), module=P, body=b"\x41\x01\x0b") + _expect_fail(gate, wb_shipped, wb_good, "wbg strict", contains="imports differ") + assert gate(wb_shipped, wb_good, "wbg", True) == [("__wbindgen_closure_wrapper10", "__wbindgen_closure_wrapper12")] + _expect_fail(gate, wb_shipped, wb_two, "wbg two", True, contains="imports differ") + gate(bad_bulk, bad_bulk, "wbg shipped feats", True) + _expect_fail(gate, shipped, bad_bulk, "wbg new feats", True, contains="bulk-memory") + cwd = os.getcwd() with tempfile.TemporaryDirectory() as td: def tree(): @@ -770,6 +804,21 @@ def full_pass(root): {"deps/undici/undici.js": (3, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}, contains="expected 3+") + log("selftest: wasm-bindgen splice renames the glue") + os.chdir(td) + glue = ("imports.wbg.__wbindgen_closure_wrapper10 = function() {};\n" + "var bytes = Buffer.from(\"%s\", \"base64\");\n" % base64.b64encode(wb_shipped).decode()) + for name, b in (("wb.js", glue.encode()), ("wb_good.wasm", wb_good)): + with open(name, "wb") as f: + f.write(b) + _expect_fail(splice, "wb.js", sha(wb_shipped), "wb_good.wasm", contains="imports differ") + splice("wb.js", sha(wb_shipped), "wb_good.wasm", True) + got = open("wb.js").read() + assert "__wbindgen_closure_wrapper12 = " in got and "wrapper10" not in got, got[:120] + with open("wb.js", "w") as f: + f.write(glue + "// __wbindgen_closure_wrapper10 again\n") + _expect_fail(splice, "wb.js", sha(wb_shipped), "wb_good.wasm", True, contains="exactly once") + log("selftest: splice rejects") root = tree() os.chdir(root) @@ -873,6 +922,8 @@ def main(argv): p.add_argument("carrier") p.add_argument("shipped_sha") p.add_argument("rebuilt") + p.add_argument("--wasm-bindgen", action="store_true", + help="wasm-bindgen module: allow its renumbered closure-wrapper import (renamed in the glue) and the shipped blob's features") p = sp.add_parser("census") p.add_argument("roots", nargs="+") p.add_argument("-C", dest="chdir") @@ -893,7 +944,7 @@ def main(argv): elif a.cmd == "build": build(a.src, a.outdir, a.version, a.llhttp, a.cc, a.sysroot, a.rt, a.dry_run) elif a.cmd == "splice": - splice(a.carrier, a.shipped_sha, a.rebuilt) + splice(a.carrier, a.shipped_sha, a.rebuilt, a.wasm_bindgen) elif a.cmd == "census": shas = [sha(open(w, "rb").read()) for w in a.wasm] if a.chdir: diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index 3e5c470e2..5d8861a3b 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -21,6 +21,7 @@ let gcc = import "../gcc/build.ncl" in let llvm = import "../llvm/build.ncl" in let wasi-libc = import "../wasi-libc/build.ncl" in let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in +let amaro-wasm = import "../amaro-wasm/build.ncl" in let patch = import "../patch/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. @@ -50,6 +51,7 @@ let version = "24.19.0" in llvm, # clang + wasm-ld for the llhttp wasm wasi-libc, llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs + amaro-wasm, # amaro's TypeScript-stripper wasm, rebuilt from swc make, pkgconf, patch, @@ -107,6 +109,15 @@ let version = "24.19.0" in } | Attrs, tests = { + type_strip = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "printf 'interface P { n: number }\\nconst p: P = { n: 6 };\\nconsole.log(p.n * 7);\\n' > t.ts"], + ["/bin/bash", "-c", "node t.ts | grep -qx 42"], + ], + } | Test, eval = { class = 'Standalone, diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 4e2f1630d..0c56718d8 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -47,13 +47,17 @@ $LH splice deps/undici/undici.js 57184a16144b5bc880e29105fa35d75c93dd318563e1b74 $LH splice deps/undici/undici.js 2434215e1f7fa0163f22ce895ddbeb83984fa0aa51e123164e053213e5aca028 ../wasm/llhttp_simd.wasm $LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm $LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm -# Never installed; amaro (swc wasm) is configured out below. +# Never installed. rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js -rm -r deps/amaro -LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm" +# amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced +# over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. +AMARO=/usr/share/amaro-wasm/1.1.10/wasm_bg.wasm +grep -q '"version": "1.1.10"' deps/amaro/package.json || { echo "deps/amaro is not 1.1.10" >&2; exit 1; } +$LH splice deps/amaro/dist/index.js 8d7f6dd739cbf565ef93a02c5c2a69b512ea06c7e02efb75112662505fcc2c5f "$AMARO" --wasm-bindgen +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" $LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ - --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" --expect deps/amaro/dist/index.js=1 \ deps lib src case $(uname -m) in @@ -67,10 +71,12 @@ export CXXFLAGS="${CFLAGS}" ./configure --prefix=/usr \ --with-intl=system-icu --shared-openssl --shared-zlib --shared-zstd --shared-sqlite --shared-libuv \ - --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-cares \ - --without-amaro + --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-cares # Note: --shared-lief is omitted; that configure option was not added until Node.js v25. make -j$(nproc) +# type stripping runs on the rebuilt amaro +printf 'const n: number = 6;\nconsole.log(n * 7);\n' > ../strip.ts +out/Release/node ../strip.ts | grep -qx 42 $LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install diff --git a/packages/node/build.ncl b/packages/node/build.ncl index 12dec87ce..49675b7f7 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -23,6 +23,7 @@ let gcc = import "../gcc/build.ncl" in let llvm = import "../llvm/build.ncl" in let wasi-libc = import "../wasi-libc/build.ncl" in let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in +let amaro-wasm = import "../amaro-wasm/build.ncl" in # undici bundled under deps/npm; its llhttp C rebuilds npm's wasm blobs. # Forwarded to build.sh as MINIMAL_ARG_NPM_UNDICI. @@ -54,6 +55,7 @@ let version = "26.10.0" in llvm, # clang + wasm-ld for the llhttp wasm wasi-libc, llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs + amaro-wasm, # amaro's TypeScript-stripper wasm, rebuilt from swc make, pkgconf, patch, @@ -110,6 +112,15 @@ let version = "26.10.0" in } | Attrs, tests = { + type_strip = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "printf 'interface P { n: number }\\nconst p: P = { n: 6 };\\nconsole.log(p.n * 7);\\n' > t.ts"], + ["/bin/bash", "-c", "node t.ts | grep -qx 42"], + ], + } | Test, eval = { class = 'Standalone, diff --git a/packages/node/build.sh b/packages/node/build.sh index 8049de531..4ce71d18d 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -56,13 +56,17 @@ $LH splice deps/undici/undici.js ab4573a06c43574936dc98c3943b10a4ee32798e9797329 $LH splice deps/undici/undici.js 18403ee56fab07f89a845028bad358bb3ebcda213f8e2ec0989a991e9c33da69 ../wasm/llhttp_simd.wasm $LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm $LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm -# Never installed; amaro (swc wasm) is configured out below. +# Never installed. rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js -rm -r deps/amaro -LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm" +# amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced +# over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. +AMARO=/usr/share/amaro-wasm/1.1.11/wasm_bg.wasm +grep -q '"version": "1.1.11"' deps/amaro/package.json || { echo "deps/amaro is not 1.1.11" >&2; exit 1; } +$LH splice deps/amaro/dist/index.js 79c79a0be6c92c7d8a1968cb1a7bed88737755d38d2b47ddfe4da5e51796a4e9 "$AMARO" --wasm-bindgen +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" $LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ - --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" --expect deps/amaro/dist/index.js=1 \ deps lib src case $(uname -m) in @@ -76,9 +80,11 @@ export CXXFLAGS="${CFLAGS}" ./configure --prefix=/usr \ --with-intl=system-icu --shared-openssl --shared-zlib --shared-zstd --shared-sqlite --shared-libuv \ - --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-lief --shared-cares \ - --without-amaro + --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-lief --shared-cares make -j$(nproc) +# type stripping runs on the rebuilt amaro +printf 'const n: number = 6;\nconsole.log(n * 7);\n' > ../strip.ts +out/Release/node ../strip.ts | grep -qx 42 $LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install From b0ec7394acfec791d1a2853463db66ac508cbb2f Mon Sep 17 00:00:00 2001 From: bryan Date: Thu, 22 Oct 2026 15:49:34 -0700 Subject: [PATCH 14/16] node, node-lts: re-derive the wasm pins for 26.12.1 / 24.22.1 - Versions come from build.ncl (MINIMAL_ARG_UNDICI, _UNDICI_LLHTTP, _NPM_UNDICI, _AMARO) instead of being typed again in build.sh. - node: undici 8.10.2 -> 8.12.2 (llhttp 9.3.1). undici 8.12 vendors milo (Rust) as two base64 wasm carriers in deps/undici/src/lib/milo; node's bundled undici.js compiles milo out, so like the llhttp source blobs they are never installed and are deleted before the census. - node-lts: undici 7.29.0 -> 7.30.1 (llhttp 9.3.0), npm undici 6.26.0 -> 6.28.0, amaro 1.1.10 -> 1.1.11 (shipped blob 79c79a0b..., node's). - Every decoded shipped blob is byte-identical to a pin this branch already carried, so only the version tripwires and amaro's pin move. - amaro-wasm builds 1.1.11 only: both lines bundle it now. - rebuild-llhttp-wasm.py: undici 8.12 renamed build/wasm.js to build/llhttp.js with the same pinned lines; check_undici_src accepts exactly one of the two, selftest covers the rename and both-or-neither. Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.ncl | 13 +++------- .../rebuild-llhttp-wasm.py | 25 +++++++++++++++---- packages/node-lts/build.sh | 13 ++++++---- packages/node/build.sh | 14 ++++++++--- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl index 39e7a3259..9e7810ded 100644 --- a/packages/amaro-wasm/build.ncl +++ b/packages/amaro-wasm/build.ncl @@ -5,9 +5,9 @@ let rust = import "../rust/build.ncl" in let toolchain = import "../toolchain/build.ncl" in let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in -# The amaro versions node (1.1.11) and node-lts (1.1.10) bundle. Each tag vendors the swc tree its -# TypeScript stripper is built from (deps/swc, with Cargo.lock). -let amaro = ["1.1.10", "1.1.11"] in +# The amaro versions node and node-lts bundle (both 1.1.11 since node 26.12.1 / 24.22.1). Each tag +# vendors the swc tree its TypeScript stripper is built from (deps/swc, with Cargo.lock). +let amaro = ["1.1.11"] in # the rust package's version; build.sh fails if rustc is not this version let rustc = "1.99.0" in { @@ -23,11 +23,6 @@ let rustc = "1.99.0" in # rustc source tarball of the same version, and llvm's wasm-ld links. build_deps = [ { file = "build.sh" } | Local, - { - url = "https://github.com/nodejs/amaro/archive/refs/tags/v1.1.10.tar.gz", - sha256 = "fdb7f889ab6a52f0477d40cecb7d2248c699793c36359ea8c3f4bed031da7fd6", - extract = false, - } | Source, { url = "https://github.com/nodejs/amaro/archive/refs/tags/v1.1.11.tar.gz", sha256 = "f65b4e8ae9ab519e8db1824377eb621d7fce66e21ae1020621d8f54b6ed774fa", @@ -80,7 +75,7 @@ let rustc = "1.99.0" in class = 'Standalone, test_deps = [base], cmds = [ - ["/bin/bash", "-c", "for v in 1.1.10 1.1.11; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], + ["/bin/bash", "-c", "for v in 1.1.11; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], ], } | Test, }, diff --git a/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py index 598003e4e..8ef91be87 100755 --- a/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py +++ b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py @@ -2,7 +2,7 @@ """Regenerate undici's llhttp wasm from in-tree C and replace the shipped blobs. check-tree node tree tripwires: pinned undici / npm-undici versions, no cjs lexer wasm - build pin build/wasm.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes + build pin build/{wasm,llhttp}.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes splice decode the pinned shipped blob in a carrier, gate tables+features, replace it once census every wasm carrier under the roots is an allowed sha or on an allowed path smoke instantiate blobs under a built node and parse a POST; optional loopback fetch @@ -26,7 +26,9 @@ MAGIC = b"\0asm\x01\0\0\0" -# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v8.10.2. +# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v7.30.1, v8.10.2; +# renamed build/llhttp.js in v8.12 (beside the new build/milo.js) with the same lines (v8.12.2). +WASM_JS_NAMES = ("build/wasm.js", "build/llhttp.js") WASM_JS_PINNED = ( "let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi'", "WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor'", @@ -574,11 +576,15 @@ def check_undici_src(src, version, llhttp): srcs = sorted(f for f in os.listdir(os.path.join(src, "deps/llhttp/src")) if f.endswith(".c")) if tuple(srcs) != LLHTTP_SRCS: raise Fail("%s/deps/llhttp/src/*.c is %s, want %s" % (src, srcs, list(LLHTTP_SRCS))) - lines = {l.strip() for l in open(os.path.join(src, "build/wasm.js"))} + found = [n for n in WASM_JS_NAMES if os.path.isfile(os.path.join(src, n))] + if len(found) != 1: + raise Fail("%s: want exactly one of %s, found %s" % (src, list(WASM_JS_NAMES), found)) + script = found[0] + lines = {l.strip() for l in open(os.path.join(src, script))} missing = [l for l in WASM_JS_PINNED if l not in lines] if missing: - raise Fail("%s/build/wasm.js flags drifted; missing:\n %s" % (src, "\n ".join(missing))) - log("@@PIN ok %s/build/wasm.js flags" % src) + raise Fail("%s/%s flags drifted; missing:\n %s" % (src, script, "\n ".join(missing))) + log("@@PIN ok %s/%s flags" % (src, script)) def compile_cmd(cc, sysroot, rt, out, simd): @@ -892,6 +898,15 @@ def full_pass(root): open(os.path.join(src, "build/wasm.js"), "w").write( "\n".join(WASM_JS_PINNED).replace("-Ofast", "-O3") + "\n") _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="flags drifted") + # undici 8.12 renamed build/wasm.js -> build/llhttp.js: either name, never both or neither + os.rename(os.path.join(src, "build/wasm.js"), os.path.join(src, "build/llhttp.js")) + open(os.path.join(src, "build/llhttp.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + check_undici_src(src, "8.10.2", "9.3.1") + open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="want exactly one") + os.unlink(os.path.join(src, "build/wasm.js")) + os.unlink(os.path.join(src, "build/llhttp.js")) + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="want exactly one") finally: os.chdir(cwd) diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 3517f88ae..a54455b17 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -21,10 +21,13 @@ fi # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" +UNDICI_LLHTTP="${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" +AMARO_VERSION="${MINIMAL_ARG_AMARO:?build.ncl forwards amaro}" NPM_LH=deps/npm/node_modules/undici/lib/llhttp -$LH check-tree --undici 7.29.0 --npm-undici "$NPM_UNDICI" -$LH build deps/undici/src ../wasm --version 7.29.0 --llhttp 9.3.0 --sysroot /usr/lib/wasi +$LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. $LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi $LH splice deps/undici/undici.js 57184a16144b5bc880e29105fa35d75c93dd318563e1b743e110de845e2ab661 ../wasm/llhttp.wasm @@ -36,9 +39,9 @@ rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js # amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced # over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. -AMARO=/usr/share/amaro-wasm/1.1.10/wasm_bg.wasm -grep -q '"version": "1.1.10"' deps/amaro/package.json || { echo "deps/amaro is not 1.1.10" >&2; exit 1; } -$LH splice deps/amaro/dist/index.js 8d7f6dd739cbf565ef93a02c5c2a69b512ea06c7e02efb75112662505fcc2c5f "$AMARO" --wasm-bindgen +AMARO=/usr/share/amaro-wasm/$AMARO_VERSION/wasm_bg.wasm +grep -q "\"version\": \"$AMARO_VERSION\"" deps/amaro/package.json || { echo "deps/amaro is not $AMARO_VERSION" >&2; exit 1; } +$LH splice deps/amaro/dist/index.js 79c79a0be6c92c7d8a1968cb1a7bed88737755d38d2b47ddfe4da5e51796a4e9 "$AMARO" --wasm-bindgen LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" $LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" --expect deps/amaro/dist/index.js=1 \ diff --git a/packages/node/build.sh b/packages/node/build.sh index b275755e6..c427993ae 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -21,10 +21,13 @@ fi # llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped # base64 blobs. Every pin fails the build on a node bump until re-derived. LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" +UNDICI_LLHTTP="${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" +AMARO_VERSION="${MINIMAL_ARG_AMARO:?build.ncl forwards amaro}" NPM_LH=deps/npm/node_modules/undici/lib/llhttp -$LH check-tree --undici 8.10.2 --npm-undici "$NPM_UNDICI" -$LH build deps/undici/src ../wasm --version 8.10.2 --llhttp 9.3.1 --sysroot /usr/lib/wasi +$LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. $LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi $LH splice deps/undici/undici.js ab4573a06c43574936dc98c3943b10a4ee32798e9797329e9f327e5f64115f54 ../wasm/llhttp.wasm @@ -34,10 +37,13 @@ $LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e # Never installed. rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js +# undici 8.12 vendors milo (Rust) as two base64 wasm carriers in its source tree; node's +# bundled undici.js compiles milo out (`getMiloEnabled } = false ? null : {}`), so never installed. +rm deps/undici/src/lib/milo/simd.js deps/undici/src/lib/milo/no-simd.js # amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced # over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. -AMARO=/usr/share/amaro-wasm/1.1.11/wasm_bg.wasm -grep -q '"version": "1.1.11"' deps/amaro/package.json || { echo "deps/amaro is not 1.1.11" >&2; exit 1; } +AMARO=/usr/share/amaro-wasm/$AMARO_VERSION/wasm_bg.wasm +grep -q "\"version\": \"$AMARO_VERSION\"" deps/amaro/package.json || { echo "deps/amaro is not $AMARO_VERSION" >&2; exit 1; } $LH splice deps/amaro/dist/index.js 79c79a0be6c92c7d8a1968cb1a7bed88737755d38d2b47ddfe4da5e51796a4e9 "$AMARO" --wasm-bindgen LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" $LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ From 87c64eb367538e0fd9efc120eafa269d5b4c3dfc Mon Sep 17 00:00:00 2001 From: bryan Date: Wed, 28 Oct 2026 17:52:10 -0700 Subject: [PATCH 15/16] amaro-wasm: rustc 1.99.1, matching main's rust The merge from main moved `rust` to 1.99.1, and amaro-wasm's build.sh fails closed when the toolchain isn't the rust_version it pins ("rustc is not 1.99.0"). Its wasm std builds from the matching rustc source tarball: rustc-1.99.1-src.tar.xz, the same mirrored object and sha256 main's packages/rust pins. The output wasm is not hash-pinned (node's splice pins only the upstream blob it replaces), only built twice and compared. Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.ncl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl index 9e7810ded..825a816aa 100644 --- a/packages/amaro-wasm/build.ncl +++ b/packages/amaro-wasm/build.ncl @@ -9,7 +9,7 @@ let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in # vendors the swc tree its TypeScript stripper is built from (deps/swc, with Cargo.lock). let amaro = ["1.1.11"] in # the rust package's version; build.sh fails if rustc is not this version -let rustc = "1.99.0" in +let rustc = "1.99.1" in { name = "amaro-wasm", # amaro is node's built-in TypeScript type stripper (`node x.ts`): swc's @@ -31,7 +31,7 @@ let rustc = "1.99.0" in { # the rust package's own source url = "gs://minimal-staging-archives/rustc-%{rustc}-src.tar.xz", - sha256 = "cc41916a8c84f5d9ec4f55561b44e39f43b647b133f8a6f51be9ea13c83f7036", + sha256 = "6451598a99e77f07571960f6b21d577399d3a30f33e80cbd2e3a4aae6535d265", extract = false, } | Source, base, From e9033a42d4bc671a193c4e7505c3835fdbe196a6 Mon Sep 17 00:00:00 2001 From: bryan Date: Fri, 30 Oct 2026 12:14:33 -0700 Subject: [PATCH 16/16] next: census the final tree (x64 SWC addon embeds wasm); forward llhttp/amaro versions (review) - The wasm census ran before the @next/swc addon and sharp were installed, so neither reached it. The x64 addon holds one raw `\0asm` module (it failed the census when run over it, per review); the build passed on amd64 only because of step order. The census and smoke now run last, with an explicit allow-path naming the x64 addon as upstream's sha-pinned prebuilt. - llhttp 8.1.0 is forwarded from build.ncl (next: undici_llhttp; node and node-lts: npm_undici_llhttp) instead of a literal in three build.sh files. - amaro-wasm: `amaro` is a plain string used by the Source URL, upstream_version and the test; next's duplicate SRC_ROOT is gone. Co-Authored-By: Claude Opus 5.5 --- packages/amaro-wasm/build.ncl | 16 ++++++++------- packages/next/build.ncl | 3 +++ packages/next/build.sh | 38 +++++++++++++++++++---------------- packages/node-lts/build.ncl | 3 +++ packages/node-lts/build.sh | 2 +- packages/node/build.ncl | 3 +++ packages/node/build.sh | 2 +- 7 files changed, 41 insertions(+), 26 deletions(-) diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl index 825a816aa..02f2073ab 100644 --- a/packages/amaro-wasm/build.ncl +++ b/packages/amaro-wasm/build.ncl @@ -5,9 +5,11 @@ let rust = import "../rust/build.ncl" in let toolchain = import "../toolchain/build.ncl" in let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in -# The amaro versions node and node-lts bundle (both 1.1.11 since node 26.12.1 / 24.22.1). Each tag -# vendors the swc tree its TypeScript stripper is built from (deps/swc, with Cargo.lock). -let amaro = ["1.1.11"] in +# The amaro version node and node-lts bundle (both 1.1.11 since node 26.12.1 / 24.22.1), used in +# the Source URL, upstream_version and the test, so a bump is one edit here (plus node and +# node-lts's own `amaro`). Its tag vendors the swc tree the TypeScript stripper is built from +# (deps/swc, with Cargo.lock). build.sh still loops over amaro_versions, should two ever differ. +let amaro = "1.1.11" in # the rust package's version; build.sh fails if rustc is not this version let rustc = "1.99.1" in { @@ -24,7 +26,7 @@ let rustc = "1.99.1" in build_deps = [ { file = "build.sh" } | Local, { - url = "https://github.com/nodejs/amaro/archive/refs/tags/v1.1.11.tar.gz", + url = "https://github.com/nodejs/amaro/archive/refs/tags/v%{amaro}.tar.gz", sha256 = "f65b4e8ae9ab519e8db1824377eb621d7fce66e21ae1020621d8f54b6ed774fa", extract = false, } | Source, @@ -50,7 +52,7 @@ let rustc = "1.99.1" in cmd = "./build.sh", build_args = { - amaro_versions = std.string.join " " amaro, + amaro_versions = amaro, rust_version = rustc, }, @@ -60,7 +62,7 @@ let rustc = "1.99.1" in attrs = { - upstream_version = "1.1.11", + upstream_version = amaro, license_spdx = "MIT AND Apache-2.0", source_provenance = { category = 'GithubRepo, @@ -75,7 +77,7 @@ let rustc = "1.99.1" in class = 'Standalone, test_deps = [base], cmds = [ - ["/bin/bash", "-c", "for v in 1.1.11; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], + ["/bin/bash", "-c", "for v in %{amaro}; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], ], } | Test, }, diff --git a/packages/next/build.ncl b/packages/next/build.ncl index 90cdf9d08..4f4da952b 100644 --- a/packages/next/build.ncl +++ b/packages/next/build.ncl @@ -17,6 +17,8 @@ let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in # undici whose llhttp C rebuilds the wasm next bundles into its edge runtime. # Forwarded to build.sh as MINIMAL_ARG_UNDICI. let undici = "6.28.0" in +# The llhttp that undici release vendors (its build.sh pins the wasm to it). +let undici_llhttp = "8.1.0" in let version = "16.4.1" in # The native SWC compiler Next loads at build/dev time. Upstream publishes it as a @@ -86,6 +88,7 @@ let swc_arm64_sha256 = "ea6ca74538e44799986dbea7ae0b7142b37abf5d6b16fcf34566221c build_args = { include version, include undici, + include undici_llhttp, }, outputs = { diff --git a/packages/next/build.sh b/packages/next/build.sh index bf4aa8465..c96ba379a 100755 --- a/packages/next/build.sh +++ b/packages/next/build.sh @@ -18,8 +18,6 @@ export CXX=g++ export PNPM_CONFIG_STRICT_DEP_BUILDS=false export PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN=false -SRC_ROOT=$PWD - # Install all dependencies using the repo's pnpm-lock.yaml for reproducibility. # The lockfile pins exact versions of every transitive dependency used during the build. pnpm install --frozen-lockfile @@ -79,8 +77,8 @@ NEXT_DIR="$OUTPUT_DIR/usr/libexec/next/lib/node_modules/next" LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" mkdir -p /tmp/llhttp -tar -xof "$SRC_ROOT/undici-$UNDICI.tar.gz" -C /tmp/llhttp -$LH build "/tmp/llhttp/undici-$UNDICI" /tmp/llhttp/wasm --version "$UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +tar -xof "$BUILD_ROOT/undici-$UNDICI.tar.gz" -C /tmp/llhttp +$LH build "/tmp/llhttp/undici-$UNDICI" /tmp/llhttp/wasm --version "$UNDICI" --llhttp "${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" --sysroot /usr/lib/wasi LH_CARRIERS="dist/compiled/edge-runtime/index.js dist/compiled/@edge-runtime/primitives/fetch.js dist/compiled/@edge-runtime/primitives/load.js dist/compiled/@vercel/blob/index.cjs" LH_EXPECT= @@ -89,19 +87,6 @@ for c in $LH_CARRIERS; do $LH splice "$NEXT_DIR/$c" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d /tmp/llhttp/wasm/llhttp_simd.wasm LH_EXPECT="$LH_EXPECT --expect $c=2" done -# Every other wasm in the tree is still upstream's, by exact path: a new carrier fails here. -# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten; also -# base64 in index.node.js), -# webpack's and loader-utils' xxhash64/md4 (AssemblyScript), source-map08 (Rust). -$LH census -C "$NEXT_DIR" --wasm /tmp/llhttp/wasm/llhttp.wasm --wasm /tmp/llhttp/wasm/llhttp_simd.wasm $LH_EXPECT \ - --allow-path 'dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/llhttp.wasm' \ - --allow-path 'dist/compiled/@vercel/og/resvg.wasm' --allow-path 'dist/compiled/@vercel/og/yoga.wasm' \ - --allow-path 'dist/compiled/@vercel/og/index.node.js' \ - --allow-path 'dist/compiled/loader-utils2/index.js' --allow-path 'dist/compiled/loader-utils3/index.js' \ - --allow-path 'dist/compiled/webpack/bundle5.js' --allow-path 'dist/compiled/source-map08/mappings.wasm' \ - . -$LH smoke node /tmp/llhttp/wasm/llhttp.wasm /tmp/llhttp/wasm/llhttp_simd.wasm \ - --require "$NEXT_DIR/dist/compiled/edge-runtime" # Ship the platform SWC compiler inside next's own node_modules, where next resolves # `@next/swc-linux--gnu` first. The source build does not produce this native # addon; upstream publishes it as a per-platform npm package, fetched above as a @@ -213,3 +198,22 @@ if (!addon || !addon.startsWith(want)) { if (!s.versions.vips) throw new Error("sharp loaded without libvips"); console.log("sharp", s.versions.sharp, "vips", s.versions.vips, "addon", path.basename(addon)); ' "$NEXT_DIR/node_modules/sharp" + +# Every other wasm in the FINAL tree is still upstream's, by exact path: a new carrier fails here. +# Runs last, after the SWC addon and sharp are installed, so nothing lands after it unseen +# (review on #817: the x64 SWC addon embeds a raw wasm module and was installed after the census). +# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten; also +# base64 in index.node.js), +# webpack's and loader-utils' xxhash64/md4 (AssemblyScript), source-map08 (Rust), and the x64 +# SWC addon: upstream's sha-pinned prebuilt (swc_amd64_sha256), one raw `\0asm` module +# ~96 MB in (from-source build: gominimal/pkgs#985). The arm64 addon embeds none. +$LH census -C "$NEXT_DIR" --wasm /tmp/llhttp/wasm/llhttp.wasm --wasm /tmp/llhttp/wasm/llhttp_simd.wasm $LH_EXPECT \ + --allow-path 'dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/llhttp.wasm' \ + --allow-path 'dist/compiled/@vercel/og/resvg.wasm' --allow-path 'dist/compiled/@vercel/og/yoga.wasm' \ + --allow-path 'dist/compiled/@vercel/og/index.node.js' \ + --allow-path 'dist/compiled/loader-utils2/index.js' --allow-path 'dist/compiled/loader-utils3/index.js' \ + --allow-path 'dist/compiled/webpack/bundle5.js' --allow-path 'dist/compiled/source-map08/mappings.wasm' \ + --allow-path 'node_modules/@next/swc-linux-x64-gnu/next-swc.linux-x64-gnu.node' \ + . +$LH smoke node /tmp/llhttp/wasm/llhttp.wasm /tmp/llhttp/wasm/llhttp_simd.wasm \ + --require "$NEXT_DIR/dist/compiled/edge-runtime" diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index 7306ec326..949cef520 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -30,6 +30,8 @@ let amaro-wasm = import "../amaro-wasm/build.ncl" in let undici = "7.30.1" in let undici_llhttp = "9.3.0" in let npm_undici = "6.28.0" in +# The llhttp that the npm-vendored undici pins (forwarded so a bump is one edit). +let npm_undici_llhttp = "8.1.0" in let amaro = "1.1.11" in let version = "24.22.1" in { @@ -81,6 +83,7 @@ let version = "24.22.1" in include undici, include undici_llhttp, include npm_undici, + include npm_undici_llhttp, include amaro, }, diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index a54455b17..27c32776d 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -29,7 +29,7 @@ NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" $LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. -$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp "${MINIMAL_ARG_NPM_UNDICI_LLHTTP:?build.ncl forwards npm_undici_llhttp}" --sysroot /usr/lib/wasi $LH splice deps/undici/undici.js 57184a16144b5bc880e29105fa35d75c93dd318563e1b743e110de845e2ab661 ../wasm/llhttp.wasm $LH splice deps/undici/undici.js 2434215e1f7fa0163f22ce895ddbeb83984fa0aa51e123164e053213e5aca028 ../wasm/llhttp_simd.wasm $LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm diff --git a/packages/node/build.ncl b/packages/node/build.ncl index ce1fddbd4..43beeb842 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -31,6 +31,8 @@ let amaro-wasm = import "../amaro-wasm/build.ncl" in let undici = "8.12.2" in let undici_llhttp = "9.3.1" in let npm_undici = "6.28.0" in +# The llhttp that the npm-vendored undici pins (forwarded so a bump is one edit). +let npm_undici_llhttp = "8.1.0" in let amaro = "1.1.11" in let version = "26.12.1" in { @@ -85,6 +87,7 @@ let version = "26.12.1" in include undici, include undici_llhttp, include npm_undici, + include npm_undici_llhttp, include amaro, }, diff --git a/packages/node/build.sh b/packages/node/build.sh index c427993ae..03bf60b1a 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -29,7 +29,7 @@ NPM_LH=deps/npm/node_modules/undici/lib/llhttp $LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" $LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. -$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp 8.1.0 --sysroot /usr/lib/wasi +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp "${MINIMAL_ARG_NPM_UNDICI_LLHTTP:?build.ncl forwards npm_undici_llhttp}" --sysroot /usr/lib/wasi $LH splice deps/undici/undici.js ab4573a06c43574936dc98c3943b10a4ee32798e9797329e9f327e5f64115f54 ../wasm/llhttp.wasm $LH splice deps/undici/undici.js 18403ee56fab07f89a845028bad358bb3ebcda213f8e2ec0989a991e9c33da69 ../wasm/llhttp_simd.wasm $LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm