diff --git a/packages/amaro-wasm/build.ncl b/packages/amaro-wasm/build.ncl new file mode 100644 index 000000000..02f2073ab --- /dev/null +++ b/packages/amaro-wasm/build.ncl @@ -0,0 +1,84 @@ +let { Attrs, BuildSpec, Local, Needs, OutputData, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let rust = import "../rust/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in +let wasm-bindgen-cli = import "../wasm-bindgen-cli/build.ncl" in + +# The amaro version node and node-lts bundle (both 1.1.11 since node 26.12.1 / 24.22.1), used in +# the Source URL, upstream_version and the test, so a bump is one edit here (plus node and +# node-lts's own `amaro`). Its tag vendors the swc tree the TypeScript stripper is built from +# (deps/swc, with Cargo.lock). build.sh still loops over amaro_versions, should two ever differ. +let amaro = "1.1.11" in +# the rust package's version; build.sh fails if rustc is not this version +let rustc = "1.99.1" in +{ + name = "amaro-wasm", + # amaro is node's built-in TypeScript type stripper (`node x.ts`): swc's + # binding_typescript_wasm, compiled to wasm32-unknown-unknown and run through wasm-bindgen, + # base64-inlined into amaro's dist/index.js. This rebuilds that module from the vendored swc + # source, the way amaro's tools/build-wasm.js does in its container (cargo --release with fat LTO + # and opt-level z, wasm-bindgen --target nodejs, no wasm-opt), so node and node-lts can splice it + # over the shipped blob. + # + # The rust package ships no wasm32 std and no rust-lld: std is built with -Zbuild-std from the + # rustc source tarball of the same version, and llvm's wasm-ld links. + build_deps = [ + { file = "build.sh" } | Local, + { + url = "https://github.com/nodejs/amaro/archive/refs/tags/v%{amaro}.tar.gz", + sha256 = "f65b4e8ae9ab519e8db1824377eb621d7fce66e21ae1020621d8f54b6ed774fa", + extract = false, + } | Source, + { + # the rust package's own source + url = "gs://minimal-staging-archives/rustc-%{rustc}-src.tar.xz", + sha256 = "6451598a99e77f07571960f6b21d577399d3a30f33e80cbd2e3a4aae6535d265", + extract = false, + } | Source, + base, + llvm, + rust, + toolchain, + wasm-bindgen-cli, + ], + + # cargo fetches the crates in swc's Cargo.lock and std's library/Cargo.lock (--locked). + needs = + { + dns = {}, + internet = {}, + } | Needs, + + cmd = "./build.sh", + build_args = { + amaro_versions = amaro, + rust_version = rustc, + }, + + outputs = { + wasm = { glob = "usr/share/amaro-wasm/**" } | OutputData, + }, + + attrs = + { + upstream_version = amaro, + license_spdx = "MIT AND Apache-2.0", + source_provenance = { + category = 'GithubRepo, + owner = "nodejs", + repo = "amaro", + }, + } | Attrs, + + tests = { + modules = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "for v in %{amaro}; do head -c4 /usr/share/amaro-wasm/$v/wasm_bg.wasm | od -An -c | grep -q 'a s m' || exit 1; done"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/amaro-wasm/build.sh b/packages/amaro-wasm/build.sh new file mode 100755 index 000000000..30490358a --- /dev/null +++ b/packages/amaro-wasm/build.sh @@ -0,0 +1,55 @@ +#!/bin/sh +set -ex + +RV="${MINIMAL_ARG_RUST_VERSION:?build.ncl forwards rust_version}" +VERSIONS="${MINIMAL_ARG_AMARO_VERSIONS:?build.ncl forwards amaro_versions}" +TOP=$(pwd) + +# std for wasm32 comes from the rustc source of the exact rustc in use. +case "$(rustc --version)" in "rustc $RV "*) ;; *) echo "rustc is not $RV: update rust_version and its sha" >&2; exit 1 ;; esac +tar -xof "rustc-$RV-src.tar.xz" "rustc-$RV-src/library" +LIB="$TOP/rustc-$RV-src/library" +export __CARGO_TESTS_ONLY_SRC_ROOT="$LIB" +export RUSTC_BOOTSTRAP=1 + +# Host artifacts (build scripts, proc macros) link with gcc; swc's .cargo/config.toml names +# cross linkers for the linux targets. +export CC=gcc +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_LINKER=wasm-ld +# amaro's swc scripts/build.sh +export CARGO_PROFILE_RELEASE_LTO=fat +export CARGO_PROFILE_RELEASE_OPT_LEVEL=z + +# build : the module amaro's tools/build-wasm.js produces, as +# /pkg/wasm_bg.wasm (before patch.mjs inlines it). +build() { + v=$1; w=$2 + rm -rf "$w"; mkdir -p "$w" + tar -xof "$TOP/v$v.tar.gz" -C "$w" "amaro-$v/deps/swc" + # From the binding's directory, as wasm-pack runs it, so its .cargo/config.toml (the 2 MiB + # stack) applies along with the workspace's; the stack size is repeated in case the env value + # replaces rather than extends it. + cd "$w/amaro-$v/deps/swc/bindings/binding_typescript_wasm" + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUSTFLAGS="-Clink-arg=-zstack-size=2097152 --remap-path-prefix=$w=/builddir --remap-path-prefix=$LIB=/rustc/library --remap-path-prefix=$HOME/.cargo=/cargo" \ + cargo build --release --locked -Zbuild-std=std,panic_abort --target wasm32-unknown-unknown + cd "$w/amaro-$v/deps/swc" + wasm-bindgen --target nodejs --out-name wasm --out-dir pkg --typescript \ + target/wasm32-unknown-unknown/release/binding_typescript_wasm.wasm + cd "$TOP" +} + +for v in $VERSIONS; do + # Built twice, at different paths: the module must not depend on where it was built. + build "$v" "$TOP/a-$v" + build "$v" "$TOP/b-$v" + A="$TOP/a-$v/amaro-$v/deps/swc/pkg/wasm_bg.wasm" + B="$TOP/b-$v/amaro-$v/deps/swc/pkg/wasm_bg.wasm" + cmp "$A" "$B" || { echo "amaro $v: two builds of wasm_bg.wasm differ" >&2; exit 1; } + D="$OUTPUT_DIR/usr/share/amaro-wasm/$v" + mkdir -p "$D" + cp "$A" "$D/wasm_bg.wasm" + cp "$TOP/a-$v/amaro-$v/deps/swc/pkg/wasm.js" "$D/wasm.js" + sha256sum "$D/wasm_bg.wasm" +done diff --git a/packages/llhttp-wasm-rebuild/build.ncl b/packages/llhttp-wasm-rebuild/build.ncl new file mode 100644 index 000000000..1526565c3 --- /dev/null +++ b/packages/llhttp-wasm-rebuild/build.ncl @@ -0,0 +1,29 @@ +let { BuildSpec, Local, OutputData, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let python = import "../python/build.ncl" in + +# Build-time helper for node and node-lts: rebuilds undici's llhttp wasm from its +# in-tree C and splices it over the shipped base64 blobs. See the script's docstring. +{ + name = "llhttp-wasm-rebuild", + build_deps = [ + { file = "build.sh" } | Local, + { file = "rebuild-llhttp-wasm.py" } | Local, + base, + python, # build.sh runs the selftest + ], + # node, node-lts and next run the installed script with python3 + runtime_deps = [python], + cmd = "./build.sh", + outputs = { + script = { glob = "usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" } | OutputData, + }, + tests = { + selftest = + { + class = 'Standalone, + test_deps = [python], + cmds = [["python3", "/usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py", "selftest"]], + } | Test, + }, +} | BuildSpec diff --git a/packages/llhttp-wasm-rebuild/build.sh b/packages/llhttp-wasm-rebuild/build.sh new file mode 100755 index 000000000..adc2e05db --- /dev/null +++ b/packages/llhttp-wasm-rebuild/build.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -ex + +python3 rebuild-llhttp-wasm.py selftest +install -Dm 0644 rebuild-llhttp-wasm.py "$OUTPUT_DIR/usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" diff --git a/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py new file mode 100755 index 000000000..8ef91be87 --- /dev/null +++ b/packages/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py @@ -0,0 +1,990 @@ +#!/usr/bin/env python3 +"""Regenerate undici's llhttp wasm from in-tree C and replace the shipped blobs. + + check-tree node tree tripwires: pinned undici / npm-undici versions, no cjs lexer wasm + build pin build/{wasm,llhttp}.js flags, compile llhttp{,_simd}.wasm twice, require identical bytes + splice decode the pinned shipped blob in a carrier, gate tables+features, replace it once + census every wasm carrier under the roots is an allowed sha or on an allowed path + smoke instantiate blobs under a built node and parse a POST; optional loopback fetch + selftest synthetic tree, proves every gate both passes and fails + +stdlib only. Exit 1 on any mismatch. +""" +import argparse +import base64 +import binascii +import collections +import fnmatch +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +MAGIC = b"\0asm\x01\0\0\0" + +# deps/undici/src/build/wasm.js, identical in undici v6.26.0, v6.28.0, v7.29.0, v7.30.1, v8.10.2; +# renamed build/llhttp.js in v8.12 (beside the new build/milo.js) with the same lines (v8.12.2). +WASM_JS_NAMES = ("build/wasm.js", "build/llhttp.js") +WASM_JS_PINNED = ( + "let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot -target wasm32-unknown-wasi'", + "WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor'", + "WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all'", + "WASM_LDFLAGS += ' -Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table'", + "WASM_LDFLAGS += ' -Wl,--export=malloc -Wl,--export=free -Wl,--no-entry'", + "const WASM_OPT_FLAGS = '-O4 --converge --strip-debug --strip-dwarf --strip-producers'", +) +CFLAGS = "-Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor".split() +LDFLAGS = ("-Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all " + "-Wl,--allow-undefined -Wl,--export-dynamic -Wl,--export-table " + "-Wl,--export=malloc -Wl,--export=free -Wl,--no-entry").split() +# clang 17's "generic" cpu; clang 21's generic adds bulk-memory, multivalue, reference-types, +# nontrapping-fptoint and call-indirect-overlong, none of which the shipped blobs use. +CPU = ["-mcpu=mvp", "-mmutable-globals"] +LLHTTP_SRCS = ("api.c", "http.c", "llhttp.c") +GATE_ALLOWED = {"mutable-globals"} + +B64_ALIGNED = re.compile(rb"AGFzbQ[A-Za-z0-9+/]*={0,2}") + + +class Fail(Exception): + pass + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +def log(msg): + print(msg, flush=True) + + +# ---------------------------------------------------------------- wasm reader + +def uleb(b, i, maxbytes=5): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated leb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + return r, i + if n >= maxbytes: + raise Fail("leb128 too long") + + +def uleb_len(b, i): + v, j = uleb(b, i) + return v, j, j - i + + +def sleb(b, i, maxbytes=10): + r = s = n = 0 + while True: + if i >= len(b): + raise Fail("truncated sleb128") + x = b[i] + i += 1 + n += 1 + r |= (x & 0x7F) << s + s += 7 + if x < 0x80: + if x & 0x40: + r -= 1 << s + return r, i + if n >= maxbytes: + raise Fail("sleb128 too long") + + +def name(b, i): + n, i = uleb(b, i) + return b[i:i + n].decode("utf-8", "replace"), i + n + + +VALTYPES = {0x7F: "i32", 0x7E: "i64", 0x7D: "f32", 0x7C: "f64", 0x7B: "v128", 0x70: "funcref", 0x6F: "externref"} + + +def valtype(b, i, feats): + t = b[i] + if t not in VALTYPES: + raise Fail("unknown valtype 0x%02x" % t) + if t == 0x7B: + feats["simd128"] += 1 + elif t in (0x70, 0x6F): + feats["reference-types"] += 1 + return VALTYPES[t], i + 1 + + +def limits(b, i, feats): + flag = b[i] + i += 1 + if flag & ~1: + feats["limits-flag-%d" % flag] += 1 + _, i = uleb(b, i, 10) + if flag & 1: + _, i = uleb(b, i, 10) + return i + + +def walk_ops(b, i, end, feats, const_expr=False): + """Walk instructions in b[i:end]; count non-MVP features; unknown opcodes fail closed.""" + while i < end: + op = b[i] + i += 1 + if op == 0x0B and const_expr: + return i + if op in (0x00, 0x01, 0x05, 0x0B, 0x0F, 0x1A, 0x1B, 0xD1) or 0x45 <= op <= 0xBF: + if op == 0xD1: + feats["reference-types"] += 1 + elif op in (0x02, 0x03, 0x04): + bt = b[i] + if bt == 0x40 or bt in VALTYPES: + if bt == 0x7B: + feats["simd128"] += 1 + i += 1 + else: + _, i = sleb(b, i, 5) + feats["multivalue"] += 1 + elif op in (0x0C, 0x0D, 0x10, 0x20, 0x21, 0x22, 0x23, 0x24): + _, i = uleb(b, i) + elif op in (0x25, 0x26, 0xD2): + _, i = uleb(b, i) + feats["reference-types"] += 1 + elif op == 0x0E: + n, i = uleb(b, i) + for _ in range(n + 1): + _, i = uleb(b, i) + elif op == 0x11: + _, i = uleb(b, i) + tbl, i, ln = uleb_len(b, i) + if ln > 1: + feats["call-indirect-overlong"] += 1 + if tbl != 0: + feats["reference-types"] += 1 + elif op == 0x1C: + n, i = uleb(b, i) + for _ in range(n): + _, i = valtype(b, i, feats) + feats["reference-types"] += 1 + elif 0x28 <= op <= 0x3E: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif op in (0x3F, 0x40): + if b[i] != 0: + feats["multi-memory"] += 1 + _, i = uleb(b, i) + elif op == 0x41: + _, i = sleb(b, i, 5) + elif op == 0x42: + _, i = sleb(b, i, 10) + elif op == 0x43: + i += 4 + elif op == 0x44: + i += 8 + elif 0xC0 <= op <= 0xC4: + feats["sign-ext"] += 1 + elif op == 0xD0: + i += 1 + feats["reference-types"] += 1 + elif op == 0xFC: + sub, i = uleb(b, i) + if sub <= 7: + feats["nontrapping-fptoint"] += 1 + elif sub <= 14: + feats["bulk-memory"] += 1 + nimm = {8: 2, 9: 1, 10: 2, 11: 1, 12: 2, 13: 1, 14: 2}[sub] + for _ in range(nimm): + _, i = uleb(b, i) + elif sub <= 17: + feats["reference-types"] += 1 + _, i = uleb(b, i) + else: + raise Fail("unknown opcode 0xfc %d" % sub) + elif op == 0xFD: + sub, i = uleb(b, i) + feats["simd128"] += 1 + if sub <= 11 or sub in (92, 93): + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + elif sub in (12, 13): + i += 16 + elif 21 <= sub <= 34: + i += 1 + elif 84 <= sub <= 91: + _, i = uleb(b, i) + _, i = uleb(b, i, 10) + i += 1 + elif sub >= 0x100: + raise Fail("relaxed-simd opcode 0xfd %d" % sub) + else: + raise Fail("unknown opcode 0x%02x" % op) + if const_expr: + raise Fail("const expr without end") + return i + + +def wasm_info(b): + if b[:8] != MAGIC: + raise Fail("not a wasm v1 module") + feats = collections.Counter() + types, funcs, imports, exports, ids = [], [], set(), set(), [] + import_funcs = [] + i = 8 + while i < len(b): + sid = b[i] + sz, i = uleb(b, i + 1) + end = i + sz + if end > len(b): + raise Fail("section %d overruns module" % sid) + s, j = b, i + ids.append(sid) + if sid == 0: + nm, _ = name(s, j) + feats["custom-section:" + nm] += 1 + elif sid == 1: + n, j = uleb(s, j) + for _ in range(n): + if s[j] != 0x60: + raise Fail("unknown type form 0x%02x" % s[j]) + j += 1 + sig = [] + for _part in range(2): + k, j = uleb(s, j) + vs = [] + for _ in range(k): + v, j = valtype(s, j, feats) + vs.append(v) + sig.append(vs) + if len(sig[1]) > 1: + feats["multivalue"] += 1 + types.append("(%s)->(%s)" % (",".join(sig[0]), ",".join(sig[1]))) + elif sid == 2: + n, j = uleb(s, j) + for _ in range(n): + mod, j = name(s, j) + fld, j = name(s, j) + kind = s[j] + j += 1 + if kind == 0: + t, j = uleb(s, j) + import_funcs.append(t) + imports.add((mod, fld, "func", types[t])) + elif kind == 1: + _, j = valtype(s, j, feats) + j = limits(s, j, feats) + imports.add((mod, fld, "table", "")) + elif kind == 2: + j = limits(s, j, feats) + imports.add((mod, fld, "memory", "")) + elif kind == 3: + v, j = valtype(s, j, feats) + if s[j]: + feats["mutable-globals"] += 1 + j += 1 + imports.add((mod, fld, "global", v)) + else: + raise Fail("unknown import kind %d" % kind) + elif sid == 3: + n, j = uleb(s, j) + for _ in range(n): + t, j = uleb(s, j) + funcs.append(t) + elif sid == 6: + n, j = uleb(s, j) + for _ in range(n): + _, j = valtype(s, j, feats) + j += 1 + j = walk_ops(s, j, end, feats, const_expr=True) + elif sid == 7: + n, j = uleb(s, j) + for _ in range(n): + nm, j = name(s, j) + kind = s[j] + idx, j = uleb(s, j + 1) + kname = {0: "func", 1: "table", 2: "memory", 3: "global"}.get(kind) + if kname is None: + raise Fail("unknown export kind %d" % kind) + desc = "" + if kind == 0: + ni = len(import_funcs) + desc = types[import_funcs[idx] if idx < ni else funcs[idx - ni]] + exports.add((nm, kname, desc)) + elif sid == 9: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["elem-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + for _ in range(k): + _, j = uleb(s, j) + elif sid == 10: + n, j = uleb(s, j) + for _ in range(n): + bsz, j = uleb(s, j) + bend = j + bsz + ng, j = uleb(s, j) + for _ in range(ng): + _, j = uleb(s, j) + _, j = valtype(s, j, feats) + j = walk_ops(s, j, bend, feats) + if j != bend or s[bend - 1] != 0x0B: + raise Fail("function body did not parse to its end") + elif sid == 11: + n, j = uleb(s, j) + for _ in range(n): + flag, j = uleb(s, j) + if flag != 0: + feats["data-flags-%d" % flag] += 1 + break + j = walk_ops(s, j, end, feats, const_expr=True) + k, j = uleb(s, j) + j += k + elif sid == 12: + feats["bulk-memory"] += 1 + elif sid in (4, 5, 8): + pass + else: + feats["section-%d" % sid] += 1 + i = end + return {"imports": imports, "exports": exports, "feats": feats, "ids": set(ids)} + + +WBG_CLOSURE = re.compile(r"__wbindgen_closure_wrapper[0-9]+") + + +def gate(shipped, rebuilt, label, wbg=False): + """Rebuilt blob must present the shipped blob's tables and no extra wasm features. + + wbg (wasm-bindgen modules): wasm-bindgen numbers its closure-wrapper shim after a function + index, so one import may differ, if both sides are such a shim with the same module, kind and + signature; it is returned as (shipped name, rebuilt name) for the carrier's glue to follow. + Features may be any the shipped blob already uses.""" + a, r = wasm_info(shipped), wasm_info(rebuilt) + errs, renames = [], [] + for k in ("imports", "exports"): + miss, extra = a[k] - r[k], r[k] - a[k] + if wbg and k == "imports": + m = [x for x in miss if WBG_CLOSURE.fullmatch(x[1])] + e = [x for x in extra if WBG_CLOSURE.fullmatch(x[1])] + if len(m) == 1 and len(e) == 1 and (m[0][0], m[0][2], m[0][3]) == (e[0][0], e[0][2], e[0][3]): + renames.append((m[0][1], e[0][1])) + miss, extra = miss - set(m), extra - set(e) + if miss or extra: + errs.append("%s differ: missing %s, extra %s" % (k, sorted(miss), sorted(extra))) + simd = "simd128" in a["feats"] + if simd != ("simd128" in r["feats"]): + errs.append("simd128 use differs: shipped %s, rebuilt %s" % (simd, not simd)) + allowed = GATE_ALLOWED | ({"simd128"} if simd else set()) | (set(a["feats"]) if wbg else set()) + extra = sorted(set(r["feats"]) - allowed) + if extra: + errs.append("features not allowed: %s" % ", ".join("%s x%d" % (f, r["feats"][f]) for f in extra)) + if r["ids"] - a["ids"]: + errs.append("sections not in shipped blob: %s" % sorted(r["ids"] - a["ids"])) + if errs: + raise Fail("%s: gate failed\n " % label + "\n ".join(errs)) + log("@@GATE ok %s imports=%d exports=%d feats=%s%s" % ( + label, len(r["imports"]), len(r["exports"]), dict(r["feats"]) or "{}", + "".join(" rename=%s->%s" % rn for rn in renames))) + return renames + + +# ---------------------------------------------------------------- carriers + +def b64_decode(run): + try: + return base64.b64decode(run, validate=True) + except (binascii.Error, ValueError): + return None + + +def _misaligned_patterns(): + out = [] + for k in (1, 2): + enc = base64.b64encode(b"\0" * k + MAGIC) + lo, hi = 8 * k, 8 * k + 8 * len(MAGIC) + first, last = -(-lo // 6), hi // 6 - 1 + out.append(enc[first:last + 1]) + return out + + +MISALIGNED = _misaligned_patterns() +OTHER_FORMS = ( + ("js-bytes", re.compile(rb"(? (p, at, len) => { seen[k] = (seen[k] || '') + Buffer.from(mem.buffer, at, len).toString(); return 0; }; + let mem; + const env = { + wasm_on_message_begin: () => 0, wasm_on_status: span('status'), + wasm_on_url: span('url'), wasm_on_header_field: span('hf'), wasm_on_header_value: span('hv'), + wasm_on_headers_complete: () => 0, wasm_on_body: span('body'), + wasm_on_message_complete: () => { seen.done = true; return 0; }, + }; + const { exports: e } = new WebAssembly.Instance(new WebAssembly.Module(fs.readFileSync(f)), { env }); + mem = e.memory; + e._initialize(); + const p = e.llhttp_alloc(1); + const buf = e.malloc(req.length); + new Uint8Array(mem.buffer, buf, req.length).set(req); + const rc = e.llhttp_execute(p, buf, req.length); + if (rc !== 0 || seen.url !== '/x?y=1' || seen.body !== 'hello' || !seen.done) { + console.error('@@SMOKE FAIL', f, rc, JSON.stringify(seen)); process.exit(1); + } + e.free(buf); e.llhttp_free(p); + console.log('@@SMOKE ok llhttp_execute', f); +} +""" + +FETCH_JS = r""" +const http = require('node:http'); +const s = http.createServer((q, r) => { let b = ''; q.on('data', (d) => b += d); q.on('end', () => r.end('echo:' + b)); }); +s.on('error', (e) => { + if (['EPERM', 'EACCES', 'EADDRNOTAVAIL'].includes(e.code)) { console.log('@@SMOKE fetch SKIP no loopback', e.code); process.exit(0); } + console.error('@@SMOKE FAIL listen', e); process.exit(1); +}); +s.listen(0, '127.0.0.1', async () => { + try { + const r = await fetch(`http://127.0.0.1:${s.address().port}/x`, { method: 'POST', body: 'hello' }); + const t = await r.text(); + if (t !== 'echo:hello') throw new Error('body ' + t); + console.log('@@SMOKE ok fetch'); s.close(); + } catch (e) { console.error('@@SMOKE FAIL fetch', e); process.exit(1); } +}); +""" + + +def smoke(node, wasms, fetch, requires): + subprocess.run([node, "-e", SMOKE_JS] + list(wasms), check=True) + for r in requires: + subprocess.run([node, "-e", "require(%s); console.log('@@SMOKE ok require', %s)" % ( + json.dumps(os.path.abspath(r)), json.dumps(r))], check=True) + if fetch: + subprocess.run([node, "-e", FETCH_JS], check=True) + + +# ---------------------------------------------------------------- selftest + +def _sec(sid, body): + return bytes([sid]) + _leb(len(body)) + body + + +def _leb(n): + out = bytearray() + while True: + x = n & 0x7F + n >>= 7 + if n: + out.append(x | 0x80) + else: + out.append(x) + return bytes(out) + + +def _str(s): + return _leb(len(s)) + s.encode() + + +def _vec(items): + return _leb(len(items)) + b"".join(items) + + +def mkwasm(imports=("wasm_on_url",), exports=("malloc", "llhttp_execute"), body=b"\x41\x00\x0b", simd=False, module="env"): + """Tiny module: imports . (i32,i32,i32)->i32, exports memory + funcs sharing one body.""" + t3 = b"\x60\x03\x7f\x7f\x7f\x01\x7f" + types = _sec(1, _vec([t3])) + imp = _sec(2, _vec([_str(module) + _str(n) + b"\x00\x00" for n in imports])) + nf = len(exports) + fn = _sec(3, _vec([b"\x00"] * nf)) + mem = _sec(5, _vec([b"\x00\x01"])) + exp = _sec(7, _vec([_str("memory") + b"\x02\x00"] + + [_str(n) + b"\x00" + _leb(len(imports) + k) for k, n in enumerate(exports)])) + if simd: + body = b"\xfd\x0c" + b"\x00" * 16 + b"\x1a" + body + code1 = _leb(0) + body + code = _sec(10, _vec([_leb(len(code1)) + code1] * nf)) + return MAGIC + types + imp + fn + mem + exp + code + + +def _expect_fail(fn, *a, contains=""): + try: + fn(*a) + except Fail as e: + if contains and contains not in str(e): + raise AssertionError("failed for the wrong reason: %s" % e) + log(" (expected failure: %s)" % str(e).splitlines()[0]) + return + raise AssertionError("%s%r should have failed" % (fn.__name__, a)) + + +def selftest(): + shipped = mkwasm() + shipped_simd = mkwasm(simd=True) + good = mkwasm(body=b"\x41\x01\x0b") + good_simd = mkwasm(body=b"\x41\x01\x0b", simd=True) + bad_import = mkwasm(imports=("wasm_on_url", "__multi3"), body=b"\x41\x02\x0b") + bad_export = mkwasm(exports=("malloc",), body=b"\x41\x03\x0b") + bad_bulk = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\xfc\x0b\x00\x41\x00\x0b") + bad_overlong = mkwasm(body=b"\x41\x00\x41\x00\x41\x00\x41\x00\x11\x00\x80\x80\x80\x80\x00\x0b") + bad_unknown = mkwasm(body=b"\x06\x40\x0b\x41\x00\x0b") + bad_signext = mkwasm(body=b"\x41\x00\xc0\x0b") + pins = {k: sha(v) for k, v in (("plain", shipped), ("simd", shipped_simd))} + + log("selftest: gate") + gate(shipped, good, "good") + _expect_fail(gate, shipped, bad_signext, "sign-ext", contains="sign-ext") + gate(shipped_simd, good_simd, "good simd") + _expect_fail(gate, shipped, bad_import, "bad import", contains="imports differ") + _expect_fail(gate, shipped, bad_export, "bad export", contains="exports differ") + _expect_fail(gate, shipped, bad_bulk, "bulk-memory", contains="bulk-memory") + _expect_fail(gate, shipped, bad_overlong, "overlong", contains="call-indirect-overlong") + _expect_fail(gate, shipped, good_simd, "simd in plain", contains="simd128") + _expect_fail(gate, shipped_simd, good, "no simd in simd", contains="simd128") + _expect_fail(wasm_info, bad_unknown, contains="unknown opcode") + _expect_fail(wasm_info, b"\0asm\x02\0\0\0", contains="not a wasm v1") + + log("selftest: wasm-bindgen gate") + P = "__wbindgen_placeholder__" + wb_shipped = mkwasm(imports=("__wbg_log_1", "__wbindgen_closure_wrapper10"), module=P) + wb_good = mkwasm(imports=("__wbg_log_1", "__wbindgen_closure_wrapper12"), module=P, body=b"\x41\x01\x0b") + wb_two = mkwasm(imports=("__wbg_log_2", "__wbindgen_closure_wrapper12"), module=P, body=b"\x41\x01\x0b") + _expect_fail(gate, wb_shipped, wb_good, "wbg strict", contains="imports differ") + assert gate(wb_shipped, wb_good, "wbg", True) == [("__wbindgen_closure_wrapper10", "__wbindgen_closure_wrapper12")] + _expect_fail(gate, wb_shipped, wb_two, "wbg two", True, contains="imports differ") + gate(bad_bulk, bad_bulk, "wbg shipped feats", True) + _expect_fail(gate, shipped, bad_bulk, "wbg new feats", True, contains="bulk-memory") + + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as td: + def tree(): + root = os.path.join(td, "t%d" % len(os.listdir(td))) + u = os.path.join(root, "deps/undici") + n = os.path.join(root, "deps/npm/node_modules/undici/lib/llhttp") + os.makedirs(n) + os.makedirs(u, exist_ok=True) + os.makedirs(os.path.join(root, "deps/v8/third_party/wasm-api/example")) + with open(os.path.join(u, "undici.js"), "w") as f: + f.write('var wasmBase64 = "%s";\n// next\nvar wasmBase64 = "%s";\n' % ( + base64.b64encode(shipped).decode(), base64.b64encode(shipped_simd).decode())) + with open(os.path.join(n, "llhttp-wasm.js"), "w") as f: + f.write("module.exports = Buffer.from('%s', 'base64')\n" % base64.b64encode(shipped).decode()) + with open(os.path.join(root, "deps/v8/third_party/wasm-api/example/hello.wasm"), "wb") as f: + f.write(mkwasm(exports=("run",))) + for name, b in (("good.wasm", good), ("good_simd.wasm", good_simd), ("bad_bulk.wasm", bad_bulk)): + with open(os.path.join(root, name), "wb") as f: + f.write(b) + return root + + def full_pass(root): + os.chdir(root) + splice("deps/undici/undici.js", pins["plain"], "good.wasm") + splice("deps/undici/undici.js", pins["simd"], "good_simd.wasm") + splice("deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js", pins["plain"], "good.wasm") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}) + + try: + log("selftest: splice + census pass") + full_pass(tree()) + + log("selftest: N+ expectations") + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (1, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (3, True), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, True)}, + contains="expected 3+") + + log("selftest: wasm-bindgen splice renames the glue") + os.chdir(td) + glue = ("imports.wbg.__wbindgen_closure_wrapper10 = function() {};\n" + "var bytes = Buffer.from(\"%s\", \"base64\");\n" % base64.b64encode(wb_shipped).decode()) + for name, b in (("wb.js", glue.encode()), ("wb_good.wasm", wb_good)): + with open(name, "wb") as f: + f.write(b) + _expect_fail(splice, "wb.js", sha(wb_shipped), "wb_good.wasm", contains="imports differ") + splice("wb.js", sha(wb_shipped), "wb_good.wasm", True) + got = open("wb.js").read() + assert "__wbindgen_closure_wrapper12 = " in got and "wrapper10" not in got, got[:120] + with open("wb.js", "w") as f: + f.write(glue + "// __wbindgen_closure_wrapper10 again\n") + _expect_fail(splice, "wb.js", sha(wb_shipped), "wb_good.wasm", True, contains="exactly once") + + log("selftest: splice rejects") + root = tree() + os.chdir(root) + _expect_fail(splice, "deps/undici/undici.js", "0" * 64, "good.wasm", contains="found 0 times") + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "bad_bulk.wasm", contains="bulk-memory") + before = open("deps/undici/undici.js", "rb").read() + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good_simd.wasm", contains="simd128") + assert open("deps/undici/undici.js", "rb").read() == before, "failed splice modified the carrier" + with open("deps/undici/undici.js", "a") as f: + f.write('var again = "%s";\n' % base64.b64encode(shipped).decode()) + _expect_fail(splice, "deps/undici/undici.js", pins["plain"], "good.wasm", contains="found 2 times") + + log("selftest: census rejects") + for label, mutate, why in ( + ("unspliced shipped blob", lambda: None, "base64 deps/undici/undici.js"), + ("rogue base64", lambda: open("deps/rogue.js", "w").write( + "x='%s'" % base64.b64encode(mkwasm(exports=("evil",))).decode()), "deps/rogue.js"), + ("raw wasm", lambda: open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))), "raw deps/x.wasm"), + ("embedded raw", lambda: open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"..."), "raw deps/x.bin"), + ("misaligned base64", lambda: open("deps/m.js", "w").write( + "y='%s'" % base64.b64encode(b"Q" + mkwasm()).decode()), "base64-misaligned deps/m.js"), + ("js byte array", lambda: open("deps/a.js", "w").write("[0, 97, 115, 109, 1, 0, 0, 0]"), "js-bytes"), + ("truncated base64", lambda: open("deps/t.js", "w").write("z='AGFzbQEAAAABJw'"), "undecodable"), + ("expect count", lambda: open("deps/undici/undici.js", "a").write( + "var dup = '%s';\n" % base64.b64encode(good).decode()), "expected 2 allowed blob(s)"), + ("unexpected carrier", lambda: open("deps/copy.js", "w").write( + "c='%s'" % base64.b64encode(good).decode()), "unexpected file deps/copy.js"), + ): + log(" case: " + label) + root = tree() + os.chdir(root) + if label != "unspliced shipped blob": + full_pass(root) + mutate() + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], + ["deps/v8/third_party/wasm-api/example/*"], + {"deps/undici/undici.js": (2, False), + "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)}, + contains=why) + + log("selftest: census allow-raw is an exact count") + root = tree() + os.chdir(root) + full_pass(root) + open("deps/x.bin", "wb").write(b"ELF..." + MAGIC + b"........" + MAGIC + b"...") + exp = {"deps/undici/undici.js": (2, False), "deps/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js": (1, False)} + census(["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, {"deps/x.bin": 2}) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.bin": 1}, contains="expected 1 raw header(s)") + open("deps/x.wasm", "wb").write(mkwasm(exports=("evil",))) + _expect_fail(census, ["deps"], [sha(good), sha(good_simd)], ["deps/v8/third_party/wasm-api/example/*"], exp, + {"deps/x.wasm": 1}, contains="raw deps/x.wasm") + + log("selftest: source pins") + src = os.path.join(td, "undici-src") + os.makedirs(os.path.join(src, "deps/llhttp/src")) + os.makedirs(os.path.join(src, "deps/llhttp/include")) + os.makedirs(os.path.join(src, "build")) + json.dump({"version": "8.10.2"}, open(os.path.join(src, "package.json"), "w")) + open(os.path.join(src, "deps/llhttp/include/llhttp.h"), "w").write( + "#define LLHTTP_VERSION_MAJOR 9\n#define LLHTTP_VERSION_MINOR 3\n#define LLHTTP_VERSION_PATCH 1\n") + for s in LLHTTP_SRCS: + open(os.path.join(src, "deps/llhttp/src", s), "w").write("") + open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + build(src, os.path.join(td, "out"), "8.10.2", "9.3.1", "clang", "/usr/share/wasi-sysroot", None, True) + _expect_fail(check_undici_src, src, "8.10.3", "9.3.1", contains="recipe pins 8.10.3") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.0", contains="recipe pins 9.3.0") + open(os.path.join(src, "deps/llhttp/src/extra.c"), "w").write("") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="src/*.c") + os.unlink(os.path.join(src, "deps/llhttp/src/extra.c")) + open(os.path.join(src, "build/wasm.js"), "w").write( + "\n".join(WASM_JS_PINNED).replace("-Ofast", "-O3") + "\n") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="flags drifted") + # undici 8.12 renamed build/wasm.js -> build/llhttp.js: either name, never both or neither + os.rename(os.path.join(src, "build/wasm.js"), os.path.join(src, "build/llhttp.js")) + open(os.path.join(src, "build/llhttp.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + check_undici_src(src, "8.10.2", "9.3.1") + open(os.path.join(src, "build/wasm.js"), "w").write("\n".join(WASM_JS_PINNED) + "\n") + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="want exactly one") + os.unlink(os.path.join(src, "build/wasm.js")) + os.unlink(os.path.join(src, "build/llhttp.js")) + _expect_fail(check_undici_src, src, "8.10.2", "9.3.1", contains="want exactly one") + finally: + os.chdir(cwd) + + cmd = compile_cmd("clang", "/s", None, "o.wasm", True) + for f in CFLAGS + LDFLAGS + CPU + ["-msimd128", "-mexec-model=reactor", "-lc", "--no-wasm-opt"]: + assert f in cmd, f + log("@@SELFTEST PASS") + + +# ---------------------------------------------------------------- cli + +def main(argv): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sp = ap.add_subparsers(dest="cmd", required=True) + p = sp.add_parser("check-tree") + p.add_argument("--undici", required=True) + p.add_argument("--npm-undici", required=True) + p = sp.add_parser("build") + p.add_argument("src") + p.add_argument("outdir") + p.add_argument("--version", required=True) + p.add_argument("--llhttp", required=True) + p.add_argument("--cc", default="clang") + p.add_argument("--sysroot", default="/usr/lib/wasi", help="packages/wasi-libc install root") + p.add_argument("--rt", help="explicit wasm32 builtins archive, if llhttp needs one") + p.add_argument("--dry-run", action="store_true") + p = sp.add_parser("splice") + p.add_argument("carrier") + p.add_argument("shipped_sha") + p.add_argument("rebuilt") + p.add_argument("--wasm-bindgen", action="store_true", + help="wasm-bindgen module: allow its renumbered closure-wrapper import (renamed in the glue) and the shipped blob's features") + p = sp.add_parser("census") + p.add_argument("roots", nargs="+") + p.add_argument("-C", dest="chdir") + p.add_argument("--wasm", action="append", default=[], help="rebuilt blob whose sha is allowed") + p.add_argument("--allow-path", action="append", default=[], help="fnmatch glob, relative to -C") + p.add_argument("--expect", action="append", default=[], help="PATH=N (exact) or PATH=N+ allowed blobs in PATH") + p.add_argument("--allow-raw", action="append", default=[], help="PATH=N: exactly N undecodable raw wasm headers in PATH") + p = sp.add_parser("smoke") + p.add_argument("node") + p.add_argument("wasm", nargs="+") + p.add_argument("--fetch", action="store_true") + p.add_argument("--require", action="append", default=[]) + sp.add_parser("selftest") + a = ap.parse_args(argv) + try: + if a.cmd == "check-tree": + check_tree(a.undici, a.npm_undici) + elif a.cmd == "build": + build(a.src, a.outdir, a.version, a.llhttp, a.cc, a.sysroot, a.rt, a.dry_run) + elif a.cmd == "splice": + splice(a.carrier, a.shipped_sha, a.rebuilt, a.wasm_bindgen) + elif a.cmd == "census": + shas = [sha(open(w, "rb").read()) for w in a.wasm] + if a.chdir: + os.chdir(a.chdir) + expect = {} + for e in a.expect: + k, _, v = e.rpartition("=") + expect[os.path.normpath(k)] = (int(v.rstrip("+")), v.endswith("+")) + raw = {} + for e in a.allow_raw: + k, _, v = e.rpartition("=") + raw[os.path.normpath(k)] = int(v) + census(a.roots, shas, a.allow_path, expect, raw) + elif a.cmd == "smoke": + smoke(a.node, a.wasm, a.fetch, a.require) + elif a.cmd == "selftest": + selftest() + except Fail as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print("ERROR: %s" % e, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/packages/next/build.ncl b/packages/next/build.ncl index 0021ce940..4f4da952b 100644 --- a/packages/next/build.ncl +++ b/packages/next/build.ncl @@ -10,6 +10,15 @@ let pkgconf = import "../pkgconf/build.ncl" in let pnpm = import "../pnpm/build.ncl" in let python = import "../python/build.ncl" in let toolchain = import "../toolchain/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in + +# undici whose llhttp C rebuilds the wasm next bundles into its edge runtime. +# Forwarded to build.sh as MINIMAL_ARG_UNDICI. +let undici = "6.28.0" in +# The llhttp that undici release vendors (its build.sh pins the wasm to it). +let undici_llhttp = "8.1.0" in let version = "16.4.1" in # The native SWC compiler Next loads at build/dev time. Upstream publishes it as a @@ -35,6 +44,13 @@ let swc_arm64_sha256 = "ea6ca74538e44799986dbea7ae0b7142b37abf5d6b16fcf34566221c extract = true, strip_prefix = "next.js-%{version}", } | Source, + { + url = "gs://minimal-staging-archives/undici-%{undici}.tar.gz", + sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", + } | Source, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, + llhttp-wasm-rebuild, match { { arch = 'Amd64, .. } => { @@ -71,6 +87,8 @@ let swc_arm64_sha256 = "ea6ca74538e44799986dbea7ae0b7142b37abf5d6b16fcf34566221c cmd = "./build.sh", build_args = { include version, + include undici, + include undici_llhttp, }, outputs = { diff --git a/packages/next/build.sh b/packages/next/build.sh index 79fce0b55..c96ba379a 100755 --- a/packages/next/build.sh +++ b/packages/next/build.sh @@ -72,6 +72,21 @@ done NEXT_DIR="$OUTPUT_DIR/usr/libexec/next/lib/node_modules/next" +# llhttp wasm: the edge runtime bundles undici 6's llhttp pair base64 in four files. +# Rebuilt from undici's C and spliced over the sha-pinned shipped blobs. +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" +mkdir -p /tmp/llhttp +tar -xof "$BUILD_ROOT/undici-$UNDICI.tar.gz" -C /tmp/llhttp +$LH build "/tmp/llhttp/undici-$UNDICI" /tmp/llhttp/wasm --version "$UNDICI" --llhttp "${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" --sysroot /usr/lib/wasi +LH_CARRIERS="dist/compiled/edge-runtime/index.js dist/compiled/@edge-runtime/primitives/fetch.js + dist/compiled/@edge-runtime/primitives/load.js dist/compiled/@vercel/blob/index.cjs" +LH_EXPECT= +for c in $LH_CARRIERS; do + $LH splice "$NEXT_DIR/$c" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 /tmp/llhttp/wasm/llhttp.wasm + $LH splice "$NEXT_DIR/$c" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d /tmp/llhttp/wasm/llhttp_simd.wasm + LH_EXPECT="$LH_EXPECT --expect $c=2" +done # Ship the platform SWC compiler inside next's own node_modules, where next resolves # `@next/swc-linux--gnu` first. The source build does not produce this native # addon; upstream publishes it as a per-platform npm package, fetched above as a @@ -183,3 +198,22 @@ if (!addon || !addon.startsWith(want)) { if (!s.versions.vips) throw new Error("sharp loaded without libvips"); console.log("sharp", s.versions.sharp, "vips", s.versions.vips, "addon", path.basename(addon)); ' "$NEXT_DIR/node_modules/sharp" + +# Every other wasm in the FINAL tree is still upstream's, by exact path: a new carrier fails here. +# Runs last, after the SWC addon and sharp are installed, so nothing lands after it unseen +# (review on #817: the x64 SWC addon embeds a raw wasm module and was installed after the census). +# Still upstream's: @mswjs llhttp (test mode only), og's resvg (Rust) and yoga (emscripten; also +# base64 in index.node.js), +# webpack's and loader-utils' xxhash64/md4 (AssemblyScript), source-map08 (Rust), and the x64 +# SWC addon: upstream's sha-pinned prebuilt (swc_amd64_sha256), one raw `\0asm` module +# ~96 MB in (from-source build: gominimal/pkgs#985). The arm64 addon embeds none. +$LH census -C "$NEXT_DIR" --wasm /tmp/llhttp/wasm/llhttp.wasm --wasm /tmp/llhttp/wasm/llhttp_simd.wasm $LH_EXPECT \ + --allow-path 'dist/compiled/@mswjs/interceptors/ClientRequest/llhttp/llhttp.wasm' \ + --allow-path 'dist/compiled/@vercel/og/resvg.wasm' --allow-path 'dist/compiled/@vercel/og/yoga.wasm' \ + --allow-path 'dist/compiled/@vercel/og/index.node.js' \ + --allow-path 'dist/compiled/loader-utils2/index.js' --allow-path 'dist/compiled/loader-utils3/index.js' \ + --allow-path 'dist/compiled/webpack/bundle5.js' --allow-path 'dist/compiled/source-map08/mappings.wasm' \ + --allow-path 'node_modules/@next/swc-linux-x64-gnu/next-swc.linux-x64-gnu.node' \ + . +$LH smoke node /tmp/llhttp/wasm/llhttp.wasm /tmp/llhttp/wasm/llhttp_simd.wasm \ + --require "$NEXT_DIR/dist/compiled/edge-runtime" diff --git a/packages/node-lts/build.ncl b/packages/node-lts/build.ncl index 174f6d35d..7674155b7 100644 --- a/packages/node-lts/build.ncl +++ b/packages/node-lts/build.ncl @@ -18,8 +18,22 @@ let zstd = import "../zstd/build.ncl" in let gtest = import "../gtest/build.ncl" in let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in +let amaro-wasm = import "../amaro-wasm/build.ncl" in let ca-certificates = import "../ca-certificates/build.ncl" in +# What this node tree bundles, verified by build.sh (check-tree / build fail on a mismatch), +# so a node bump re-derives each one here: undici under deps/undici (its llhttp C rebuilds +# the wasm js2c'd into node), the undici npm vendors under deps/npm (rebuilt from the +# mirrored release tarball below), and amaro (its swc wasm comes from packages/amaro-wasm). +let undici = "7.30.1" in +let undici_llhttp = "9.3.0" in +let npm_undici = "6.28.0" in +# The llhttp that the npm-vendored undici pins (forwarded so a bump is one edit). +let npm_undici_llhttp = "8.1.0" in +let amaro = "1.1.11" in let version = "24.22.1" in { name = "node-lts", @@ -29,9 +43,17 @@ let version = "24.22.1" in url = "https://nodejs.org/dist/v%{version}/node-v%{version}.tar.gz", sha256 = "001773d9bd53fbd27a25efd42aa6a9693ebfe500998ca9bec635e9b6ee79bc91", } | Source, + { + url = "gs://minimal-staging-archives/undici-%{npm_undici}.tar.gz", + sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", + } | Source, # 24.22.1 carries the CVE-2026-111943 fixes upstream (nodejs/node#66871 SPKI # blocklist, #66877 NSS 3.131.1 roots), so the backports are no longer carried. base, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, + llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs + amaro-wasm, # amaro's TypeScript-stripper wasm, rebuilt from swc make, pkgconf, python, @@ -62,6 +84,11 @@ let version = "24.22.1" in cmd = "./build.sh", build_args = { include version, + include undici, + include undici_llhttp, + include npm_undici, + include npm_undici_llhttp, + include amaro, }, outputs = { @@ -90,6 +117,15 @@ let version = "24.22.1" in } | Attrs, tests = { + type_strip = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "printf 'interface P { n: number }\\nconst p: P = { n: 6 };\\nconsole.log(p.n * 7);\\n' > t.ts"], + ["/bin/bash", "-c", "node t.ts | grep -qx 42"], + ], + } | Test, eval = { class = 'Standalone, diff --git a/packages/node-lts/build.sh b/packages/node-lts/build.sh index 29d3941d7..444930dfb 100755 --- a/packages/node-lts/build.sh +++ b/packages/node-lts/build.sh @@ -18,6 +18,35 @@ if grep -q "emSign ECC Root CA - G3" src/node_root_certs.h; then exit 1 fi +# llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped +# base64 blobs. Every pin fails the build on a node bump until re-derived. +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" +UNDICI_LLHTTP="${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" +NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" +AMARO_VERSION="${MINIMAL_ARG_AMARO:?build.ncl forwards amaro}" +NPM_LH=deps/npm/node_modules/undici/lib/llhttp +$LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi +tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp "${MINIMAL_ARG_NPM_UNDICI_LLHTTP:?build.ncl forwards npm_undici_llhttp}" --sysroot /usr/lib/wasi +$LH splice deps/undici/undici.js 57184a16144b5bc880e29105fa35d75c93dd318563e1b743e110de845e2ab661 ../wasm/llhttp.wasm +$LH splice deps/undici/undici.js 2434215e1f7fa0163f22ce895ddbeb83984fa0aa51e123164e053213e5aca028 ../wasm/llhttp_simd.wasm +$LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm +$LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm +# Never installed. +rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ + deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js +# amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced +# over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. +AMARO=/usr/share/amaro-wasm/$AMARO_VERSION/wasm_bg.wasm +grep -q "\"version\": \"$AMARO_VERSION\"" deps/amaro/package.json || { echo "deps/amaro is not $AMARO_VERSION" >&2; exit 1; } +$LH splice deps/amaro/dist/index.js 79c79a0be6c92c7d8a1968cb1a7bed88737755d38d2b47ddfe4da5e51796a4e9 "$AMARO" --wasm-bindgen +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" +$LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" --expect deps/amaro/dist/index.js=1 \ + deps lib src + case $(uname -m) in x86_64) MARCH="-march=x86-64-v3" ;; aarch64) MARCH="-march=armv8-a" ;; @@ -47,6 +76,11 @@ export CXXFLAGS="${CFLAGS}" --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-cares # Note: --shared-lief is omitted; that configure option was not added until Node.js v25. make -j$(nproc) +# type stripping runs on the rebuilt amaro +printf 'const n: number = 6;\nconsole.log(n * 7);\n' > ../strip.ts +out/Release/node ../strip.ts | grep -qx 42 +$LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ + ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install # The flag must have reached the binary: its --help marks the OpenSSL store as @@ -63,3 +97,10 @@ make DESTDIR=$OUTPUT_DIR install # user config) pointing globals at ~/.local, whose bin/ is already on the # session PATH. See gominimal/inbox#559. printf 'prefix=~/.local\n' > "$OUTPUT_DIR/usr/lib/node_modules/npm/npmrc" + +# Installed wasm is exactly the rebuilt blobs: js2c'd into node, verbatim in npm's undici. +# V8 test runtime (runtime-test-wasm.cc) compiles in two type-section-only modules. +$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ --allow-raw usr/bin/node=2 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js=1 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp_simd-wasm.js=1 \ + usr diff --git a/packages/node/build.ncl b/packages/node/build.ncl index e995b93bf..03f781cc6 100644 --- a/packages/node/build.ncl +++ b/packages/node/build.ncl @@ -19,8 +19,22 @@ let zstd = import "../zstd/build.ncl" in let gtest = import "../gtest/build.ncl" in let glibc = import "../glibc/build.ncl" in let gcc = import "../gcc/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let wasi-libc = import "../wasi-libc/build.ncl" in +let llhttp-wasm-rebuild = import "../llhttp-wasm-rebuild/build.ncl" in +let amaro-wasm = import "../amaro-wasm/build.ncl" in let ca-certificates = import "../ca-certificates/build.ncl" in +# What this node tree bundles, verified by build.sh (check-tree / build fail on a mismatch), +# so a node bump re-derives each one here: undici under deps/undici (its llhttp C rebuilds +# the wasm js2c'd into node), the undici npm vendors under deps/npm (rebuilt from the +# mirrored release tarball below), and amaro (its swc wasm comes from packages/amaro-wasm). +let undici = "8.12.2" in +let undici_llhttp = "9.3.1" in +let npm_undici = "6.28.0" in +# The llhttp that the npm-vendored undici pins (forwarded so a bump is one edit). +let npm_undici_llhttp = "8.1.0" in +let amaro = "1.1.11" in let version = "26.12.1" in { name = "node", @@ -36,7 +50,15 @@ let version = "26.12.1" in url = "https://nodejs.org/dist/v%{version}/node-v%{version}.tar.gz", sha256 = "717244a8d757672ec86aae5077fca31e50f84b094441ec51bdf397aab12c304e", } | Source, + { + url = "gs://minimal-staging-archives/undici-%{npm_undici}.tar.gz", + sha256 = "555a14624f66628d8aa7fcd0691ab2371f3e4512c40358bb7320bfabbb6763ec", + } | Source, base, + llvm, # clang + wasm-ld for the llhttp wasm + wasi-libc, + llhttp-wasm-rebuild, # rebuilds and splices undici's llhttp wasm blobs + amaro-wasm, # amaro's TypeScript-stripper wasm, rebuilt from swc make, pkgconf, python, @@ -66,6 +88,11 @@ let version = "26.12.1" in cmd = "./build.sh", build_args = { include version, + include undici, + include undici_llhttp, + include npm_undici, + include npm_undici_llhttp, + include amaro, }, outputs = { @@ -94,6 +121,15 @@ let version = "26.12.1" in } | Attrs, tests = { + type_strip = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "printf 'interface P { n: number }\\nconst p: P = { n: 6 };\\nconsole.log(p.n * 7);\\n' > t.ts"], + ["/bin/bash", "-c", "node t.ts | grep -qx 42"], + ], + } | Test, eval = { class = 'Standalone, diff --git a/packages/node/build.sh b/packages/node/build.sh index a0a28a4d8..a731e1feb 100755 --- a/packages/node/build.sh +++ b/packages/node/build.sh @@ -18,6 +18,38 @@ if grep -q "emSign ECC Root CA - G3" src/node_root_certs.h; then exit 1 fi +# llhttp wasm: rebuilt from undici's C and spliced over the sha-pinned shipped +# base64 blobs. Every pin fails the build on a node bump until re-derived. +LH="python3 /usr/share/llhttp-wasm-rebuild/rebuild-llhttp-wasm.py" +UNDICI="${MINIMAL_ARG_UNDICI:?build.ncl forwards undici}" +UNDICI_LLHTTP="${MINIMAL_ARG_UNDICI_LLHTTP:?build.ncl forwards undici_llhttp}" +NPM_UNDICI="${MINIMAL_ARG_NPM_UNDICI:?build.ncl forwards npm_undici}" +AMARO_VERSION="${MINIMAL_ARG_AMARO:?build.ncl forwards amaro}" +NPM_LH=deps/npm/node_modules/undici/lib/llhttp +$LH check-tree --undici "$UNDICI" --npm-undici "$NPM_UNDICI" +$LH build deps/undici/src ../wasm --version "$UNDICI" --llhttp "$UNDICI_LLHTTP" --sysroot /usr/lib/wasi +tar -xof "../undici-${NPM_UNDICI}.tar.gz" -C .. +$LH build "../undici-${NPM_UNDICI}" ../wasm-npm --version "$NPM_UNDICI" --llhttp "${MINIMAL_ARG_NPM_UNDICI_LLHTTP:?build.ncl forwards npm_undici_llhttp}" --sysroot /usr/lib/wasi +$LH splice deps/undici/undici.js ab4573a06c43574936dc98c3943b10a4ee32798e9797329e9f327e5f64115f54 ../wasm/llhttp.wasm +$LH splice deps/undici/undici.js 18403ee56fab07f89a845028bad358bb3ebcda213f8e2ec0989a991e9c33da69 ../wasm/llhttp_simd.wasm +$LH splice "$NPM_LH/llhttp-wasm.js" b96063c7ce14045f91f17489d8b30a2bf5129308bd801d7dde715579d16d0e21 ../wasm-npm/llhttp.wasm +$LH splice "$NPM_LH/llhttp_simd-wasm.js" 989f2025b23e92ae5093ceb357093df7bdf2e1e7f1f1bf383b0a4dc69a78151d ../wasm-npm/llhttp_simd.wasm +# Never installed. +rm deps/undici/src/lib/llhttp/llhttp.wasm deps/undici/src/lib/llhttp/llhttp_simd.wasm \ + deps/undici/src/lib/llhttp/llhttp-wasm.js deps/undici/src/lib/llhttp/llhttp_simd-wasm.js +# undici 8.12 vendors milo (Rust) as two base64 wasm carriers in its source tree; node's +# bundled undici.js compiles milo out (`getMiloEnabled } = false ? null : {}`), so never installed. +rm deps/undici/src/lib/milo/simd.js deps/undici/src/lib/milo/no-simd.js +# amaro (`node x.ts` type stripping): swc's wasm, rebuilt by packages/amaro-wasm and spliced +# over the sha-pinned shipped blob; --wasm-bindgen also renames the glue's one renumbered shim. +AMARO=/usr/share/amaro-wasm/$AMARO_VERSION/wasm_bg.wasm +grep -q "\"version\": \"$AMARO_VERSION\"" deps/amaro/package.json || { echo "deps/amaro is not $AMARO_VERSION" >&2; exit 1; } +$LH splice deps/amaro/dist/index.js 79c79a0be6c92c7d8a1968cb1a7bed88737755d38d2b47ddfe4da5e51796a4e9 "$AMARO" --wasm-bindgen +LH_WASM="--wasm ../wasm/llhttp.wasm --wasm ../wasm/llhttp_simd.wasm --wasm ../wasm-npm/llhttp.wasm --wasm ../wasm-npm/llhttp_simd.wasm --wasm $AMARO" +$LH census $LH_WASM --allow-path 'deps/v8/third_party/wasm-api/example/*' --allow-path 'deps/v8/test/*' --allow-path 'deps/v8/src/runtime/runtime-test-wasm.cc' --allow-path 'deps/v8/samples/*' \ + --expect deps/undici/undici.js=2 --expect "$NPM_LH/llhttp-wasm.js=1" --expect "$NPM_LH/llhttp_simd-wasm.js=1" --expect deps/amaro/dist/index.js=1 \ + deps lib src + case $(uname -m) in x86_64) MARCH="-march=x86-64-v3" ;; aarch64) MARCH="-march=armv8-a" ;; @@ -46,6 +78,11 @@ export CXXFLAGS="${CFLAGS}" --with-intl=system-icu --shared-openssl --shared-zlib --shared-zstd --shared-sqlite --shared-libuv \ --shared-nghttp2 --shared-nghttp3 --shared-ngtcp2 --shared-gtest --shared-lief --shared-cares make -j$(nproc) +# type stripping runs on the rebuilt amaro +printf 'const n: number = 6;\nconsole.log(n * 7);\n' > ../strip.ts +out/Release/node ../strip.ts | grep -qx 42 +$LH smoke out/Release/node ../wasm/llhttp.wasm ../wasm/llhttp_simd.wasm \ + ../wasm-npm/llhttp.wasm ../wasm-npm/llhttp_simd.wasm --require deps/npm/node_modules/undici --fetch make DESTDIR=$OUTPUT_DIR install # The flag must have reached the binary: its --help marks the OpenSSL store as @@ -62,3 +99,10 @@ make DESTDIR=$OUTPUT_DIR install # user config) pointing globals at ~/.local, whose bin/ is already on the # session PATH. See gominimal/inbox#559. printf 'prefix=~/.local\n' > "$OUTPUT_DIR/usr/lib/node_modules/npm/npmrc" + +# Installed wasm is exactly the rebuilt blobs: js2c'd into node, verbatim in npm's undici. +# V8 test runtime (runtime-test-wasm.cc) compiles in two type-section-only modules. +$LH census -C "$OUTPUT_DIR" $LH_WASM --expect usr/bin/node=2+ --allow-raw usr/bin/node=2 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp-wasm.js=1 \ + --expect usr/lib/node_modules/npm/node_modules/undici/lib/llhttp/llhttp_simd-wasm.js=1 \ + usr diff --git a/packages/wasi-libc/build.ncl b/packages/wasi-libc/build.ncl new file mode 100644 index 000000000..feec65ff9 --- /dev/null +++ b/packages/wasi-libc/build.ncl @@ -0,0 +1,74 @@ +let { Attrs, BuildSpec, Local, OutputData, OutputLib, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let llvm = import "../llvm/build.ncl" in +let make = import "../make/build.ncl" in + +# wasi-libc has no releases of its own; it is tagged with the wasi-sdk release +# that ships it. wasi-sdk-21 is commit c5264e2b, packaged by Alpine as 0.20231215. +let tag = "wasi-sdk-21" in +{ + name = "wasi-libc", + + build_deps = [ + { file = "build.sh" } | Local, + # Mirror of the upstream tag archive + # https://github.com/WebAssembly/wasi-libc/archive/refs/tags/%{tag}.tar.gz + # To audit it, fetch that URL and compare sha256. The archive is a + # `git archive`, so `gunzip -c | git get-tar-commit-id` must also + # print the commit the tag points at (c5264e2bbe532994d06b039005f2af91bedcc1a6). + { + url = "gs://minimal-staging-archives/wasi-libc-%{tag}.tar.gz", + sha256 = "4a2a3e3b120ba1163c57f34ac79c3de720a8355ee3a753d81f1f0c58c4cf6017", + extract = true, + strip_prefix = "wasi-libc-%{tag}", + } | Source, + base, + llvm, # clang, llvm-ar, llvm-nm + make, + ], + + cmd = "./build.sh", + outputs = { + # wasm32 objects and archives, not ELF. + libs = { glob = "usr/lib/wasi/lib/wasm32-wasi/*.{a,o}", allow_data = true } | OutputLib, + includes = { glob = "usr/lib/wasi/include/**" } | OutputData, + licenses = { glob = "usr/share/licenses/wasi-libc/*" } | OutputData, + }, + + attrs = + { + upstream_version = "0.20231215", + license_spdx = "(Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT) AND CC0-1.0 AND BSD-2-Clause AND MIT", + repology_project = "wasi-libc", + source_provenance = { + category = 'GithubRepo, + owner = "WebAssembly", + repo = "wasi-libc", + }, + } | Attrs, + + tests = { + # memcpy/memset lowered to memory.copy/memory.fill means the build picked up + # clang 21's default CPU instead of the pinned feature set. + no_bulk_memory = + { + class = 'Standalone, + test_deps = [base, llvm], + cmds = [ + ["/bin/bash", "-c", "set -eu; L=/usr/lib/wasi/lib/wasm32-wasi; llvm-objdump -d $L/libc.a $L/crt1-reactor.o > dis.txt; test -s dis.txt; ! grep -Eq 'memory\\.(copy|fill)|i(32|64)\\.trunc_sat' dis.txt"], + ], + } | Test, + + # A reactor module exporting malloc/free, the shape llhttp is linked in. + # -nodefaultlibs skips the compiler-rt builtins clang otherwise adds. + reactor_link = + { + class = 'Standalone, + test_deps = [base, llvm], + cmds = [ + ["/bin/bash", "-c", "printf '#include \\nint f(char *d, const char *s, int n) { memcpy(d, s, n); return (int)strlen(d); }\\n' > t.c"], + ["/bin/bash", "-c", "set -eu; clang --target=wasm32-wasi --sysroot=/usr/lib/wasi -mcpu=mvp -mmutable-globals -O2 -mexec-model=reactor -nodefaultlibs -Wl,--export=malloc -Wl,--export=free -Wl,--export=f -Wl,--no-entry -Wl,--strip-all -o t.wasm t.c -lc; test \"$(head -c 4 t.wasm | od -An -tx1 | tr -d ' ')\" = 0061736d"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/wasi-libc/build.sh b/packages/wasi-libc/build.sh new file mode 100755 index 000000000..64b3c1338 --- /dev/null +++ b/packages/wasi-libc/build.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -ex + +# clang 21's default wasm CPU turns on bulk-memory, multivalue, reference-types, +# nontrapping-fptoint and call-indirect-overlong; clang 17's enabled only +# sign-ext and mutable-globals. This pins the clang 17 feature set. +WASM_FEATURES="-mcpu=mvp -mmutable-globals" + +# clang 21 puts -Wunterminated-string-initialization in -Wextra; musl's +# vfprintf.c xdigits[16] trips it under the Makefile's -Werror. +NOWERR="-Wno-error=unterminated-string-initialization" + +# Empty BULK_MEMORY_SOURCES compiles musl's memcpy/memmove/memset without the +# per-file -mbulk-memory the Makefile adds. +# +# `finish` is not built: it depends on check-symbols, whose predefined-macro +# list stops at clang 18, so the steps it runs after `libc` are done below. +make -j"$(nproc)" \ + CC=clang \ + AR=llvm-ar \ + NM=llvm-nm \ + SYSROOT="$PWD/sysroot" \ + THREAD_MODEL=single \ + BULK_MEMORY_SOURCES= \ + EXTRA_CFLAGS="-O2 -DNDEBUG $WASM_FEATURES $NOWERR -ffile-prefix-map=$PWD=/builddir" \ + startup_files libc + +# The placeholder archives `finish` creates. +for name in m rt pthread crypt util xnet resolv; do + llvm-ar crs "sysroot/lib/wasm32-wasi/lib${name}.a" +done + +mkdir -p "$OUTPUT_DIR/usr/lib/wasi" +cp -r sysroot/include sysroot/lib "$OUTPUT_DIR/usr/lib/wasi/" + +mkdir -p "$OUTPUT_DIR/usr/share/licenses/wasi-libc" +cp LICENSE LICENSE-APACHE LICENSE-APACHE-LLVM LICENSE-MIT \ + "$OUTPUT_DIR/usr/share/licenses/wasi-libc/" +# The bundled third-party notices: cloudlibc (BSD-2-Clause) and musl (MIT). +# Renamed on install because cloudlibc's file is also called LICENSE. +cp libc-bottom-half/cloudlibc/LICENSE "$OUTPUT_DIR/usr/share/licenses/wasi-libc/LICENSE-cloudlibc" +cp libc-top-half/musl/COPYRIGHT "$OUTPUT_DIR/usr/share/licenses/wasi-libc/COPYRIGHT-musl" diff --git a/packages/wasm-bindgen-cli/build.ncl b/packages/wasm-bindgen-cli/build.ncl new file mode 100644 index 000000000..5a2289559 --- /dev/null +++ b/packages/wasm-bindgen-cli/build.ncl @@ -0,0 +1,70 @@ +let { subsetOf, Attrs, BuildSpec, Local, Needs, OutputBin, Source, Test, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let rust = import "../rust/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in + +let gcc = import "../gcc/build.ncl" in +let glibc = import "../glibc/build.ncl" in + +let version = "0.2.100" in +{ + name = "wasm-bindgen-cli", + # The wasm-bindgen post-processor: turns a rustc wasm32-unknown-unknown module plus its + # wasm-bindgen custom sections into the final module and its JS glue. Only the `wasm-bindgen` + # binary ships; its version must equal the wasm-bindgen crate locked by the code it processes. + build_deps = [ + { file = "build.sh" } | Local, + { + url = "https://static.crates.io/crates/wasm-bindgen-cli/wasm-bindgen-cli-%{version}.crate", + sha256 = "9e7f06f41580c87b6f4f84176d033f09c0838db1523a8d1899064b53cbb92c59", + # a .crate is a gzipped tar; build.sh unpacks it (Source extraction keys on the extension) + extract = false, + } | Source, + base, + rust, + toolchain, + ], + + runtime_deps = [ + glibc, + subsetOf gcc ["libgcc"], + ], + + # cargo fetches the crates in the published Cargo.lock (--locked) during the build. + needs = + { + dns = {}, + internet = {}, + } | Needs, + + cmd = "./build.sh", + build_args = { + include version, + }, + + outputs = { + bins = { glob = "usr/bin/wasm-bindgen" } | OutputBin, + }, + + attrs = + { + upstream_version = version, + license_spdx = "MIT OR Apache-2.0", + source_provenance = { + category = 'GithubRepo, + owner = "rustwasm", + repo = "wasm-bindgen", + }, + } | Attrs, + + tests = { + version_check = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "wasm-bindgen --version | grep -qx 'wasm-bindgen %{version}'"], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/wasm-bindgen-cli/build.sh b/packages/wasm-bindgen-cli/build.sh new file mode 100755 index 000000000..a4de81451 --- /dev/null +++ b/packages/wasm-bindgen-cli/build.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -ex + +V="${MINIMAL_ARG_VERSION:?build.ncl forwards version}" +tar -xzof "wasm-bindgen-cli-$V.crate" +cd "wasm-bindgen-cli-$V" + +export CC=gcc +export LD=gcc +export RUSTFLAGS="-C linker=gcc --remap-path-prefix=$(pwd)=/builddir --remap-path-prefix=$HOME/.cargo=/cargo -C codegen-units=1" + +cargo build --release --locked --bin wasm-bindgen + +mkdir -p "$OUTPUT_DIR/usr/bin" +cp target/release/wasm-bindgen "$OUTPUT_DIR/usr/bin/"