diff --git a/packages/async-profiler/DeterministicJar.java b/packages/async-profiler/DeterministicJar.java new file mode 100644 index 000000000..2456c54e1 --- /dev/null +++ b/packages/async-profiler/DeterministicJar.java @@ -0,0 +1,77 @@ +// Writes a jar whose bytes depend only on its inputs: entries in sorted path +// order, directory entries included, and one fixed timestamp on every entry. +// +// `jar --create` cannot do both at once. Fed only files (a sorted list), it +// writes no directory entries, which code that looks up a package directory +// with ClassLoader.getResource("") needs. Fed directories, it recurses in +// filesystem order, so the entry order would depend on the build host. +// +// Shared shape with packages/tlaplus/DeterministicJar.java; keep them in step. +// +// Run in source-file mode: java DeterministicJar.java +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class DeterministicJar { + // The zip (DOS) epoch; jar --date accepts nothing earlier. + private static final LocalDateTime EPOCH = LocalDateTime.of(1980, 1, 1, 0, 0, 2); + + public static void main(String[] args) throws IOException { + Path out = Paths.get(args[0]); + Path manifest = Paths.get(args[1]); + Path root = Paths.get(args[2]); + + List names = new ArrayList<>(); + try (Stream walk = Files.walk(root)) { + walk.filter(p -> !p.equals(root)).forEach(p -> { + String name = root.relativize(p).toString().replace('\\', '/'); + if (name.equals("META-INF") || name.equals("META-INF/MANIFEST.MF")) { + return; // written first, below + } + names.add(Files.isDirectory(p) ? name + "/" : name); + }); + } + Collections.sort(names); + + try (OutputStream fos = Files.newOutputStream(out); + ZipOutputStream zip = new ZipOutputStream(fos)) { + zip.setLevel(Deflater.BEST_COMPRESSION); + // The manifest leads, as java.util.jar.JarInputStream expects. + putDir(zip, "META-INF/"); + putFile(zip, "META-INF/MANIFEST.MF", Files.readAllBytes(manifest)); + for (String name : names) { + if (name.endsWith("/")) { + putDir(zip, name); + } else { + putFile(zip, name, Files.readAllBytes(root.resolve(name))); + } + } + } + } + + private static void putDir(ZipOutputStream zip, String name) throws IOException { + ZipEntry e = new ZipEntry(name); + e.setTimeLocal(EPOCH); + zip.putNextEntry(e); + zip.closeEntry(); + } + + private static void putFile(ZipOutputStream zip, String name, byte[] data) throws IOException { + ZipEntry e = new ZipEntry(name); + e.setTimeLocal(EPOCH); + zip.putNextEntry(e); + zip.write(data); + zip.closeEntry(); + } +} diff --git a/packages/async-profiler/build.ncl b/packages/async-profiler/build.ncl new file mode 100644 index 000000000..50692c16a --- /dev/null +++ b/packages/async-profiler/build.ncl @@ -0,0 +1,175 @@ +let { Attrs, BuildSpec, Local, OutputBin, OutputData, OutputLib, Source, Test, .. } = import "minimal.ncl" in +# async-profiler: a low-overhead sampling CPU / allocation / lock profiler for +# the JVM that writes flame graphs, collapsed stacks and JFR. Built from source +# (see build.sh): the native agent libasyncProfiler.so, the asprof attach +# launcher, the jfrconv converter and the Java API jar. +let base = import "../base/build.ncl" in +let bash = import "../bash/build.ncl" in +let glibc = import "../glibc/build.ncl" in +let jdk = import "../jdk/build.ncl" in +let make = import "../make/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in + +let version = "4.5" in +{ + name = "async-profiler", + build_deps = [ + { file = "build.sh" } | Local, + { file = "DeterministicJar.java" } | Local, + { + url = "https://github.com/async-profiler/async-profiler/archive/refs/tags/v%{version}.tar.gz", + sha256 = "807b83aefc86fcb067395d526684f3ea127919d611fe3bde1647938af7522869", + extract = true, + strip_prefix = "async-profiler-%{version}", + } | Source, + base, + toolchain, + make, + ], + + runtime_deps = [ + # libasyncProfiler.so and asprof link only libc / libdl / libpthread / + # librt; upstream links libstdc++ and libgcc statically into the .so. + glibc, + # jfrconv is a /bin/sh launcher that runs `java -jar` on itself; the agent + # is useless without a JVM to load it into, and the build needs javac. + bash, + jdk, + ], + + cmd = "./build.sh", + build_args = { + include version, + }, + + outputs = { + asprof = { glob = "usr/bin/asprof" } | OutputBin, + jfrconv = { glob = "usr/bin/jfrconv" } | OutputBin, + agent = { glob = "usr/lib/libasyncProfiler.so" } | OutputLib, + header = { glob = "usr/include/asprof.h" } | OutputData, + jars = { glob = "usr/share/async-profiler/*.jar" } | OutputData, + }, + + attrs = + { + upstream_version = version, + license_spdx = "Apache-2.0", + source_provenance = { + category = 'GithubRepo, + owner = "async-profiler", + repo = "async-profiler", + }, + } | Attrs, + + # Every profiling test uses event=itimer (setitimer + SIGPROF), which needs + # no privileges. The default event=cpu uses perf_events, which a container + # or sandbox usually forbids (perf_event_paranoid / seccomp). + tests = { + smoke = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + ["/bin/bash", "-c", "asprof --version | grep -q 'Async-profiler %{version} '"], + ["/bin/bash", "-c", "jfrconv --version | grep -q 'JFR converter %{version} '"], + ], + } | Test, + + # Load the agent at JVM start-up, profile a busy loop, and require the + # loop's own method in the dumped stacks; then do the same through JFR + # output and jfrconv (which exercises the converter jar). + agent = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + [ + "/bin/bash", + "-c", + m%" + set -eu + cd "$(mktemp -d)" + cat > Busy.java <<'EOF' + public class Busy { + static long burnCpu(long n) { + long x = 0; + for (long i = 0; i < n; i++) { + x += (i * i) ^ (x >>> 3); + } + return x; + } + public static void main(String[] args) { + long end = System.nanoTime() + Long.parseLong(args[0]) * 1000000000L; + long s = 0; + while (System.nanoTime() < end) { + s += burnCpu(1000000); + } + System.out.println(s != 42 ? "done" : ""); + } + } + EOF + javac Busy.java + agent=/usr/lib/libasyncProfiler.so + noinline=-XX:CompileCommand=quiet,dontinline,Busy::burnCpu + + java $noinline -agentpath:$agent=start,event=itimer,file=out.collapsed -cp . Busy 3 > run1.out 2>&1 \ + || { cat run1.out; exit 1; } + grep -q 'Busy.burnCpu' out.collapsed || { head -c 4000 out.collapsed; exit 1; } + + java $noinline -agentpath:$agent=start,event=itimer,file=out.jfr -cp . Busy 3 > run2.out 2>&1 \ + || { cat run2.out; exit 1; } + test -s out.jfr + jfrconv out.jfr conv.collapsed + grep -q 'Busy.burnCpu' conv.collapsed + jfrconv out.jfr conv.html + grep -qi '' conv.html + "%, + ], + ], + } | Test, + + # asprof attaches to an already-running JVM (HotSpot dynamic attach, no + # ptrace), profiles it for two seconds and writes collapsed stacks. + attach = + { + class = 'Standalone, + test_deps = [base], + cmds = [ + [ + "/bin/bash", + "-c", + m%" + set -eu + cd "$(mktemp -d)" + cat > Spin.java <<'EOF' + public class Spin { + static long spinLoop(long n) { + long x = 0; + for (long i = 0; i < n; i++) { + x += (i * 31) ^ (x >>> 5); + } + return x; + } + public static void main(String[] args) { + long end = System.nanoTime() + 20L * 1000000000L; + long s = 0; + while (System.nanoTime() < end) { + s += spinLoop(1000000); + } + System.out.println(s != 42 ? "done" : ""); + } + } + EOF + javac Spin.java + java -XX:CompileCommand=quiet,dontinline,Spin::spinLoop -cp . Spin > spin.out 2>&1 & + pid=$! + trap 'kill $pid 2>/dev/null || true' EXIT + sleep 3 + asprof -e itimer -d 2 -o collapsed -f "$PWD/attach.collapsed" $pid + grep -q 'Spin.spinLoop' attach.collapsed || { head -c 4000 attach.collapsed; exit 1; } + "%, + ], + ], + } | Test, + }, +} | BuildSpec diff --git a/packages/async-profiler/build.sh b/packages/async-profiler/build.sh new file mode 100755 index 000000000..c567fd1f0 --- /dev/null +++ b/packages/async-profiler/build.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Builds async-profiler from source: the native agent (libasyncProfiler.so) +# and the asprof launcher with upstream's Makefile, then the two jars and the +# jfrconv launcher by hand so the jars are reproducible (see below). +# +# Upstream's `make all` downloads nothing; only the test targets fetch jars. +set -eux + +TOP=$(pwd) +export JAVA_HOME=/usr/lib/jvm +# asprof and jattach print __DATE__ in --version, and jfrconv a build date +# (below). Both come from SOURCE_DATE_EPOCH, which the sandbox already sets +# (AGENTS.md), so they're deterministic without overriding it here. + +REPRO="-ffile-prefix-map=$TOP=/builddir -gno-record-gcc-switches -Wl,--build-id=none" + +# --- helper classes --------------------------------------------------------- +# src/helper/one/profiler/*.class are committed, prebuilt class files that the +# agent embeds with .incbin and injects into the target JVM. Rebuild them from +# the .java next to them, exactly as upstream's `%.class: %.java` rule does +# (-source/-target 8, not --release 8: JfrSync uses jdk.jfr, which the Java 8 +# API signature does not have). +for src in src/helper/one/profiler/*.java; do + rm -f "${src%.java}.class" + "$JAVA_HOME/bin/javac" -source 8 -target 8 -Xlint:-options -g:none "$src" +done + +# --- native: libasyncProfiler.so + asprof ------------------------------------ +# The Makefile's recipes take no LDFLAGS, so the linker flag rides in the +# *_EXTRA compiler flags (both are passed to the link). The .so is linked with +# -static-libstdc++ -static-libgcc by upstream, so it needs only glibc. +# CC/CXX on the command line: make's built-in CC=cc beats the Makefile's +# `CC ?= gcc`, and there is no `cc` here. +make -j"$(nproc)" \ + CC=gcc CXX=g++ \ + JAVA_HOME="$JAVA_HOME" \ + CFLAGS_EXTRA="$REPRO" \ + CXXFLAGS_EXTRA="$REPRO" \ + build/bin build/lib build/lib/libasyncProfiler.so build/bin/asprof + +# Upstream's release strips debug info from the .so (and ships it separately). +strip -g build/lib/libasyncProfiler.so + +# --- jars ------------------------------------------------------------------- +# Upstream builds these with `jar cfm` / `jar cfe`, which stamps the build time +# on every entry and walks directories in filesystem order. DeterministicJar +# writes sorted entries with one fixed timestamp instead. +javac_opts="--release 8 -Xlint:-options -encoding UTF-8" + +rm -rf build/api && mkdir -p build/api build/jar +find src/api/one/profiler -name '*.java' | LC_ALL=C sort > build/api-sources.txt +"$JAVA_HOME/bin/javac" $javac_opts -d build/api @build/api-sources.txt +{ + printf 'Manifest-Version: 1.0\n' + cat src/api/META-INF/MANIFEST.MF +} > build/api-manifest.txt +"$JAVA_HOME/bin/java" "$TOP/DeterministicJar.java" \ + build/jar/async-profiler.jar build/api-manifest.txt build/api + +# jfr-converter.jar = converter classes + the html templates in src/res + the +# native-image metadata in src/converter/META-INF (upstream: -C each of them). +rm -rf build/converter && mkdir -p build/converter +find src/converter -name '*.java' | LC_ALL=C sort > build/converter-sources.txt +"$JAVA_HOME/bin/javac" $javac_opts -d build/converter @build/converter-sources.txt +cp src/res/* build/converter/ +cp -R src/converter/META-INF build/converter/ +printf 'Manifest-Version: 1.0\nMain-Class: one.convert.Main\n' > build/converter-manifest.txt +"$JAVA_HOME/bin/java" "$TOP/DeterministicJar.java" \ + build/jar/jfr-converter.jar build/converter-manifest.txt build/converter + +# jfrconv: upstream's shell launcher with the converter jar appended to it +# (the launcher runs `java -jar "$0"`). BUILD_DATE comes from the sandbox's +# SOURCE_DATE_EPOCH. +build_date=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y") +sed -e "s/PROFILER_VERSION/$MINIMAL_ARG_VERSION/g" -e "s/BUILD_DATE/$build_date/g" \ + src/launcher/launcher.sh > build/bin/jfrconv +cat build/jar/jfr-converter.jar >> build/bin/jfrconv + +# --- install ---------------------------------------------------------------- +# asprof finds the agent at ../lib/libasyncProfiler.so relative to its own +# path, so /usr/bin + /usr/lib keeps upstream's bin/ + lib/ relationship. +install -Dm0755 build/bin/asprof "$OUTPUT_DIR/usr/bin/asprof" +install -Dm0755 build/bin/jfrconv "$OUTPUT_DIR/usr/bin/jfrconv" +install -Dm0755 build/lib/libasyncProfiler.so "$OUTPUT_DIR/usr/lib/libasyncProfiler.so" +install -Dm0644 src/asprof.h "$OUTPUT_DIR/usr/include/asprof.h" +install -Dm0644 build/jar/async-profiler.jar "$OUTPUT_DIR/usr/share/async-profiler/async-profiler.jar" +install -Dm0644 build/jar/jfr-converter.jar "$OUTPUT_DIR/usr/share/async-profiler/jfr-converter.jar"