Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions packages/icu/build.ncl
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,24 @@ let gcc = import "../gcc/build.ncl" in
let toolchain = import "../toolchain/build.ncl" in
let make = import "../make/build.ncl" in
let glibc = import "../glibc/build.ncl" in
let python = import "../python/build.ncl" in

let version = "78.3" in
{
name = "icu",
build_deps = [
{ file = "build.sh" } | Local,
# Builds ICU's data generators from its BUILD.bazel files without Bazel.
{ file = "minibazel.py" } | Local,
# The tag, not the release tarballs: the data as text and the generators' sources, no prebuilt data.
# Mirror of https://github.com/unicode-org/icu/archive/refs/tags/release-78.3.tar.gz (byte-identical).
{
url = "https://github.com/unicode-org/icu/releases/download/release-%{version}/icu4c-%{version}-sources.tgz",
sha256 = "3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0",
url = "gs://minimal-staging-archives/icu-release-%{version}.tar.gz",
sha256 = "f06bcab72736ee9d55689033b8198a178562354128cf38edb2afc2e67e3fd931",
} | Source,
base,
make,
python,
toolchain,
],
runtime_deps = [
Expand Down
35 changes: 33 additions & 2 deletions packages/icu/build.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,39 @@
#!/bin/sh
set -e

tar -xof "icu4c-${MINIMAL_ARG_VERSION}-sources.tgz"
cd icu/source
# The release tarballs carry ICU's data prebuilt (icudt*.dat and the core Unicode property tables). The tag
# tarball has all of it as text, plus the generators; everything below is compiled from that.
tar -xof "icu-release-${MINIMAL_ARG_VERSION}.tar.gz"
S=$PWD/icu-release-${MINIMAL_ARG_VERSION}
C=$S/icu4c/source

# Core property tables: delete the checked-in copies, then regenerate them as ICU's
# data/unidata/generate.sh does, with its gennorm2/genprops/genuca built from its BUILD.bazel files (minibazel.py).
rm -f "$C"/common/norm2_nfc_data.h "$C"/common/propname_data.h "$C"/common/*_props_data.h \
"$C"/data/in/*.icu "$C"/data/in/*.nrm "$C"/data/in/coll/*.icu
if ls "$C"/data/in/*.dat > /dev/null 2>&1; then
echo "ERROR: prebuilt ICU data archive in the source tree" >&2
exit 1
fi
T=$PWD/gen-tools
N=$C/data/unidata/norm2
IN=$C/data/in
python3 minibazel.py "$S" "$T" //icu4c/source/tools/gennorm2
"$T/gennorm2" -o "$C/common/norm2_nfc_data.h" -s "$N" nfc.txt --csource
"$T/gennorm2" -o "$IN/nfc.nrm" -s "$N" nfc.txt
"$T/gennorm2" -o "$IN/nfkc.nrm" -s "$N" nfc.txt nfkc.txt
"$T/gennorm2" -o "$IN/nfkc_cf.nrm" -s "$N" nfc.txt nfkc.txt nfkc_cf.txt
"$T/gennorm2" -o "$IN/nfkc_scf.nrm" -s "$N" nfc.txt nfkc.txt nfkc_scf.txt
"$T/gennorm2" -o "$IN/uts46.nrm" -s "$N" nfc.txt uts46.txt
python3 minibazel.py "$S" "$T" //tools/unicode/c/genprops
"$T/genprops" "$S/icu4c"
python3 minibazel.py "$S" "$T" //tools/unicode/c/genuca
"$T/genuca" --hanOrder implicit "$S/icu4c"
"$T/genuca" --hanOrder radical-stroke "$S/icu4c"
"$T/genuca" --icu4x --hanOrder implicit "$S/icu4c"
"$T/genuca" --icu4x --hanOrder radical-stroke "$S/icu4c"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low · This PR moves every ICU data file to in-build generation. The package test is still only uconv --version (build.ncl:60), and that test reads none of this data. Today the byte-identity evidence exists only in the lab. On the next bump, a generator or minibazel.py regression can produce wrong normalization, property or collation data, and no package test will fail. Add a standalone test that loads the generated data. Examples: uconv -x any-nfkc on a known input, a converter round trip with uconv -f utf-8 -t iso-8859-1, and icuinfo to confirm that the data loads.


cd "$C"

case $(uname -m) in
x86_64) MARCH="-march=x86-64-v3" ;;
Expand Down
95 changes: 95 additions & 0 deletions packages/icu/minibazel.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""icu-minibazel.py ROOT OUT TARGET...: build ICU's cc_binary targets from its own BUILD.bazel files, without Bazel.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit · The docstring names the script icu-minibazel.py, but the file is minibazel.py, and build.sh calls it as minibazel.py. Use the real name.

Suggested change
"""icu-minibazel.py ROOT OUT TARGET...: build ICU's cc_binary targets from its own BUILD.bazel files, without Bazel.
"""minibazel.py ROOT OUT TARGET...: build ICU's cc_binary targets from its own BUILD.bazel files, without Bazel.


ICU's BUILD files use only cc_library/cc_binary with srcs, hdrs, deps, includes, local_defines, linkopts and
glob(include, exclude). They are evaluated as Python with those as stubs; each TARGET's transitive closure is
compiled (C with $CC, C++ with $CXX) and linked to OUT/<name>. ROOT is the repo root (holding icu4c/ and tools/).
"""
import glob as globmod
import os
import subprocess
import sys

root, out, targets = os.path.abspath(sys.argv[1]), os.path.abspath(sys.argv[2]), sys.argv[3:]
CC, CXX = os.environ.get("CC", "gcc"), os.environ.get("CXX", "g++")
CFLAGS = os.environ.get("CFLAGS", "-O2").split()
rules = {} # "//pkg:name" -> dict


def load_pkg(pkg):
path = os.path.join(root, pkg, "BUILD.bazel")
if not os.path.exists(path):
path = os.path.join(root, pkg, "BUILD")

def rule(kind):
def f(name, **kw):
rules["//%s:%s" % (pkg, name)] = dict(kw, kind=kind, pkg=pkg, name=name)
return f

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low · rule() accepts any keyword argument, but the build reads only srcs, deps, includes, local_defines and linkopts. A BUILD.bazel that adds defines, copts or implementation_deps is evaluated without error, and that attribute is dropped. The generators then compile with different flags or miss a dependency. They can emit data that differs from upstream, and that data ships in libicuuc/libicudata. The PR body says the 79 bump will run minibazel.py against BUILD files it was not tested on. Make rule() fail on any keyword outside the modeled set, so a new attribute stops the build in place of a silent change. The dedupe at line 87 has the same issue: a source listed by two rules gets only the first rule's local_defines. A check that the defines match can make that case fail too.


def glob(include, exclude=(), **_):
got = set()
for pat in include:
got.update(os.path.relpath(p, os.path.join(root, pkg))
for p in globmod.glob(os.path.join(root, pkg, pat)))
for pat in exclude:
got -= {os.path.relpath(p, os.path.join(root, pkg)) for p in globmod.glob(os.path.join(root, pkg, pat))}
return sorted(got)

env = {"cc_library": rule("lib"), "cc_binary": rule("bin"), "glob": glob,
"load": lambda *a, **k: None, "package": lambda **k: None, "exports_files": lambda *a, **k: None}
exec(compile(open(path).read(), path, "exec"), env)


def get(label, frm):
if label.startswith(":"):
label = "//%s%s" % (frm, label)
if ":" not in label:
label += ":" + label.rsplit("/", 1)[-1]
pkg = label[2:].split(":")[0]
if label not in rules:
load_pkg(pkg)
return rules[label]


def closure(label):
seen, order = set(), []

def walk(r):
key = "//%s:%s" % (r["pkg"], r["name"])
if key in seen:
return
seen.add(key)
for d in r.get("deps", []):
walk(get(d, r["pkg"]))
order.append(r)
walk(get(label, ""))
return order


os.makedirs(out, exist_ok=True)
for t in targets:
rs = closure(t)
incs, objs, links = set(), [], []
for r in rs:
for i in r.get("includes", []):
incs.add(os.path.normpath(os.path.join(root, r["pkg"], i)))
incs.add(os.path.join(root, r["pkg"]))
links += r.get("linkopts", [])
inc = ["-I" + i for i in sorted(incs)]
for r in rs:
defs = ["-D" + d for d in r.get("local_defines", [])]
for s in r.get("srcs", []):
if not s.endswith((".c", ".cpp", ".cc")):
continue
src = os.path.join(root, r["pkg"], s)
o = os.path.join(out, "obj", r["pkg"], s + ".o")
os.makedirs(os.path.dirname(o), exist_ok=True)
if o in objs: # a source listed by more than one cc_library (Bazel links archives; we link objects)
continue
if not os.path.exists(o):
cmd = ([CC] if s.endswith(".c") else [CXX, "-std=c++17"]) + CFLAGS + defs + inc + ["-c", src, "-o", o]
subprocess.run(cmd, check=True)
objs.append(o)
name = t.rsplit(":", 1)[-1] if ":" in t else t.rsplit("/", 1)[-1]
subprocess.run([CXX] + objs + ["-o", os.path.join(out, name)] + sorted(set(links)) + ["-lpthread", "-ldl"], check=True)
print("built %s from %d rules, %d objects" % (t, len(rs), len(objs)))
Loading