Repository navigation
Expand file tree
/
Copy pathkernel-store-producer
More file actions
executable file
·243 lines (214 loc) · 9.61 KB
/
Copy pathkernel-store-producer
File metadata and controls
executable file
·243 lines (214 loc) · 9.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
#!/usr/bin/env bash
#
# kernel-store-producer — build the kernel store and deploy it to the workers.
#
# Builds the whole store (distro + upstream kernels) into a temp dir, tars
# it, and ships the tarball to every CI executor host in ktest-ci.json5. On each
# worker the tarball is unpacked into ~/.ktest/kernels-new and swapped into
# place against ~/.ktest/kernels with `mv --exchange`; the displaced old
# store is then removed.
#
# The tarball is kept across runs — overwritten by the next pack — so a
# failed deploy can be retried with --deploy-only without rebuilding.
#
# Partial failure is not fatal: a dead distro mirror or one broken kernel
# variant must not stop the store from being refreshed at all. Whatever built
# is packed and deployed, the gaps are named in the log, and the run still
# exits non-zero so the unit shows failed. The alternative — abort on the
# first failure — was what we had, and it was silent in the way that matters:
# a Fedora release going EOL froze the entire CI kernel store on 7.1.2 for
# seven weeks (2026-07-02 .. 2026-08-19) while every daily run "failed" in a
# log nobody reads. A missing kernel breaks the jobs that ask for it, loudly,
# on the next run; a frozen store silently invalidates every result.
#
# Modes (default is build + pack + deploy in one shot):
# --pack-only Build + pack, then stop.
# --deploy-only Skip build/pack; deploy the existing tarball.
# --dry-run, -n Print scp/ssh deploy commands instead of running them.
set -o nounset
set -o errexit
set -o errtrace
set -o pipefail
ktest_dir=$(dirname "$(readlink -f "$0")")
KTEST_DATA="${KTEST_DATA:-$HOME/.ktest}"
CONFIG_PATH="${KTEST_CI_CONFIG:-$KTEST_DATA/ktest-ci.json5}"
SSH="ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
# Same options for scp. A freshly-(re)deployed worker presents a new host key
# that accept-new must add; a bare scp fails non-interactively on the unknown
# key (manifests as "scp: Connection closed") before the $SSH step below could
# accept it.
SCP="scp -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
# Build the store into a real-disk staging dir (it's many GB — keep it off
# /tmp, which may be tmpfs). Script-scoped, not main()-local, so the EXIT
# trap can still see them after main() returns.
new="$KTEST_DATA/kernels-new"
tarball="$new.tar.zst"
log() { echo "kernel-store-producer: $*"; }
usage() {
cat <<EOF
Usage: kernel-store-producer [OPTIONS]
--pack-only Build and pack only; leave the tarball at
$tarball for a later deploy.
--deploy-only Skip build/pack; deploy the existing tarball at
$tarball.
--dry-run, -n Print scp/ssh deploy commands instead of running them.
-h, --help Show this help.
Default: build + pack + deploy.
EOF
}
do_pack=true
do_deploy=true
dry_run=false
while (($#)); do
case $1 in
--pack-only) do_deploy=false ;;
--deploy-only) do_pack=false ;;
--dry-run|-n) dry_run=true ;;
-h|--help) usage; exit 0 ;;
*) log "unknown argument: $1"; usage >&2; exit 1 ;;
esac
shift
done
# Clean up the build staging dir on exit. The tarball is intentionally
# kept across runs: it gets overwritten by the next pack anyway, and
# keeping it means --deploy-only can retry after a deploy failure
# without rebuilding the whole kernel store.
if $do_pack; then
trap 'rm -rf "$new"' EXIT
fi
# JSON5 → JSON: strip // comments (only after whitespace/SOL so URLs
# survive), drop trailing commas, quote unquoted object keys. Same
# reader as upstream-kernel-build / ktest-deploy.
json5_to_json() {
perl -0777 -pe '
s{(^|\s)//[^\n]*}{$1}gm;
s/,(\s*[}\]])/$1/g;
s/([\{,\[]\s*|^\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*):/$1"$2"$3:/gm;
' "$1"
}
# The deploy targets, one host per line. The kernel store goes to exactly
# the machines that run tests, so this is the same ktest-ci.json5 executor
# list ci-daemon / ktest-deploy read — one source of truth.
read_workers() {
[[ -f $CONFIG_PATH ]] || { log "no config at $CONFIG_PATH"; exit 1; }
json5_to_json "$CONFIG_PATH" | jq -r '.executors | keys[]'
}
# Print-or-run wrapper for --dry-run. Used for the outward-facing deploy
# commands (scp/ssh) only; local pack/build commands are not gated, since
# you can already skip them with --deploy-only.
run() {
if $dry_run; then
echo "kernel-store-producer: DRY-RUN: $*"
else
"$@"
fi
}
# Deploy the packed store to a single worker. Returns non-zero on any failure
# (unreachable host, scp/ssh error) instead of aborting, so the deploy loop can
# tolerate individual worker failures — one dead box must not waste the whole
# build.
deploy_to() {
local worker=$1
run $SCP -q "$tarball" "$worker:.ktest/kernel-store.tar.zst" || return 1
run $SSH "$worker" '
set -e
cd .ktest
rm -rf kernels-new
tar --zstd -xf kernel-store.tar.zst
# -T: kernels is the literal target, not a dir to move into.
# mkdir -p: --exchange needs both dirs (first deploy has none).
mkdir -p kernels
mv -T --exchange kernels-new kernels
rm -rf kernels-new
' || return 1
}
main() {
# Build stages that partially failed. Named here, re-reported at the end,
# and turned into the exit status — but they do not stop the deploy.
local -a degraded=()
if $do_pack; then
log "fetching distro kernels"
"$ktest_dir/target/release/distro-kernel-fetch" --output "$new" ||
degraded+=("distro fetch")
log "building upstream kernels"
"$ktest_dir/upstream-kernel-build" -o "$new" ||
degraded+=("upstream builds")
if (( ${#degraded[@]} )); then
log "WARNING: incomplete store (${degraded[*]}) — deploying what built anyway"
fi
# A failed build leaves its destination directory behind, empty or
# half-written, and packing that ships something worse than a gap: the
# store lists the variant, so it reads as present, but there is no
# kernel in it. That is the failure this script is supposed to have
# stopped making — an absence you can't see. It is not hypothetical;
# the first run after the tolerate-partial-failure change shipped an
# empty stable-kasan/aarch64/7.1.9.work.3 to four workers.
#
# manifest.json is the build-succeeded marker (it is exactly what
# upstream-kernel-build's already-built check keys on), so an entry
# without one is a husk. Distro entries carry manifests too, so this
# could widen to the whole store later — scoped to upstream/ for now
# because that's where the failure is demonstrated, and this deletes.
local husk
while IFS= read -r husk; do
log "dropping incomplete store entry ${husk#"$new/"}"
rm -rf "$husk"
done < <(find "$new/upstream" -mindepth 3 -maxdepth 3 -type d \
'!' -exec test -e '{}/manifest.json' ';' -print 2>/dev/null)
# ...and the <release>-<variant>/<arch>/ dirs the husk left behind,
# or `ls upstream/` still lists a variant that has no kernel under it.
# -delete implies -depth, so parents empty out bottom-up in one pass.
find "$new/upstream" -mindepth 1 -type d -empty -delete 2>/dev/null || true
# A store with gaps is acceptable, an empty one is not. The deploy
# swaps the worker's store wholesale, so shipping a store with no
# upstream kernels would delete every kernel the CI has instead of
# leaving the previous one in place. Tolerating partial failure is
# only safe with this floor under it.
# find exits non-zero when $new/upstream doesn't exist at all, and
# pipefail passes that on — which under errexit would abort here
# instead of reporting. That case is just "zero kernels".
local nr_kernels
nr_kernels=$(find "$new/upstream" -name manifest.json 2>/dev/null | wc -l) ||
nr_kernels=0
if (( nr_kernels == 0 )); then
log "no upstream kernels in the new store — refusing to deploy, keeping what the workers have"
exit 1
fi
log "new store has $nr_kernels upstream kernel(s)"
log "packing tarball"
# Archive the kernels-new/ directory itself, so the tarball has a
# single toplevel dir rather than exploding into the cwd.
tar --zstd -cf "$tarball" -C "$KTEST_DATA" kernels-new
fi
if $do_deploy; then
[[ -f $tarball ]] || { log "no tarball at $tarball (run --pack-only first?)"; exit 1; }
# A single unreachable / decommissioned worker must not abort the run:
# the build is expensive and the reachable workers still need the
# store. Warn-and-continue per worker; fail only if none took it.
local -a workers failed=()
mapfile -t workers < <(read_workers)
local ok=0
for worker in "${workers[@]}"; do
log "deploying to $worker"
if deploy_to "$worker"; then
ok=$((ok + 1))
else
log "WARNING: deploy to $worker failed — store NOT updated there, continuing"
failed+=("$worker")
fi
done
if (( ${#failed[@]} )); then
log "deployed to $ok/${#workers[@]} workers; failed: ${failed[*]}"
fi
if (( ok == 0 )); then
log "deploy failed: no workers received the store"
exit 1
fi
fi
if (( ${#degraded[@]} )); then
log "done, DEGRADED: ${degraded[*]} — store deployed with gaps"
exit 1
fi
log "done"
}
main