From b2b0804b8bb179bc63cc19dca7c00061c877a002 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 22:08:51 +0000 Subject: [PATCH] fix(install): restart a microsandbox rerun cleanly and stop waiting with the logs and why Running the install line again over a microsandbox install could sit at "Waiting for the web app..." forever with no output: each check was a curl with no time limit, so a port that takes the connection but never answers held it for good, and the loop only printed when it gave up. - A rerun stops the running sandbox first (a minute to close the database, then forced), waits for its port to be free, and says who holds it if not. - Each check gives up after 5 seconds; the wait is time based (LENS_WAIT). - While waiting, it shows the steps the sandbox logs, and a line a minute. - It stops at once when a step fails or the sandbox stops, and on timeout, printing the last log lines and the reason (including a port that doesn't reach a Lens that is up inside the sandbox). - run.sh logs finer steps and a "Failed (exit N): step" line on error. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01C5hhPc28UXdCfRpV9qP9it --- .github/workflows/install.yml | 13 +++++- docs/get-started.md | 3 +- docs/microsandbox.md | 6 ++- install.sh | 84 +++++++++++++++++++++++++++++++---- microsandbox/run.sh | 27 ++++++++--- 5 files changed, 117 insertions(+), 16 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 5f7af3c17..34703228b 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -244,12 +244,23 @@ jobs: - name: Running it again updates in place and keeps the secrets run: | + # while it waited, it showed the steps the sandbox logged + grep -q '^ Building Lens' install.log || { echo "install.sh didn't show the sandbox's progress"; exit 1; } before="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" - LENS_BACKEND='' sh < install.sh + LENS_BACKEND='' sh < install.sh | tee again.log after="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" test "$before" = "$after" || { echo "the second run changed the secrets"; exit 1; } + grep -q "Stopping the Lens that's running" again.log || { echo "the second run didn't stop the running sandbox first"; exit 1; } curl -fsS -o /dev/null "$(cat setup-url)" + - name: A start that takes too long stops waiting and says why + run: | + if LENS_BACKEND='' LENS_WAIT=8 sh < install.sh >slow.log 2>&1; then cat slow.log; echo "it didn't stop waiting"; exit 1; fi + cat slow.log + grep -q "Lens didn't answer on port" slow.log + grep -q 'The last lines Lens logged:' slow.log + grep -q '^\[lens\] ' slow.log # the sandbox's own log + - name: What the sandbox said if: failure() run: | diff --git a/docs/get-started.md b/docs/get-started.md index c589d67e5..e9c3409b9 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -14,7 +14,8 @@ code already filled in. On a server without a desktop, Lens is reachable from th server's address. Run the same line again to update: the secrets, the database and the archive are kept. Settings go before `sh`: `LENS_DIR`, `LENS_PORT`, `LENS_PUBLIC` (`1`: from the network, `0`: this machine only), `LENS_TARGET` (`lean` for the smaller image) and `LENS_REF` (a branch or tag) change the defaults; the top of `install.sh` lists -them. +them. If Lens doesn't answer in time (`LENS_WAIT`, in seconds: 10 minutes with Docker, 30 in a microsandbox), the +installer stops waiting and shows the last lines Lens logged and why. Where it can (Linux with KVM, or a Mac with Apple silicon), a first install asks whether to run Lens in Docker (the default) or in a [microsandbox](microsandbox.md) microVM, which needs no Docker; `LENS_BACKEND=microsandbox` chooses diff --git a/docs/microsandbox.md b/docs/microsandbox.md index cf99feeb8..4f96e56f3 100644 --- a/docs/microsandbox.md +++ b/docs/microsandbox.md @@ -61,7 +61,11 @@ msb stop lens # stop it; ~/lens/.lens-sandbox starts it again msb exec lens -- sh # a shell inside it ``` -To update, run the install line again. +To update, run the install line again. It stops the running sandbox first (giving the database up to a minute to +close cleanly) and waits for its port to be free, then starts Lens in a new one. While it waits for the web app, it +shows the steps the sandbox logs (installing packages, building, starting the database). If the sandbox stops, a step +fails, or the web app doesn't answer within 30 minutes (`LENS_WAIT`, in seconds), it stops waiting and shows the last +lines the sandbox logged and why, for example a port that another program holds. ## Not yet diff --git a/install.sh b/install.sh index f1425683f..a57aa63be 100755 --- a/install.sh +++ b/install.sh @@ -20,6 +20,8 @@ # LENS_NO_OPEN 1: don't open a browser # LENS_YES 1: install what Lens needs without asking (it only asks at a terminal) # LENS_UPGRADE_AGENT 0: don't set up the agent that lets the web app install new releases (docs/updates.md) +# LENS_WAIT how long to wait for the web app to answer, in seconds (default: 1800 in a microsandbox, whose +# first start builds everything, else 600) # LENS_WIZARD 1: answer everything in the web installer, in the browser (the default for a first install at a # terminal, where Python 3 is there); 0: here in the terminal set -eu @@ -134,6 +136,7 @@ case "$BACKEND" in microsandbox) msb_ok || fail "microsandbox needs KVM on 64-bit Linux or a Mac with Apple silicon. Use LENS_BACKEND=docker." ;; *) fail "LENS_BACKEND is docker or microsandbox, not $BACKEND." ;; esac +case "${LENS_WAIT:-0}" in *[!0-9]*) fail "LENS_WAIT is a number of seconds, not ${LENS_WAIT:-}." ;; esac # --- What's missing: listed, and installed once you say yes --------------------------------------------------------- # Asked only at a terminal; piped in by a script or CI (no terminal, or LENS_YES=1) it goes ahead, as before. @@ -387,10 +390,27 @@ if [ "$BACKEND" = microsandbox ]; then echo } >.lens-sandbox chmod +x .lens-sandbox + # Run again: the running sandbox stops first, so its database closes cleanly (a replace kills it after 10 seconds) + # and its port is free for the new one + if msb ls --running -q /dev/null | grep -qx lens; then + say "Stopping the Lens that's running..." + msb stop -q -t 60 lens /dev/null 2>&1 || msb stop -q -f lens /dev/null 2>&1 || true + fi + i=0 + until free "$PORT"; do + i=$((i + 1)) + if [ $i -gt 15 ]; then + who="$(lsof -nP -iTCP:"$PORT" -sTCP:LISTEN 2>/dev/null | awk 'NR == 2 {print $1 " (process " $2 ")"}' || true)" + fail "Port $PORT is in use${who:+ by $who}, so Lens can't start on it. Stop that, or run this again with LENS_PORT set to a free port." + fi + sleep 2 + done + since="$(date -u +%Y-%m-%dT%H:%M:%SZ)" # what the new sandbox logs, for the progress below ./.lens-sandbox >/dev/null LOGS="msb logs -f lens" + TAIL="msb logs lens --tail 40" STOP="msb stop lens" - WAIT=900 # 30 minutes: the first start installs the system packages and builds everything + WAIT=1800 # 30 minutes: the first start installs the system packages and builds everything # back after a restart of the machine, as Docker's restart policy does if [ "$os" = Linux ] && have systemctl && [ -d /run/systemd/system ]; then unit=/etc/systemd/system/lens-microsandbox.service @@ -415,8 +435,9 @@ else say "Building and starting Lens (the first build takes a few minutes)..." $COMPOSE -f docker-compose.prod.yml up -d --build LOGS="$COMPOSE -f docker-compose.prod.yml logs" + TAIL="$COMPOSE -f docker-compose.prod.yml logs --tail 40" STOP="$COMPOSE -f docker-compose.prod.yml down" - WAIT=300 + WAIT=600 fi # --- The upgrade agent: runs packaging/host/lens-upgrade.sh when the web app asks for a new release --------------- @@ -465,12 +486,59 @@ PLIST fi say "Waiting for the web app..." -i=0 -# the page and the API behind it: the setup page needs both, and the API starts a few seconds after the page -until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/login" 2>/dev/null && - curl -fsS -o /dev/null "http://127.0.0.1:$PORT/api/v1/auth/status" 2>/dev/null; do - i=$((i + 1)) - [ $i -gt $WAIT ] && fail "Lens didn't answer within $((WAIT * 2 / 60)) minutes. See what happened: cd $DIR && $LOGS" +# the page and the API behind it: the setup page needs both, and the API starts a few seconds after the page. Each +# try gives up after a few seconds: a port that takes the connection but never answers mustn't hold this forever. +up() { + curl -fsS -o /dev/null --max-time 5 "http://127.0.0.1:$PORT/login" 2>/dev/null && + curl -fsS -o /dev/null --max-time 5 "http://127.0.0.1:$PORT/api/v1/auth/status" 2>/dev/null +} +# why it stopped waiting, after the last lines Lens logged +stuck() { + echo "The last lines Lens logged:" + $TAIL 2>&1 /dev/null | grep '^\[lens\] ' || true)" + n="$(printf '%s\n' "$lines" | grep -c . || true)" + if [ "$n" -gt "$seen" ]; then + printf '%s\n' "$lines" | sed -n "$((seen + 1)),\$p" | sed 's/^\[lens\]/ /' + seen=$n + shown=$now + step="$(printf '%s\n' "$lines" | tail -n 1 | sed 's/^\[lens\] //; s/[.][.][.]$//')" + fi + failed="$(printf '%s\n' "$lines" | sed -n 's/^\[lens\] \(Failed.*\)/\1/p' | tail -n 1)" + [ -z "$failed" ] || stuck "Lens couldn't start in its microsandbox. $failed." + if running="$(msb ls --running -q /dev/null)"; then # twice in a row, as it may be booting + if printf '%s\n' "$running" | grep -qx lens; then down=0; else down=$((down + 1)); fi + fi + [ $down -lt 2 ] || stuck "Lens's microsandbox stopped${step:+ at: $step}." +} +WAIT="${LENS_WAIT:-$WAIT}" +start=$(date +%s) +shown=$start checked=0 +until up; do + now=$(date +%s) + if [ "$BACKEND" = microsandbox ] && [ $((now - checked)) -ge 10 ]; then + checked=$now + sandbox_progress + fi + if [ $((now - start)) -ge "$WAIT" ]; then + why="Lens didn't answer on port $PORT within $(((now - start) / 60)) min${step:+ (its last step: $step)}." + # up in the sandbox, but not on this machine's port: the port is the problem, not Lens + if [ "$BACKEND" = microsandbox ] && + msb exec lens -- curl -fsS -o /dev/null --max-time 5 http://127.0.0.1:3000/login /dev/null 2>&1; then + why="Lens is running in its microsandbox, but port $PORT on this machine doesn't reach it. Is another program using the port?" + fi + stuck "$why" + fi + if [ $((now - shown)) -ge 60 ]; then # never silent for long + echo " still starting ($(((now - start) / 60)) min so far)..." + shown=$now + fi sleep 2 done diff --git a/microsandbox/run.sh b/microsandbox/run.sh index e3f6ee7e8..b74f64b2d 100755 --- a/microsandbox/run.sh +++ b/microsandbox/run.sh @@ -19,7 +19,21 @@ SURREAL_VERSION=v3.2.4 RCLONE_VERSION=v1.75.1 NODE_MAJOR=22 -log() { printf '[lens] %s\n' "$*"; } +# Each step is logged as "[lens] ...", which install.sh shows while it waits. A step that fails says so, and stops +# what was started: install.sh stops waiting and shows why. +step="" +log() { + step="$*" + printf '[lens] %s\n' "$*" +} +failed() { + code=$? + [ "$code" = 0 ] && return + printf '[lens] Failed (exit %s): %s\n' "$code" "${step%...}" + # shellcheck disable=SC2086 + for pid in ${pids:-}; do kill "$pid" 2>/dev/null || true; done +} +trap failed EXIT # The settings install.sh wrote (secrets, port, target), the same .env Compose reads set -a @@ -90,6 +104,7 @@ if [ "$(cat "$OPT/build-id" 2>/dev/null)" != "$build_id" ]; then (cd "$CODE/fastapi_backend" && tar -c --exclude .venv --exclude __pycache__ .) | tar -x -C "$OPT/backend" (cd "$CODE/nextjs-frontend" && tar -c --exclude node_modules --exclude .next .) | tar -x -C "$OPT/web-src" + log "Installing the backend's Python packages..." # shellcheck disable=SC2046 (cd "$OPT/backend" && UV_PROJECT_ENVIRONMENT="$OPT/venv" UV_PYTHON_DOWNLOADS=never UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy \ @@ -97,10 +112,11 @@ if [ "$(cat "$OPT/build-id" 2>/dev/null)" != "$build_id" ]; then uv sync --frozen --no-dev --python "$(command -v python3)" \ $(for e in ${EXTRAS:-}; do printf -- '--extra %s ' "$e"; done)) - (cd "$OPT/web-src" && - export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_HOME="$OPT/corepack" && - corepack pnpm install --frozen-lockfile && - AUTH_SECRET=build-only NEXT_TELEMETRY_DISABLED=1 NEXT_OUTPUT=standalone corepack pnpm build) + web() { (cd "$OPT/web-src" && exec env COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_HOME="$OPT/corepack" "$@"); } + log "Installing the web app's packages..." + web corepack pnpm install --frozen-lockfile + log "Building the web app..." + web AUTH_SECRET=build-only NEXT_TELEMETRY_DISABLED=1 NEXT_OUTPUT=standalone corepack pnpm build rm -rf "$OPT/web" && mkdir -p "$OPT/web/.next" cp -a "$OPT/web-src/.next/standalone/." "$OPT/web/" cp -a "$OPT/web-src/.next/static" "$OPT/web/.next/static" @@ -172,6 +188,7 @@ keep() { surreal_ready() { surreal is-ready --endpoint http://127.0.0.1:8001 >/dev/null 2>&1; } +log "Starting the database..." keep surrealdb "$DATA" surreal start --log info --bind 127.0.0.1:8001 --user "$SURREAL_USER" --pass "$SURREAL_PASS" \ "surrealkv:$DATA/surreal/lens.db" db=$!