From 03f2c270dce1814bcc910b84e4ee3f84d1cf45e0 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Wed, 14 Oct 2026 10:05:55 +0000 Subject: [PATCH] fix(install): move a microsandbox install whose keys don't open its database to a volume of its own A reinstall made before #358, after ~/lens was deleted, wrote new keys but kept the earlier install's lens-data volume, and every update after it kept both: the API stopped at every start with "SURREAL_USER/SURREAL_PASS don't match". #358 only stopped a new first install from landing there. run.sh now signs in with .env's SURREAL_USER/SURREAL_PASS right after the database starts. When SurrealDB answers 401, it says "Another install's database", closes the database and stops instead of starting the API again and again. install.sh, seeing that line, moves Lens to a volume of its own (the first lens-data-N with no database, as a first install does), keeps it in .env as LENS_VOLUME, rewrites .lens-sandbox and starts again, once. The keys and the earlier volume are left as they are. CI: the "keeps stopping" check breaks ACCESS_SECRET_KEY instead of SURREAL_PASS, and a new step gives an install new keys and no LENS_VOLUME, as on Adeel's Mac, and checks that the line moves it to lens-data-3 and stays there. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_016M1M7EXU6uJVnqfp77L19k --- .github/workflows/install.yml | 34 ++++++++++++-- CHANGELOG.md | 7 +++ docs/microsandbox.md | 6 +++ install.sh | 88 ++++++++++++++++++++++++----------- microsandbox/run.sh | 23 +++++++++ 5 files changed, 127 insertions(+), 31 deletions(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index fc5ddacda..13b5c9e81 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -288,14 +288,14 @@ jobs: - name: A part of Lens that keeps stopping ends the wait at once, with what Lens logged run: | - # keys that don't open the database: the API stops at each start, as after a reinstall with new keys + # a key the API won't start with (shorter than 16 characters): it stops at each start cp "$LENS_DIR/.env" env.bak - sed -i 's/^SURREAL_PASS=.*/SURREAL_PASS=not-the-one/' "$LENS_DIR/.env" + sed -i 's/^ACCESS_SECRET_KEY=.*/ACCESS_SECRET_KEY=short/' "$LENS_DIR/.env" if LENS_BACKEND='' sh < install.sh >loop.log 2>&1; then cat loop.log; echo "it didn't stop waiting"; exit 1; fi cat loop.log grep -q "keeps stopping in its microsandbox" loop.log grep -q 'The last lines Lens logged:' loop.log - grep -q 'cannot open SurrealDB' loop.log # the reason, in Lens's own words + grep -q 'ACCESS_SECRET_KEY' loop.log # the reason, in Lens's own words cp env.bak "$LENS_DIR/.env" LENS_BACKEND='' sh < install.sh >/dev/null curl -fsS -o /dev/null "$(cat setup-url)" @@ -326,6 +326,34 @@ jobs: grep -q '^LENS_VOLUME=lens-data-2$' "$LENS_DIR/.env" curl -fsS -o /dev/null "$url" + - name: An install whose keys don't open its volume's database moves to a volume of its own when the line runs again + run: | + export PATH="$HOME/.local/bin:$HOME/.microsandbox/bin:$PATH" + # as an install that a reinstall gave new keys before LENS_VOLUME was kept: it runs on lens-data, whose database + # the first install's keys made (lens-data-2 holds the one this .env's earlier keys made) + sed -i '/^LENS_VOLUME=/d' "$LENS_DIR/.env" + for k in ACCESS_SECRET_KEY ARCHIVE_SECRET_KEY AUTH_SECRET SURREAL_PASS; do + sed -i "s/^$k=.*/$k=$(openssl rand -hex 32)/" "$LENS_DIR/.env" + done + before="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" + LENS_BACKEND='' sh < install.sh | tee moved.log + grep -q "Another install's database" moved.log || { echo "the sandbox didn't say the keys don't open the database"; exit 1; } + grep -q 'Starting Lens on a microsandbox volume of its own, lens-data-3' moved.log + grep -q '^LENS_VOLUME=lens-data-3$' "$LENS_DIR/.env" || { echo "the install didn't move to a volume of its own"; exit 1; } + grep -q "lens-data-3" "$LENS_DIR/.lens-sandbox" || { echo ".lens-sandbox still starts the earlier volume"; exit 1; } + after="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" + test "$before" = "$after" || { echo "moving changed the keys"; exit 1; } + for v in lens-data lens-data-2; do msb volume ls -q | grep -qx "$v" || { echo "the volume $v is gone"; exit 1; }; done + url="$(grep -Eo 'http://[^ ]+/setup\?code=[A-Za-z0-9]{12}' moved.log | head -1)" + test -n "$url" || { echo "install.sh didn't hand over a setup address"; exit 1; } + base="http://127.0.0.1:${url##*:}"; base="${base%%/setup*}" + curl -fsS "$base/api/v1/auth/status" | grep -q '"setup_required":true' + # running the line again stays on it + LENS_BACKEND='' sh < install.sh | tee stay.log + if grep -q 'volume of its own' stay.log; then echo "it moved again"; exit 1; fi + grep -q '^LENS_VOLUME=lens-data-3$' "$LENS_DIR/.env" + curl -fsS -o /dev/null "$url" + - name: What the sandbox said if: failure() run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 90116938e..ddd29b94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ The backend (`fastapi_backend`) and the frontend (`nextjs-frontend`) are version ## Unreleased +- **A microsandbox install whose keys don't open its database comes up when the line runs again.** A reinstall made + before 0.8.1, after `~/lens` was deleted, wrote new keys but kept the earlier install's `lens-data` volume, and every + update after it kept both: its API stopped at every start with `SURREAL_USER/SURREAL_PASS don't match`. The sandbox + now checks that the keys in `.env` open the database before it starts the API, and when they don't, the install line + moves Lens to a volume of its own (`lens-data-2`, kept in `.env` as `LENS_VOLUME`) and starts it there. The earlier + volume is left as it is ([microsandbox](docs/microsandbox.md#how-it-runs)). + ## 0.8.1 October 12, 2026 {id="0.8.1"} - **The installer stops waiting when Lens can't start, and shows why.** A microsandbox install that couldn't start Lens diff --git a/docs/microsandbox.md b/docs/microsandbox.md index 2eff9740e..5fcade5d1 100644 --- a/docs/microsandbox.md +++ b/docs/microsandbox.md @@ -53,6 +53,12 @@ after `~/lens` was deleted, leaves it as it is and starts on a volume of its own `.env` keeps that name (`LENS_VOLUME`), so running the line again uses the same one. Put the earlier `.env` back in the folder to use the earlier data again, or remove it with `msb volume rm lens-data`. +An install made before that check (0.8.0 or earlier) may already run on such a volume: its API stops at every start +with `SURREAL_USER/SURREAL_PASS don't match the user this database was created with`. Each start, the sandbox checks +that the keys in `.env` open the database. When they don't, it stops and says `Another install's database`, and the +install line moves Lens to a volume of its own the same way, keeping `.env`'s keys. So to recover, run the install line +again. The earlier volume is left as it is, for the `.env` that made it. + The sandbox can reach the internet, your network (mail servers, model servers) and this machine. On this machine, Ollama or LM Studio are at `host.docker.internal`, the same address as with Docker. Only the web app's port and the sensor ports are published, on the same addresses Docker would use (`LENS_BIND`, `LENS_SENSOR_BIND`). diff --git a/install.sh b/install.sh index aa1058711..eaeafeb44 100755 --- a/install.sh +++ b/install.sh @@ -423,30 +423,26 @@ if [ "$BACKEND" = microsandbox ]; then fi done [ -n "$image" ] || fail "Couldn't download the base image (python:3.12-slim-bookworm): $out" - # The internet, this network (mail servers, model servers) and this machine (Ollama, LM Studio); only the web app - # the sensor ports and the file shares' ports (S3, WebDAV) are published, as with Docker. - set -- run -d -q --name lens --replace -c "$cpus" -m "${mem}M" --root-disk 16G --net public,private,host \ - -p "$bind:$PORT:3000" \ - -p "$sensors:${LENS_MQTT_PORT:-1883}:1883" \ - -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" \ - -p "${LENS_SHARE_BIND:-$bind}:${LENS_S3_PORT:-9555}:9555" -p "${LENS_SHARE_BIND:-$bind}:${LENS_WEBDAV_PORT:-9556}:9556" - if [ "$LENS_HTTPS" = 1 ]; then # Caddy listens on 8443 and 8080 inside (docs/https.md) - set -- "$@" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTPS_PORT:8443" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTP_PORT:8080" - fi # The volume its data is kept in (the database, the archive, the builds). A database opens only with the keys of # the install that made it, which are in that install's .env. So a first install that finds the volume of an earlier # one whose folder is gone (deleted, or installed elsewhere) starts on a volume of its own, and leaves that one as it - # is: its data is still there for that install's .env. A volume without a database yet is used as it is. + # is: its data is still there for that install's .env. A volume without a database yet is used as it is. An install + # that already runs on such a volume moves to one of its own the same way, once run.sh finds that the keys in .env + # don't open its database (other_volume, below). + # own_volume [NAME]: the first of lens-data, lens-data-2, ... that holds no database (and isn't NAME) + own_volume() { + volumes="$(msb volume ls -q /dev/null || true)" + VOLUME=lens-data n=1 + while [ "$VOLUME" = "${1:-}" ] || { printf '%s\n' "$volumes" | grep -qx "$VOLUME" && has_db "$VOLUME"; }; do + n=$((n + 1)) + VOLUME="lens-data-$n" + done + } VOLUME="$(grep '^LENS_VOLUME=' .env | cut -d= -f2- || true)" if [ -z "$VOLUME" ]; then VOLUME=lens-data if [ "$fresh" = 1 ]; then - volumes="$(msb volume ls -q /dev/null || true)" - n=1 - while printf '%s\n' "$volumes" | grep -qx "$VOLUME" && has_db "$VOLUME"; do - n=$((n + 1)) - VOLUME="lens-data-$n" - done + own_volume if [ "$VOLUME" != lens-data ]; then say "Starting Lens on a microsandbox volume of its own, $VOLUME..." echo " The volume lens-data holds an earlier install's data, which opens only with that install's .env. It's left" @@ -455,19 +451,32 @@ if [ "$BACKEND" = microsandbox ]; then fi setenv LENS_VOLUME "$VOLUME" fi - set -- "$@" -v "$PWD:/lens:ro" -v "$PWD/.lens-control:/control" --mount-named "$VOLUME:/data" \ - --entrypoint /bin/sh "$image" -- /lens/microsandbox/run.sh - say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." # The command that starts it, kept for a restart of the machine: a sandbox started again (msb start) boots without # running Lens, so it is always replaced; its packages come from the cache in the volume, its build stays there. - { - echo "#!/bin/sh" - echo "# Starts Lens in its microsandbox (written by install.sh)" - printf 'exec %s' "'$(command -v msb)'" - for a in "$@"; do printf " '%s'" "$a"; done - echo - } >.lens-sandbox - chmod +x .lens-sandbox + write_sandbox() { + # The internet, this network (mail servers, model servers) and this machine (Ollama, LM Studio); only the web app + # the sensor ports and the file shares' ports (S3, WebDAV) are published, as with Docker. + set -- run -d -q --name lens --replace -c "$cpus" -m "${mem}M" --root-disk 16G --net public,private,host \ + -p "$bind:$PORT:3000" \ + -p "$sensors:${LENS_MQTT_PORT:-1883}:1883" \ + -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" \ + -p "${LENS_SHARE_BIND:-$bind}:${LENS_S3_PORT:-9555}:9555" -p "${LENS_SHARE_BIND:-$bind}:${LENS_WEBDAV_PORT:-9556}:9556" + if [ "$LENS_HTTPS" = 1 ]; then # Caddy listens on 8443 and 8080 inside (docs/https.md) + set -- "$@" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTPS_PORT:8443" -p "${LENS_HTTPS_BIND:-0.0.0.0}:$HTTP_PORT:8080" + fi + set -- "$@" -v "$PWD:/lens:ro" -v "$PWD/.lens-control:/control" --mount-named "$VOLUME:/data" \ + --entrypoint /bin/sh "$image" -- /lens/microsandbox/run.sh + { + echo "#!/bin/sh" + echo "# Starts Lens in its microsandbox (written by install.sh)" + printf 'exec %s' "'$(command -v msb)'" + for a in "$@"; do printf " '%s'" "$a"; done + echo + } >.lens-sandbox + chmod +x .lens-sandbox + } + say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." + write_sandbox # Run again: the running sandbox stops first, so its database closes cleanly (a replace kills it after 10 seconds) # and its port is free for the new one if msb ls --running -q /dev/null | grep -qx lens; then @@ -579,6 +588,25 @@ stuck() { $TAIL 2>&1 /dev/null 2>&1 || true + since="$(date -u +%Y-%m-%dT%H:%M:%SZ)" seen=0 down=0 step="" + ./.lens-sandbox >/dev/null + start=$(date +%s) # a first start on this volume: it installs and builds everything again + shown=$start +} # the steps the sandbox reports (run.sh's "[lens] ..." lines), each shown once; it stops waiting when one failed or # the sandbox stopped seen=0 down=0 step="" @@ -592,6 +620,10 @@ sandbox_progress() { shown=$now step="$(printf '%s\n' "$lines" | tail -n 1 | sed 's/^\[lens\] //; s/[.][.][.]$//')" fi + if [ -z "$moved" ] && printf '%s\n' "$lines" | grep -q "^\[lens\] Another install's database"; then + other_volume + return 0 + fi failed="$(printf '%s\n' "$lines" | sed -n 's/^\[lens\] \(Failed.*\)/\1/p' | tail -n 1)" [ -z "$failed" ] || stuck "Lens couldn't start in its microsandbox. $failed." # a part that stops again and again (run.sh starts it again each time) won't start by waiting longer diff --git a/microsandbox/run.sh b/microsandbox/run.sh index b39cb2553..3a8ad26cc 100755 --- a/microsandbox/run.sh +++ b/microsandbox/run.sh @@ -198,6 +198,12 @@ keep() { } surreal_ready() { surreal is-ready --endpoint http://127.0.0.1:8001 >/dev/null 2>&1; } +# surreal_signin: how SurrealDB answers .env's SURREAL_USER and SURREAL_PASS (200: they open it, 401: they don't) +surreal_signin() { + python3 -c 'import json, os; print(json.dumps({"user": os.environ["SURREAL_USER"], "pass": os.environ["SURREAL_PASS"]}))' | + curl -s -o /dev/null -w '%{http_code}' --max-time 10 -X POST -H 'Accept: application/json' --data-binary @- \ + http://127.0.0.1:8001/signin || true +} # --- Programs: LibreOffice and Chromium, installed the first time Lens needs one (the full edition; a lean install # never fetches them) -------------------------------------------------------------------------------------------- @@ -219,6 +225,23 @@ keep surrealdb "$DATA" surreal start --log info --bind 127.0.0.1:8001 --user "$S "surrealkv:$DATA/surreal/lens.db" db=$! until surreal_ready; do sleep 1; done +# The database keeps the user it was first started with, so the keys in .env must be the ones that made it. Those of +# another install (a reinstall after the install folder was deleted wrote new ones, and kept this volume) never open +# it: rather than starting the API again and again, say so and stop. install.sh then starts Lens on a volume of its own. +signin="" +for _ in 1 2 3 4 5; do + signin="$(surreal_signin)" + case "$signin" in 200 | 401) break ;; esac + sleep 1 +done +if [ "$signin" = 401 ]; then + log "Another install's database: the keys in .env don't open the one in this volume" + trap - EXIT + # shellcheck disable=SC2086 + for pid in $pids; do kill "$pid" 2>/dev/null || true; done + wait # the database closes cleanly + exit 3 +fi keep api "$OPT/backend" fastapi run app/main.py --host 127.0.0.1 --port 8000 --proxy-headers keep worker "$OPT/backend" lens worker keep web "$OPT/web" env HOSTNAME=0.0.0.0 PORT=3000 node lens-server.js