From 06add959eb6906251b3e4884de8fc6ee19352168 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 12:40:53 +0000 Subject: [PATCH 1/4] feat(install): run Lens in a microsandbox microVM as an alternative to Docker install.sh asks on a first install (where KVM or Apple silicon is there) whether to run Lens in Docker, the default, or in one microsandbox microVM; LENS_BACKEND chooses without asking and is kept in .env. microsandbox/run.sh sets up and runs SurrealDB, the API, the worker and the web app inside the sandbox from the stock Python image. A CI job runs the one-line install in a microsandbox on a KVM runner. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VvAGWoQxiwkGkk374pvkSh --- .github/workflows/install.yml | 62 ++++++++++++ .gitignore | 2 + docs/get-started.md | 4 + docs/microsandbox.md | 70 +++++++++++++ install.sh | 123 +++++++++++++++++++++-- microsandbox/run.sh | 178 ++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 7 files changed, 432 insertions(+), 8 deletions(-) create mode 100644 docs/microsandbox.md create mode 100755 microsandbox/run.sh diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e049abda0..9750f33da 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -12,6 +12,7 @@ on: - fastapi_backend/pyproject.toml - fastapi_backend/uv.lock - fastapi_backend/start.sh + - microsandbox/** - nextjs-frontend/Dockerfile - nextjs-frontend/package.json - nextjs-frontend/pnpm-lock.yaml @@ -21,6 +22,7 @@ on: paths: - install.sh - docker-compose.prod.yml + - microsandbox/** - .github/workflows/install.yml schedule: - cron: "17 4 * * 1" @@ -83,3 +85,63 @@ jobs: cd "$LENS_DIR" docker compose -f docker-compose.prod.yml down -v || true fi + + microsandbox: + name: One-line install in a microsandbox + # GitHub's own Linux runners: they have KVM, which microsandbox needs + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + LENS_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + LENS_REF: ${{ github.event.pull_request.head.ref || github.ref_name }} + LENS_DIR: /tmp/lens-install + LENS_NO_OPEN: "1" + LENS_BACKEND: microsandbox + LENS_TARGET: lean + steps: + - uses: actions/checkout@v4 + + - name: KVM for the runner's user + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + + - name: Install (as a user would, piped into sh) + run: | + sh < install.sh | tee install.log + grep -Eo 'http://[^ ]+/setup\?code=[A-Za-z0-9]{12}' install.log | head -1 > setup-url + test -s setup-url || { echo "install.sh didn't hand over a setup address"; exit 1; } + + - name: The setup address answers, and Lens waits for its first admin + run: | + url="$(cat setup-url)" + base="http://127.0.0.1:${url##*:}"; base="${base%%/setup*}" + curl -fsS -o /dev/null "$url" + status="$(curl -fsS "$base/api/v1/auth/status")" + echo "$status" + echo "$status" | grep -q '"setup_required":true' + grep -q '^LENS_BACKEND=microsandbox$' "$LENS_DIR/.env" + + - name: Running it again updates in place and keeps the secrets + run: | + before="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" + LENS_BACKEND= sh < install.sh + after="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" + test "$before" = "$after" || { echo "the second run changed the secrets"; exit 1; } + curl -fsS -o /dev/null "$(cat setup-url)" + + - name: What the sandbox said + if: failure() + run: | + export PATH="$HOME/.local/bin:$PATH" + msb doctor || true + msb ls || true + msb logs lens --tail 300 || true + + - name: Stop + if: always() + run: | + export PATH="$HOME/.local/bin:$PATH" + msb rm -f lens || true diff --git a/.gitignore b/.gitignore index ab5c272ad..3fdb9c328 100644 --- a/.gitignore +++ b/.gitignore @@ -170,3 +170,5 @@ local-shared-data/* # ci/local-ci.sh logs /.local-ci/ +.lens-build +.lens-sandbox diff --git a/docs/get-started.md b/docs/get-started.md index 2049d2d0f..7cd8a1be3 100644 --- a/docs/get-started.md +++ b/docs/get-started.md @@ -14,6 +14,10 @@ before `sh`: `LENS_DIR`, `LENS_PORT`, `LENS_PUBLIC` (`1`: from the network, `0`: (`lean` for the smaller image) and `LENS_REF` (a branch or tag) change the defaults; the top of `install.sh` lists them. +Where it can (Linux with KVM, or a Mac with Apple silicon), a first install asks whether to run Lens in Docker (the +default) or in a [microsandbox](microsandbox.md) microVM, which needs no Docker; `LENS_BACKEND=microsandbox` chooses +it without asking. + ## With Docker and make You need Docker with Compose, and `make`. diff --git a/docs/microsandbox.md b/docs/microsandbox.md new file mode 100644 index 000000000..f55760930 --- /dev/null +++ b/docs/microsandbox.md @@ -0,0 +1,70 @@ +# microsandbox + +Lens can run in a [microsandbox](https://github.com/superradcompany/microsandbox) microVM instead of Docker: one small +virtual machine with its own kernel, and no Docker on the machine. Docker stays the default. + +**Status: built, early.** microsandbox itself is beta software. + +## Install + +Run the one-line install. On a first install where microsandbox can run, it asks which to use: + +``` +How should Lens run? + 1) Docker (default): a container for each part, the usual way + 2) microsandbox: one lightweight virtual machine, no Docker needed +``` + +Or choose without the question: `curl -fsSL .../install.sh | LENS_BACKEND=microsandbox sh`. The choice is kept in +`.env` (`LENS_BACKEND`), so running the line again updates the same install. An install that already runs on Docker +stays on Docker, because its database and archive are in Docker volumes. + +It needs hardware virtualization: + +- **Linux** on x86-64 or 64-bit ARM (a Raspberry Pi 4 or 5 on a 64-bit OS) with `/dev/kvm`. The installer adds you to + the `kvm` group. `msb doctor` checks the machine. +- **A Mac** with Apple silicon. + +## How it runs + +microsandbox runs images but can't build them, and its sandboxes can't reach each other by name, so Lens runs as one +sandbox named `lens`, started from the stock `python:3.12-slim-bookworm` image. Inside it, +[`microsandbox/run.sh`](https://github.com/adeelahmad/lense/blob/main/microsandbox/run.sh) does what the Docker images +do: + +1. Installs the system packages, the same ones as the backend image (`LENS_TARGET=lean` leaves out LibreOffice and + Chromium). +2. Fetches SurrealDB 3.2.4, rclone, Node 22 and uv. +3. Builds the backend and the web app from the code, read-only at `/lens`. It builds again only when the code + changes. +4. Runs SurrealDB, the API, a job worker and the web app, and restarts any of them that stops. + +The database, the archive and the builds are in the named volume `lens-data`, which outlives the sandbox. The first +start installs and builds everything, which takes several minutes (longer on a Raspberry Pi). Later starts take +seconds. + +The sandbox can reach the internet, your network (mail servers, model servers) and this machine. On this machine, +Ollama or LM Studio are at `host.docker.internal`, the same address as with Docker. Only the web app's port and the +sensor ports are published, on the same addresses Docker would use (`LENS_BIND`, `LENS_SENSOR_BIND`). + +On Linux with systemd, the installer adds a `lens-microsandbox` service that starts the sandbox when the machine +starts. On a Mac, it adds a LaunchAgent that starts it at sign-in. + +## Day to day + +```bash +msb logs -f lens # follow the logs +msb stop lens # stop it; msb start lens starts it again +msb exec lens -- sh # a shell inside it +``` + +To update, run the install line again. + +## Not yet + +- **Matterbridge, Fedora and OpenSearch.** The opt-in Compose services don't run in the sandbox yet. Use Docker for + these. +- **Prebuilt images.** Each machine builds Lens itself, as `install.sh` does with Docker. Published images would make + the first start quicker. +- **Moving an install.** A Docker install doesn't move into a sandbox yet: its database and archive stay in Docker's + volumes. diff --git a/install.sh b/install.sh index 2ddf766d1..b8cde9438 100755 --- a/install.sh +++ b/install.sh @@ -3,9 +3,9 @@ # # curl -fsSL https://raw.githubusercontent.com/adeelahmad/lense/main/install.sh | sh # -# Gets the code (or updates it), writes the secrets once, builds and starts the stack with Docker, waits until the web -# app answers, then opens it on the setup page with the setup code already filled in. Run it again to update: the -# secrets, the database and the archive are kept. +# Gets the code (or updates it), writes the secrets once, builds and starts the stack with Docker (or in a microsandbox +# microVM, if chosen), waits until the web app answers, then opens it on the setup page with the setup code already +# filled in. Run it again to update: the secrets, the database and the archive are kept. # # Settings, all optional (put them before `sh`: `... | LENS_DIR=/srv/lens sh`): # LENS_DIR where Lens goes (default: ~/lens) @@ -14,6 +14,8 @@ # LENS_PUBLIC 1: reachable from other machines on the network, 0: this machine only (default: 1 on a server # without a desktop, else 0) # LENS_TARGET full (reads Office files, web pages and emails too) or lean (smaller) (default: full) +# LENS_BACKEND docker or microsandbox (one microVM, no Docker: docs/microsandbox.md). Asked on a first install +# when microsandbox can run here (Linux with KVM, or a Mac with Apple silicon); default: docker # GITHUB_TOKEN a GitHub token to use when downloading (not needed; avoids GitHub's rate limits) # LENS_NO_OPEN 1: don't open a browser set -eu @@ -39,8 +41,58 @@ code() { LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null; then + printf '\n\033[1mHow should Lens run?\033[0m\n' + echo " 1) Docker (default): a container for each part, the usual way" + echo " 2) microsandbox: one lightweight virtual machine, no Docker needed" + printf 'Choose [1]: ' + read -r answer /dev/null || true + # the group counts from the next sign-in; for this run, access for this user alone + have setfacl && $(sudo_ok) setfacl -m "u:$(id -un):rw" /dev/kvm 2>/dev/null || true + [ -r /dev/kvm ] && [ -w /dev/kvm ] || fail "Can't use /dev/kvm. Sign out and in again (or run: msb doctor), then run this again." +fi +else +# --- Docker ------------------------------------------------------------------------------------------------------- if ! have docker; then case "$os" in Linux) @@ -82,6 +134,7 @@ elif have docker-compose; then else fail "Docker Compose is missing. Install the Compose plugin (docker-compose-plugin), then run this again." fi +fi # --- The code ----------------------------------------------------------------------------------------------------- auth_url="https://github.com/$REPO.git" @@ -154,6 +207,7 @@ if [ ! -f .env ]; then setenv SURREAL_PASS "$(secret)" fi grep -q '^LENS_SETUP_CODE=' .env || setenv LENS_SETUP_CODE "$(code)" # only used until the first admin exists +setenv LENS_BACKEND "$BACKEND" setenv LENS_TARGET "$TARGET" setenv LENS_BIND "$bind" setenv LENS_PORT "$PORT" @@ -161,8 +215,61 @@ setenv FRONTEND_URL "http://$host:$PORT" setup_code="$(grep '^LENS_SETUP_CODE=' .env | cut -d= -f2-)" # --- Build and start ---------------------------------------------------------------------------------------------- -say "Building and starting Lens (the first build takes a few minutes)..." -$COMPOSE -f docker-compose.prod.yml up -d --build +if [ "$BACKEND" = microsandbox ]; then + # the code it was built from: the sandbox builds again when this changes + { git rev-parse HEAD 2>/dev/null || date +%s; } >.lens-build + # Memory: three quarters of this machine's, 2 to 8 GB (building the web app needs about 2) + if [ "$os" = Darwin ]; then mem=$(($(sysctl -n hw.memsize) / 1048576)); else mem=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo) / 1024)); fi + mem=$((mem * 3 / 4)) + [ "$mem" -lt 2048 ] && mem=2048 + [ "$mem" -gt 8192 ] && mem=8192 + cpus="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)" + sensors="${LENS_SENSOR_BIND:-$bind}" + # The internet, this network (mail servers, model servers) and this machine (Ollama, LM Studio); only the web app + # and the sensor ports are published, as with Docker. + set -- run -d -q --name lens --replace -c "$cpus" -m "${mem}M" --root-disk 16G --net public,private,host \ + -p "$bind:$PORT:3000" \ + -p "$sensors:${LENS_MQTT_PORT:-1883}:1883" \ + -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/udp" -p "$sensors:${LENS_SYSLOG_PORT:-5514}:5514/tcp" \ + -v "$PWD:/lens:ro" --mount-named lens-data:/data \ + --entrypoint /bin/sh python:3.12-slim-bookworm -- /lens/microsandbox/run.sh + say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." + if [ "$*" = "$(cat .lens-sandbox 2>/dev/null)" ] && msb ls -q 2>/dev/null | grep -qx lens; then + msb restart lens >/dev/null # same settings: keeps its installed packages + else + msb "$@" >/dev/null + printf '%s' "$*" >.lens-sandbox + fi + LOGS="msb logs -f lens" + STOP="msb stop lens" + WAIT=900 # 30 minutes: the first start installs the system packages and builds everything + # back after a restart of the machine, as Docker's restart policy does + if [ "$os" = Linux ] && have systemctl && [ -d /run/systemd/system ]; then + unit=/etc/systemd/system/lens-microsandbox.service + printf '%s\n' "[Unit]" "Description=Lens (microsandbox)" "After=network-online.target" "Wants=network-online.target" "" \ + "[Service]" "Type=oneshot" "RemainAfterExit=yes" "User=$(id -un)" "ExecStart=$(command -v msb) start lens" \ + "ExecStop=$(command -v msb) stop lens" "" "[Install]" "WantedBy=multi-user.target" | + $(sudo_ok) tee "$unit" >/dev/null && $(sudo_ok) systemctl enable lens-microsandbox.service >/dev/null 2>&1 || true + elif [ "$os" = Darwin ]; then + plist="$HOME/Library/LaunchAgents/dev.lens.microsandbox.plist" + mkdir -p "$(dirname "$plist")" + cat >"$plist" < + + + Labeldev.lens.microsandbox + ProgramArguments$(command -v msb)startlens + RunAtLoad + +PLIST + fi +else + say "Building and starting Lens (the first build takes a few minutes)..." + $COMPOSE -f docker-compose.prod.yml up -d --build + LOGS="$COMPOSE -f docker-compose.prod.yml logs" + STOP="$COMPOSE -f docker-compose.prod.yml down" + WAIT=300 +fi say "Waiting for the web app..." i=0 @@ -170,14 +277,14 @@ i=0 until curl -fsS -o /dev/null "http://127.0.0.1:$PORT/login" 2>/dev/null && curl -fsS -o /dev/null "http://127.0.0.1:$PORT/api/v1/auth/status" 2>/dev/null; do i=$((i + 1)) - [ $i -gt 300 ] && fail "Lens didn't answer within 10 minutes. See what happened: cd $DIR && $COMPOSE -f docker-compose.prod.yml logs" + [ $i -gt $WAIT ] && fail "Lens didn't answer within $((WAIT * 2 / 60)) minutes. See what happened: cd $DIR && $LOGS" sleep 2 done url="http://$host:$PORT/setup?code=$setup_code" say "Lens is running: $url" echo " (once the admin account exists, the link opens the sign-in page)" -echo " Stop: cd $DIR && $COMPOSE -f docker-compose.prod.yml down Update: run the same line again" +echo " Stop: cd $DIR && $STOP Update: run the same line again" if [ "${LENS_NO_OPEN:-0}" != 1 ]; then if [ "$os" = Darwin ]; then open "$url" 2>/dev/null || true diff --git a/microsandbox/run.sh b/microsandbox/run.sh new file mode 100755 index 000000000..5e202e86d --- /dev/null +++ b/microsandbox/run.sh @@ -0,0 +1,178 @@ +#!/bin/sh +# Lens in one microsandbox microVM (docs/microsandbox.md): SurrealDB, the API, a job worker and the web app, with no +# Docker on the machine. install.sh starts it with LENS_BACKEND=microsandbox: +# +# msb run -d --name lens ... -v ~/lens:/lens:ro --mount-named lens-data:/data python:3.12-slim-bookworm \ +# -- sh /lens/microsandbox/run.sh +# +# microsandbox runs images but can't build them, and sandboxes can't reach each other by name, so the stack the Docker +# images hold is set up here, inside one sandbox started from the stock Python image: the system packages once, then +# the backend and the web app from the code in /lens (read-only) each time it changes. Everything the stack keeps (the +# database, the archive, the built code) is in /data, the named volume that outlives the sandbox. Each process is +# restarted if it stops, as `restart: unless-stopped` does in Compose. +set -eu + +CODE=/lens +DATA=/data +OPT=$DATA/.lens # tools and builds, kept with the data so a replaced sandbox doesn't download them again +SURREAL_VERSION=v3.2.4 +RCLONE_VERSION=v1.75.1 +NODE_MAJOR=22 + +log() { printf '[lens] %s\n' "$*"; } + +# The settings install.sh wrote (secrets, port, target), the same .env Compose reads +set -a +# shellcheck disable=SC1091 +. "$CODE/.env" +set +a +TARGET="${LENS_TARGET:-full}" + +case "$(uname -m)" in + x86_64) arch=amd64 node_arch=x64 ;; + aarch64 | arm64) arch=arm64 node_arch=arm64 ;; + *) log "Unsupported CPU: $(uname -m)" && exit 1 ;; +esac + +mkdir -p "$OPT/bin" "$DATA/surreal" /tmp/lens +export PATH="$OPT/bin:$OPT/node/bin:$OPT/venv/bin:$PATH" + +# --- System packages: in the sandbox's own disk, so installed again if the sandbox is replaced --------------------- +packages="ffmpeg tesseract-ocr antiword poppler-utils ca-certificates curl xz-utils unzip" +[ "$TARGET" = full ] && packages="$packages libreoffice-writer-nogui libreoffice-calc-nogui libreoffice-impress-nogui + chromium fonts-dejavu-core fonts-liberation2 fonts-noto-core fonts-noto-cjk" +stamp="/var/lib/lens-packages.$TARGET" +if [ ! -f "$stamp" ]; then + log "Installing system packages ($TARGET)..." + export DEBIAN_FRONTEND=noninteractive + apt-get update -q + # shellcheck disable=SC2086 + apt-get install -y -q --no-install-recommends $packages + rm -rf /var/lib/apt/lists/* + id lens >/dev/null 2>&1 || useradd --create-home --uid 1000 lens + touch "$stamp" +fi + +# --- Tools: SurrealDB, rclone, Node, uv, pinned like the Docker images ------------------------------------------- +fetch() { curl -fsSL --retry 5 --retry-delay 3 "$@"; } +if [ ! -x "$OPT/bin/surreal" ] || ! "$OPT/bin/surreal" version 2>/dev/null | grep -q "${SURREAL_VERSION#v}"; then + log "Getting SurrealDB $SURREAL_VERSION..." + fetch "https://download.surrealdb.com/$SURREAL_VERSION/surreal-$SURREAL_VERSION.linux-$arch.tgz" | tar -xz -C "$OPT/bin" +fi +if [ ! -x "$OPT/bin/rclone" ]; then + log "Getting rclone $RCLONE_VERSION..." + fetch -o /tmp/lens/rclone.zip "https://downloads.rclone.org/$RCLONE_VERSION/rclone-$RCLONE_VERSION-linux-$arch.zip" + unzip -q -o -j /tmp/lens/rclone.zip '*/rclone' -d "$OPT/bin" && rm /tmp/lens/rclone.zip +fi +if [ ! -x "$OPT/node/bin/node" ]; then + log "Getting Node $NODE_MAJOR..." + base="https://nodejs.org/dist/latest-v$NODE_MAJOR.x" + file="$(fetch "$base/SHASUMS256.txt" | awk "/linux-$node_arch.tar.xz\$/ {print \$2}")" + rm -rf "$OPT/node" && mkdir -p "$OPT/node" + fetch "$base/$file" | tar -xJ -C "$OPT/node" --strip-components 1 +fi +if [ ! -x "$OPT/bin/uv" ]; then + log "Getting uv..." + python3 -m pip install -q --no-cache-dir --target /tmp/lens/uv "uv>=0.8,<0.9" + cp /tmp/lens/uv/bin/uv "$OPT/bin/uv" && rm -rf /tmp/lens/uv +fi + +# --- Build: the backend and the web app, again only when the code changed ---------------------------------------- +build_id="$(cat "$CODE/.lens-build" 2>/dev/null || echo unknown)-$TARGET" +if [ "$(cat "$OPT/build-id" 2>/dev/null)" != "$build_id" ]; then + log "Building Lens (the first build takes a few minutes)..." + # copies, so the build writes nothing into the code on the host; the backend is installed where it runs (the + # virtualenv points at it) + rm -rf "$OPT/backend" "$OPT/web-src" && mkdir -p "$OPT/backend" "$OPT/web-src" + (cd "$CODE/fastapi_backend" && tar -c --exclude .venv --exclude __pycache__ .) | tar -x -C "$OPT/backend" + (cd "$CODE/nextjs-frontend" && tar -c --exclude node_modules --exclude .next .) | tar -x -C "$OPT/web-src" + + # shellcheck disable=SC2046 + (cd "$OPT/backend" && + UV_PROJECT_ENVIRONMENT="$OPT/venv" UV_PYTHON_DOWNLOADS=never UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy \ + UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 UV_CACHE_DIR="$OPT/uv-cache" \ + uv sync --frozen --no-dev --python "$(command -v python3)" \ + $(for e in ${EXTRAS:-}; do printf -- '--extra %s ' "$e"; done)) + + (cd "$OPT/web-src" && + export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 COREPACK_HOME="$OPT/corepack" && + corepack pnpm install --frozen-lockfile && + AUTH_SECRET=build-only NEXT_TELEMETRY_DISABLED=1 NEXT_OUTPUT=standalone corepack pnpm build) + rm -rf "$OPT/web" && mkdir -p "$OPT/web/.next" + cp -a "$OPT/web-src/.next/standalone/." "$OPT/web/" + cp -a "$OPT/web-src/.next/static" "$OPT/web/.next/static" + cp -a "$OPT/web-src/public" "$OPT/web-src/lens-server.js" "$OPT/web/" + rm -rf "$OPT/web-src" + chown -R lens:lens "$OPT/web" + echo "$build_id" >"$OPT/build-id" +fi +# the archive is the volume itself (data_dir: /data, as in the containers); the database is a folder in it +chown lens:lens "$DATA" +chown -R lens:lens "$DATA/surreal" + +# --- Run --------------------------------------------------------------------------------------------------------- +# The same settings docker-compose.prod.yml gives the containers, with every service on this sandbox's localhost +export SURREAL_URL=ws://127.0.0.1:8001 SURREAL_USER="${SURREAL_USER:-root}" +export RUN_BACKGROUND=false +export FRONTEND_URL="${FRONTEND_URL:-http://localhost:3000}" +export CORS_ORIGINS="[\"$FRONTEND_URL\"]" +export ARCHIVE_ALLOWED_HOSTS="${ARCHIVE_ALLOWED_HOSTS:-localhost,127.0.0.1}" +export LENS_TUNNEL_ORIGIN=http://127.0.0.1:3000 +export LENS_TRUSTED_PROXY_HOSTS=localhost +export LENS_WEB_PROXY_HOSTS=localhost +export API_BASE_URL=http://127.0.0.1:8000 +export AUTH_TRUST_HOST=true NODE_ENV=production NEXT_TELEMETRY_DISABLED=1 +export HOME=/home/lens XDG_CACHE_HOME="$DATA/.cache" +mkdir -p "$DATA/.cache" && chown lens:lens "$DATA/.cache" +# the archive in the volume, as /data in the containers; the config file the backend image ships +export ARCHIVE_CONFIG="$OPT/backend/docker/archive.yaml" + +# model servers on the host machine (Ollama, LM Studio) under the name the Docker setup uses for them too +if ! grep -q host.docker.internal /etc/hosts; then + host_ip="$(getent hosts host.microsandbox.internal | awk '{print $1; exit}')" + [ -n "$host_ip" ] && echo "$host_ip host.docker.internal" >>/etc/hosts +fi + +pids="" db="" +stop() { + log "Stopping..." + # the web app, the API and the worker first, then the database they write to + # shellcheck disable=SC2086 + for pid in $pids; do [ "$pid" = "$db" ] || kill "$pid" 2>/dev/null || true; done + # shellcheck disable=SC2086 + for pid in $pids; do [ "$pid" = "$db" ] || wait "$pid" 2>/dev/null || true; done + kill "$db" 2>/dev/null || true + wait + exit 0 +} +trap stop TERM INT + +# keep NAME DIR COMMAND...: run it as the lens user, again 5 seconds after it stops +keep() { + name=$1 dir=$2 + shift 2 + ( + child= + trap 'kill $child 2>/dev/null; wait $child; exit 0' TERM + while :; do + (cd "$dir" && exec setpriv --reuid lens --regid lens --init-groups -- "$@") & + child=$! + wait $child || true + log "$name stopped; starting it again in 5 seconds" + sleep 5 + done + ) & + pids="$pids $!" +} + +surreal_ready() { surreal is-ready --endpoint http://127.0.0.1:8001 >/dev/null 2>&1; } + +keep surrealdb "$DATA" surreal start --log info --bind 127.0.0.1:8001 --user "$SURREAL_USER" --pass "$SURREAL_PASS" \ + "surrealkv:$DATA/surreal/lens.db" +db=$! +until surreal_ready; do sleep 1; done +keep api "$OPT/backend" fastapi run app/main.py --host 127.0.0.1 --port 8000 --proxy-headers +keep worker "$OPT/backend" lens worker +keep web "$OPT/web" env HOSTNAME=0.0.0.0 PORT=3000 node lens-server.js +log "Lens is starting on port 3000" +wait diff --git a/mkdocs.yml b/mkdocs.yml index 599dcb3b4..812bcc9f3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -94,6 +94,7 @@ nav: - Encryption: encryption.md - Remote access: remote-access.md - Deployment: deployment.md + - microsandbox: microsandbox.md - Technology: technology-selection.md - Contributing: contributing.md - Changelog: CHANGELOG.md From 883351f757ed52f242ab52692991e40929536174 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 12:42:36 +0000 Subject: [PATCH 2/4] ci(install): quote the empty LENS_BACKEND for shellcheck Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VvAGWoQxiwkGkk374pvkSh --- .github/workflows/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 9750f33da..08d3b9a91 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -127,7 +127,7 @@ jobs: - name: Running it again updates in place and keeps the secrets run: | before="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" - LENS_BACKEND= sh < install.sh + LENS_BACKEND='' sh < install.sh after="$(grep -E '^(ACCESS_SECRET_KEY|ARCHIVE_SECRET_KEY|AUTH_SECRET|SURREAL_PASS|LENS_SETUP_CODE)=' "$LENS_DIR/.env" | sha256sum)" test "$before" = "$after" || { echo "the second run changed the secrets"; exit 1; } curl -fsS -o /dev/null "$(cat setup-url)" From 8af9f2e9990de014e9e842a88bdfd4437b310a4d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 12:56:25 +0000 Subject: [PATCH 3/4] fix(sensors): take another free syslog port when TCP has the one UDP picked With port 0, UDP chose a free port and TCP then had to bind the same number; when something already held it on TCP the listener failed to start, which made test_hub_takes_devices_in_and_organises_them fail now and then. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VvAGWoQxiwkGkk374pvkSh --- fastapi_backend/app/domain/syslog.py | 19 ++++++++++++------- fastapi_backend/tests/api/test_sensors.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/fastapi_backend/app/domain/syslog.py b/fastapi_backend/app/domain/syslog.py index a9d87a16e..27bf5a07c 100644 --- a/fastapi_backend/app/domain/syslog.py +++ b/fastapi_backend/app/domain/syslog.py @@ -131,13 +131,18 @@ class TcpServer(socketserver.ThreadingTCPServer): if ":" in host: UdpServer.address_family = TcpServer.address_family = socket.AF_INET6 - self.udp = UdpServer((host, port), Udp) - self.port = self.udp.server_address[1] - try: - self.tcp = TcpServer((host, self.port), Tcp) if tcp else None - except OSError: - self.udp.server_close() - raise + # Port 0 (any free one): UDP picks it, and TCP needs the same number free too; when something else holds it + # on TCP, try another rather than fail + for attempt in range(1 if port else 20): + self.udp = UdpServer((host, port), Udp) + self.port = self.udp.server_address[1] + try: + self.tcp = TcpServer((host, self.port), Tcp) if tcp else None + break + except OSError: + self.udp.server_close() + if port or attempt == 19: + raise self.threads = [] def start(self): diff --git a/fastapi_backend/tests/api/test_sensors.py b/fastapi_backend/tests/api/test_sensors.py index 661794539..ca35d963c 100644 --- a/fastapi_backend/tests/api/test_sensors.py +++ b/fastapi_backend/tests/api/test_sensors.py @@ -138,6 +138,26 @@ def test_syslog_tcp_framing(): assert list(syslog._frames(io.BufferedReader(io.BytesIO(lines)))) == [b"<30>a b: c\n", b"<30>d e: f\r\n"] +def test_syslog_any_free_port_skips_one_taken_on_tcp(monkeypatch): + """Port 0: UDP picks a port that may be in use on TCP (other tests, other programs); another is tried.""" + import socketserver + + bind, tries = socketserver.ThreadingTCPServer.server_bind, [] + + def busy_once(self): + tries.append(self.server_address) + if len(tries) == 1: + raise OSError(98, "Address already in use") + bind(self) + + monkeypatch.setattr(socketserver.ThreadingTCPServer, "server_bind", busy_once) + listener = syslog.Listener("127.0.0.1", 0, [], lambda *a: None).start() + try: + assert len(tries) == 2 and listener.tcp.server_address[1] == listener.port + finally: + listener.stop() + + # ---------- the broker on its own ---------- def test_broker_is_an_ordinary_local_broker(): got = [] From 6d404efb4b88c39d47be4b1bc07febe6eb86f59d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 14:03:37 +0000 Subject: [PATCH 4/4] fix(install): replace the microsandbox on every start, with packages cached in its volume A sandbox started again with msb restart or msb start boots without running Lens, so the second install never answered. install.sh now writes .lens-sandbox, the command that replaces the sandbox, runs it, and starts it at boot; run.sh keeps the downloaded packages in the volume, so a fresh sandbox is up in about half a minute. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VvAGWoQxiwkGkk374pvkSh --- docs/microsandbox.md | 14 ++++++++------ install.sh | 21 +++++++++++++-------- microsandbox/run.sh | 7 +++++-- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/docs/microsandbox.md b/docs/microsandbox.md index f55760930..c763fcbb4 100644 --- a/docs/microsandbox.md +++ b/docs/microsandbox.md @@ -39,22 +39,24 @@ do: changes. 4. Runs SurrealDB, the API, a job worker and the web app, and restarts any of them that stops. -The database, the archive and the builds are in the named volume `lens-data`, which outlives the sandbox. The first -start installs and builds everything, which takes several minutes (longer on a Raspberry Pi). Later starts take -seconds. +The database, the archive, the builds and the downloaded packages are in the named volume `lens-data`, which outlives +the sandbox. A sandbox that is started again boots without running Lens, so Lens always starts in a fresh one: +`~/lens/.lens-sandbox`, which install.sh writes, replaces the sandbox. The first start downloads and builds everything, +which takes several minutes (longer on a Raspberry Pi). Later starts install the packages from the volume and reuse the +build, so they take a minute or two. The sandbox can reach the internet, your network (mail servers, model servers) and this machine. On this machine, Ollama or LM Studio are at `host.docker.internal`, the same address as with Docker. Only the web app's port and the sensor ports are published, on the same addresses Docker would use (`LENS_BIND`, `LENS_SENSOR_BIND`). -On Linux with systemd, the installer adds a `lens-microsandbox` service that starts the sandbox when the machine -starts. On a Mac, it adds a LaunchAgent that starts it at sign-in. +On Linux with systemd, the installer adds a `lens-microsandbox` service that runs `.lens-sandbox` when the machine +starts. On a Mac, it adds a LaunchAgent that runs it at sign-in. ## Day to day ```bash msb logs -f lens # follow the logs -msb stop lens # stop it; msb start lens starts it again +msb stop lens # stop it; ~/lens/.lens-sandbox starts it again msb exec lens -- sh # a shell inside it ``` diff --git a/install.sh b/install.sh index b8cde9438..e979e1200 100755 --- a/install.sh +++ b/install.sh @@ -234,12 +234,17 @@ if [ "$BACKEND" = microsandbox ]; then -v "$PWD:/lens:ro" --mount-named lens-data:/data \ --entrypoint /bin/sh python:3.12-slim-bookworm -- /lens/microsandbox/run.sh say "Starting Lens in a microsandbox (the first start installs and builds it, which takes a while)..." - if [ "$*" = "$(cat .lens-sandbox 2>/dev/null)" ] && msb ls -q 2>/dev/null | grep -qx lens; then - msb restart lens >/dev/null # same settings: keeps its installed packages - else - msb "$@" >/dev/null - printf '%s' "$*" >.lens-sandbox - fi + # The command that starts it, kept for a restart of the machine: a sandbox started again (msb start) boots without + # running Lens, so it is always replaced; its packages come from the cache in the volume, its build stays there. + { + echo "#!/bin/sh" + echo "# Starts Lens in its microsandbox (written by install.sh)" + printf 'exec %s' "'$(command -v msb)'" + for a in "$@"; do printf " '%s'" "$a"; done + echo + } >.lens-sandbox + chmod +x .lens-sandbox + ./.lens-sandbox >/dev/null LOGS="msb logs -f lens" STOP="msb stop lens" WAIT=900 # 30 minutes: the first start installs the system packages and builds everything @@ -247,7 +252,7 @@ if [ "$BACKEND" = microsandbox ]; then if [ "$os" = Linux ] && have systemctl && [ -d /run/systemd/system ]; then unit=/etc/systemd/system/lens-microsandbox.service printf '%s\n' "[Unit]" "Description=Lens (microsandbox)" "After=network-online.target" "Wants=network-online.target" "" \ - "[Service]" "Type=oneshot" "RemainAfterExit=yes" "User=$(id -un)" "ExecStart=$(command -v msb) start lens" \ + "[Service]" "Type=oneshot" "RemainAfterExit=yes" "User=$(id -un)" "ExecStart=$PWD/.lens-sandbox" \ "ExecStop=$(command -v msb) stop lens" "" "[Install]" "WantedBy=multi-user.target" | $(sudo_ok) tee "$unit" >/dev/null && $(sudo_ok) systemctl enable lens-microsandbox.service >/dev/null 2>&1 || true elif [ "$os" = Darwin ]; then @@ -258,7 +263,7 @@ if [ "$BACKEND" = microsandbox ]; then Labeldev.lens.microsandbox - ProgramArguments$(command -v msb)startlens + ProgramArguments$PWD/.lens-sandbox RunAtLoad PLIST diff --git a/microsandbox/run.sh b/microsandbox/run.sh index 5e202e86d..73c1ad81f 100755 --- a/microsandbox/run.sh +++ b/microsandbox/run.sh @@ -37,7 +37,8 @@ esac mkdir -p "$OPT/bin" "$DATA/surreal" /tmp/lens export PATH="$OPT/bin:$OPT/node/bin:$OPT/venv/bin:$PATH" -# --- System packages: in the sandbox's own disk, so installed again if the sandbox is replaced --------------------- +# --- System packages: in the sandbox's own disk, installed on each start (install.sh replaces the sandbox), from +# the packages kept in the volume, so only the first start downloads them ------------------------------------------ packages="ffmpeg tesseract-ocr antiword poppler-utils ca-certificates curl xz-utils unzip" [ "$TARGET" = full ] && packages="$packages libreoffice-writer-nogui libreoffice-calc-nogui libreoffice-impress-nogui chromium fonts-dejavu-core fonts-liberation2 fonts-noto-core fonts-noto-cjk" @@ -45,9 +46,11 @@ stamp="/var/lib/lens-packages.$TARGET" if [ ! -f "$stamp" ]; then log "Installing system packages ($TARGET)..." export DEBIAN_FRONTEND=noninteractive + rm -f /etc/apt/apt.conf.d/docker-clean # the image's setting that deletes downloaded packages + mkdir -p "$OPT/apt/partial" apt-get update -q # shellcheck disable=SC2086 - apt-get install -y -q --no-install-recommends $packages + apt-get install -y -q --no-install-recommends -o Dir::Cache::archives="$OPT/apt" $packages rm -rf /var/lib/apt/lists/* id lens >/dev/null 2>&1 || useradd --create-home --uid 1000 lens touch "$stamp"