Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions .github/workflows/install.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ on:
- install.ps1
- installer/**
- docker-compose.prod.yml
- docker-compose.images.yml
- fastapi_backend/Dockerfile
- fastapi_backend/pyproject.toml
- fastapi_backend/uv.lock
Expand All @@ -30,6 +31,7 @@ on:
- install.ps1
- installer/**
- docker-compose.prod.yml
- docker-compose.images.yml
- microsandbox/**
- packaging/host/**
- ci/scorecard/measure_install.py
Expand Down Expand Up @@ -232,6 +234,65 @@ jobs:
docker compose -f docker-compose.prod.yml down -v || true
fi

# LENS_IMAGES=pull (docs/deployment.md#prebuilt-images): a release installed from its prebuilt images, downloaded
# rather than built. The images are built here as images.yml builds them and pushed to a registry on this runner
# (LENS_IMAGES_REGISTRY), for a release this code pretends to be (v9.9.9, the branch install.sh makes for a release).
images:
name: One-line install from prebuilt images
if: github.event.action != 'closed'
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
timeout-minutes: 60
env:
LENS_DIR: /tmp/lens-images
REGISTRY: localhost:5055
steps:
- uses: actions/checkout@v7

- name: Release images in a registry on this runner
run: |
docker run -d --name lens-ci-registry -p "127.0.0.1:${REGISTRY##*:}:5000" registry:2
docker build --target lean -t "$REGISTRY/lens-backend:9.9.9-lean" fastapi_backend
docker build -f nextjs-frontend/Dockerfile.prod -t "$REGISTRY/lens-frontend:9.9.9" nextjs-frontend
docker push -q "$REGISTRY/lens-backend:9.9.9-lean"
docker push -q "$REGISTRY/lens-frontend:9.9.9"
# gone from this machine, so the install has to download them
docker image rm "$REGISTRY/lens-backend:9.9.9-lean" "$REGISTRY/lens-frontend:9.9.9"

- name: Install a release from its images
run: |
cp -r . "$LENS_DIR"
git -C "$LENS_DIR" checkout -q -b v9.9.9
LENS_KEEP_CODE=1 LENS_IMAGES=pull LENS_IMAGES_REGISTRY="$REGISTRY" LENS_TARGET=lean LENS_NO_OPEN=1 LENS_YES=1 \
LENS_WIZARD=0 LENS_UPGRADE_AGENT=0 sh "$LENS_DIR/install.sh" | tee install.log
test "${PIPESTATUS[0]}" = 0 || { echo "install.sh failed"; exit 1; }
grep -q "Downloading Lens 9.9.9" install.log || { echo "install.sh didn't download the images"; exit 1; }
! grep -q "Building and starting" install.log || { echo "install.sh built Lens instead"; exit 1; }

- name: Lens runs the downloaded images and keeps them named for later runs
run: |
cd "$LENS_DIR"
grep -qx "LENS_IMAGES=pull" .env && grep -qx "LENS_BACKEND_IMAGE=$REGISTRY/lens-backend:9.9.9-lean" .env
images="$(docker compose -f docker-compose.prod.yml ps --format '{{.Service}} {{.Image}}')"
echo "$images"
echo "$images" | grep -qx "backend $REGISTRY/lens-backend:9.9.9-lean"
echo "$images" | grep -qx "worker $REGISTRY/lens-backend:9.9.9-lean"
echo "$images" | grep -qx "frontend $REGISTRY/lens-frontend:9.9.9"
url="$(grep -Eo 'http://[^ ]+/setup\?code=[A-Za-z0-9]{12}' "$GITHUB_WORKSPACE/install.log" | head -1)"
curl -fsS -o /dev/null "$url"

- name: What the containers said
if: failure()
run: cd "$LENS_DIR" && docker compose -f docker-compose.prod.yml ps -a && docker compose -f docker-compose.prod.yml logs --tail 200

- name: Stop
if: always()
run: |
if [ -d "$LENS_DIR" ]; then
cd "$LENS_DIR"
docker compose -f docker-compose.prod.yml down -v || true
fi
docker rm -f lens-ci-registry || true

# The scorecard's install numbers (ci/scorecard): how long the one-line install takes to answer, and the RAM and disk
# Lens then takes idle, on arm64 as a Raspberry Pi is. GitHub's arm64 runner (4 cores, 16 GB, Docker already there)
# is faster than a Pi, so the time is a floor; RAM and disk are what Lens takes there.
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ in PowerShell; or skip the terminal and double-click the launcher for your syste
the menu bar (built, early: not signed yet).
[Deployment](docs/deployment.md) covers servers and Cloudron; there are also packages for
[Synology](packaging/synology/README.md), [QNAP](packaging/qnap/README.md) and [Proxmox](proxmox/README.md), and each
release publishes Lens's images for amd64 and arm64 (a Raspberry Pi) to GHCR ([prebuilt
images](docs/deployment.md#prebuilt-images)).
release publishes Lens's images for amd64 and arm64 (a Raspberry Pi) to GHCR, which `LENS_IMAGES=pull` before `sh`
downloads instead of building ([prebuilt images](docs/deployment.md#prebuilt-images)).
The [checks](docs/requirements.md#install) say which of these CI proves.

To work on the code instead, `make run` builds and runs everything from a checkout, `make setup-code` prints the
Expand Down
16 changes: 16 additions & 0 deletions docker-compose.images.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Lens's prebuilt images instead of building them (docs/deployment.md#prebuilt-images), on top of
# docker-compose.prod.yml:
#
# docker compose -f docker-compose.prod.yml -f docker-compose.images.yml pull backend frontend
# docker compose -f docker-compose.prod.yml -f docker-compose.images.yml up -d
#
# with LENS_BACKEND_IMAGE and LENS_FRONTEND_IMAGE naming a release's images in .env, as install.sh writes them with
# LENS_IMAGES=pull (ghcr.io/<owner>/lens-backend:X.Y.Z-full or -lean, ghcr.io/<owner>/lens-frontend:X.Y.Z). Without
# their build, `up --build` builds only the optional containers that have one (the MCP runner, Matterbridge).
services:
backend:
build: !reset null
worker:
build: !reset null
frontend:
build: !reset null
8 changes: 5 additions & 3 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@

# Image names of their own: under the same project, the dev stack's default names (<project>-backend, ...) would be
# overwritten by these builds, and a later `docker compose up` would run the production images without hot reload.
# LENS_VERSION tags them (default `prod`), for NAS setups (Synology, QNAP) that run a named image.
# LENS_VERSION tags them (default `prod`), for NAS setups (Synology, QNAP) that run a named image. LENS_BACKEND_IMAGE
# and LENS_FRONTEND_IMAGE name a release's prebuilt images instead, which install.sh writes into .env when it pulls them
# (LENS_IMAGES=pull, with docker-compose.images.yml; docs/deployment.md#prebuilt-images).
x-backend: &backend
security_opt: ["no-new-privileges:true"] # nothing in Lens needs setuid programs
image: lens-backend:${LENS_VERSION:-prod}
image: ${LENS_BACKEND_IMAGE:-lens-backend:${LENS_VERSION:-prod}}
build:
context: fastapi_backend
target: ${LENS_TARGET:-full} # LibreOffice and Chromium, to read Office files, text, web pages and emails; lean: without
Expand Down Expand Up @@ -149,7 +151,7 @@ services:

frontend:
security_opt: ["no-new-privileges:true"]
image: lens-frontend:${LENS_VERSION:-prod}
image: ${LENS_FRONTEND_IMAGE:-lens-frontend:${LENS_VERSION:-prod}}
build:
context: ./nextjs-frontend
dockerfile: Dockerfile.prod
Expand Down
16 changes: 16 additions & 0 deletions docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,22 @@ an arm64 server), so a machine can download them instead of building
optional extras (`EXTRAS`): to bake one in, build from source as above. The images are built in CI from the release's
tag with the repository's own token; nothing else signs or hosts them.

The install line pulls them with `LENS_IMAGES=pull` ([Get started](get-started.md#in-one-line)). By hand, from a
checkout of a release, name them in `.env` and add `docker-compose.images.yml`, which takes the build away from the
API, worker and web app (the optional containers that are built, such as the MCP runner, still build):

```bash
git checkout vX.Y.Z # a release that has docker-compose.images.yml, and the same version below
cat >> .env <<'ENV'
LENS_BACKEND_IMAGE=ghcr.io/adeelahmad/lens-backend:X.Y.Z-full
LENS_FRONTEND_IMAGE=ghcr.io/adeelahmad/lens-frontend:X.Y.Z
ENV
docker compose -f docker-compose.prod.yml -f docker-compose.images.yml pull backend frontend
docker compose -f docker-compose.prod.yml -f docker-compose.images.yml up -d
```

`LENS_IMAGES_REGISTRY` points the install line at a mirror of them instead of GHCR.

## Checklist

* **HTTPS in front of the web app**: [Remote access](remote-access.md) (a Cloudflare Tunnel Lens runs itself, set up in
Expand Down
6 changes: 6 additions & 0 deletions docs/get-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ the top of `install.sh` lists them. Whether the network reaches it and the editi
(Access & embedding, Components), with no terminal ([Updates](updates.md#setting-the-machine-up-again)). If Lens doesn't answer in time (`LENS_WAIT`, in seconds: 10 minutes with Docker,
30 in a microsandbox), the installer stops waiting and shows the last lines Lens logged and why.

To download a release's prebuilt images instead of building them (much quicker on a Raspberry Pi), put
`LENS_IMAGES=pull` before `sh`: with no `LENS_REF`, a new install then gets the newest release, and later runs and
upgrades from Settings › Updates keep pulling ([prebuilt images](deployment.md#prebuilt-images)). An install that
follows a branch such as `main`, or bakes in extras (`EXTRAS`), is still built, and so is one whose download fails.
`LENS_IMAGES=build` goes back to building. Building stays the default.

Where it can (Linux with KVM, or a Mac with Apple silicon), a first install asks whether to run Lens in Docker (the
default) or in a [microsandbox](microsandbox.md) microVM, which needs no Docker; `LENS_BACKEND=microsandbox` chooses
it without asking.
Expand Down
1 change: 1 addition & 0 deletions docs/requirements.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ request, to another check that proves the same thing; loosening a requirement or
| Lens.app on a Mac with Apple silicon installs Lens and runs it without Docker | [desktop.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/desktop.yml) `Build Lens.app`, `The bundled msb runs`, `The app carries its helper and microsandbox` |
| Packages for Synology, QNAP and Cloudron | [synology.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/synology.yml) `Build`; [qnap.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/qnap.yml) `Build the package`; [cloudron.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/cloudron.yml) `build` |
| Each release publishes Lens's images for amd64 and arm64 to GHCR | [images.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/images.yml) `Build`, `One tag for both architectures`, `Both architectures are there` |
| `LENS_IMAGES=pull` installs a release from its prebuilt images, without building | [install.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/install.yml) `Install a release from its images`, `Lens runs the downloaded images and keeps them named for later runs` |
| A Proxmox container script | No check yet ([#486](https://github.com/adeelahmad/lense/issues/486)) |
| The first admin signs up with a passkey from the setup link, which fills in the setup code | [journeys.mjs](https://github.com/adeelahmad/lense/blob/main/ci/clicks/journeys.mjs) `first-admin`; [baseline.json](https://github.com/adeelahmad/lense/blob/main/ci/clicks/baseline.json) `first-admin` ≤ 1 click; [test_passkeys.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_passkeys.py) `test_first_admin_with_a_passkey_and_no_password`; [setupForm.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/setupForm.test.tsx) `fills in the code from the setup link and starts at the email` |

Expand Down
72 changes: 69 additions & 3 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@
# LENS_WIZARD 1: answer everything in the web installer, in the browser (the default for a first install at a
# terminal, where Python 3 is there); 0: here in the terminal
# LENS_KEEP_CODE 1: build and start the code already in LENS_DIR, without getting it (the upgrade agent's rollback)
# LENS_IMAGES pull: download a release's prebuilt images (amd64 or arm64) instead of building them, and without a
# LENS_REF install the newest release; build: build them here (default: as the last run left it, else
# build). Docker only; Lens is built anyway for a branch or with EXTRAS (docs/deployment.md)
# LENS_IMAGES_REGISTRY where those images are (default: ghcr.io/<owner of LENS_REPO>), for a mirror. Kept for later
# runs
set -eu

# All in a function, so nothing runs until the whole script has arrived (curl | sh streams it).
Expand Down Expand Up @@ -169,6 +174,11 @@ case "$BACKEND" in
esac
case "${LENS_WAIT:-0}" in *[!0-9]*) fail "LENS_WAIT is a number of seconds, not ${LENS_WAIT:-}." ;; esac

# --- Prebuilt images, or built here: the one asked for, else as the last run left it, else built --------------------
IMAGES="${LENS_IMAGES:-$( [ -f "$DIR/.env" ] && grep '^LENS_IMAGES=' "$DIR/.env" | cut -d= -f2- || true)}"
IMAGES="${IMAGES:-build}"
case "$IMAGES" in build | pull) ;; *) fail "LENS_IMAGES is pull or build, not $IMAGES." ;; esac

# --- What's missing: listed, and installed once you say yes ---------------------------------------------------------
# Asked only at a terminal; piped in by a script or CI (no terminal, or LENS_YES=1) it goes ahead, as before.
export PATH="$HOME/.local/bin:$HOME/.microsandbox/bin:$PATH"
Expand Down Expand Up @@ -317,6 +327,22 @@ fi
fi

# --- The code -----------------------------------------------------------------------------------------------------
# Images are published for releases (docs/deployment.md#prebuilt-images): pulling them with no LENS_REF, a new install
# or one already on a release gets the newest release. One that follows a branch keeps it (a release may be older than
# what it runs), and builds.
latest_release() {
set -- -fsSL --max-time 20
[ -n "${GITHUB_TOKEN:-}" ] && set -- "$@" -H "Authorization: Bearer $GITHUB_TOKEN"
curl "$@" "https://api.github.com/repos/$REPO/releases/latest" 2>/dev/null |
sed -n 's/^ *"tag_name": *"\(v[0-9]*\.[0-9]*\.[0-9]*\)".*/\1/p' | head -n 1
}
if [ "$IMAGES" = pull ] && [ "$BACKEND" = docker ] && [ -z "${LENS_REF:-}" ] && [ "${LENS_KEEP_CODE:-0}" != 1 ]; then
on="$(git -C "$DIR" symbolic-ref -q --short HEAD 2>/dev/null || true)"
if [ ! -f "$DIR/.env" ] || printf '%s' "$on" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
latest="$(latest_release)"
if [ -n "$latest" ]; then REF="$latest"; else say "Couldn't find Lens's newest release; getting $REF instead."; fi
fi
fi
auth_url="https://github.com/$REPO.git"
[ -n "${GITHUB_TOKEN:-}" ] && auth_url="https://x-access-token:$GITHUB_TOKEN@github.com/$REPO.git"
if [ "${LENS_KEEP_CODE:-0}" = 1 ]; then
Expand Down Expand Up @@ -395,6 +421,11 @@ setenv() { # key value: replace the line, or add it (only this user can read the
printf '%s=%s\n' "$1" "$2" >>.env
fi
}
unsetenv() { # key: drop the line, if there is one
if grep -q "^$1=" .env; then
(umask 077 && grep -v "^$1=" .env >.env.tmp) && mv .env.tmp .env
fi
}
fresh=0 # a first install: its keys are new
if [ ! -f .env ]; then
fresh=1
Expand All @@ -419,6 +450,8 @@ if [ "$LENS_HTTPS" = 1 ]; then
setenv LENS_HTTP_PORT "$HTTP_PORT"
fi
setenv LENS_DIR "$DIR"
# kept for later runs only when asked for: an install that never chose follows the default
[ -z "${LENS_IMAGES:-}" ] || setenv LENS_IMAGES "$IMAGES"
# the SMB share's port on this machine (docs/smb.md), when the install line names one: kept for later runs
case "${LENS_SMB_PORT:-}" in '' | *[!0-9]*) ;; *) setenv LENS_SMB_PORT "$LENS_SMB_PORT" ;; esac
# the microsandbox's public resolvers (microsandbox/dns.sh), when the install line names them, empty included: kept
Expand Down Expand Up @@ -558,9 +591,42 @@ PLIST
else
files="-f docker-compose.prod.yml"
[ "$LENS_HTTPS" = 1 ] && files="$files -f docker-compose.https.yml" # the HTTPS ports (docs/https.md)
say "Building and starting Lens (the first build takes a few minutes)..."
# shellcheck disable=SC2086
$COMPOSE $files up -d --build
# A release's prebuilt images (docs/deployment.md#prebuilt-images), for the release the code is: the branch install.sh
# made for it, or the tag a first clone is on. Named in .env, so every later `docker compose` runs them too.
pulled=0
if [ "$IMAGES" = pull ]; then
on="$(git symbolic-ref -q --short HEAD 2>/dev/null || git describe --tags --exact-match 2>/dev/null || echo "$REF")"
version="$(printf '%s' "$on" | sed -n 's/^v\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\)$/\1/p')"
extras="${EXTRAS:-$(grep '^EXTRAS=' .env | cut -d= -f2- || true)}"
if [ -z "$version" ]; then
say "Prebuilt images are published for releases, and $on isn't one, so Lens is built here."
elif [ ! -f docker-compose.images.yml ]; then
say "Lens $version came before prebuilt images, so it is built here."
elif [ -n "$extras" ]; then
say "The prebuilt images carry no extras (EXTRAS=$extras), so Lens is built here."
else
registry="${LENS_IMAGES_REGISTRY:-$(grep '^LENS_IMAGES_REGISTRY=' .env | cut -d= -f2- || true)}"
registry="${registry:-ghcr.io/$(printf '%s' "${REPO%%/*}" | tr '[:upper:]' '[:lower:]')}"
[ -z "${LENS_IMAGES_REGISTRY:-}" ] || setenv LENS_IMAGES_REGISTRY "$registry"
setenv LENS_BACKEND_IMAGE "$registry/lens-backend:$version-$TARGET"
setenv LENS_FRONTEND_IMAGE "$registry/lens-frontend:$version"
say "Downloading Lens $version and starting it..."
# shellcheck disable=SC2086
if $COMPOSE $files -f docker-compose.images.yml pull backend frontend &&
$COMPOSE $files -f docker-compose.images.yml up -d --build; then
pulled=1
else
say "Couldn't download Lens's images, so it is built here instead."
fi
fi
fi
if [ "$pulled" = 0 ]; then
unsetenv LENS_BACKEND_IMAGE
unsetenv LENS_FRONTEND_IMAGE
say "Building and starting Lens (the first build takes a few minutes)..."
# shellcheck disable=SC2086
$COMPOSE $files up -d --build
fi
LOGS="$COMPOSE $files logs"
TAIL="$COMPOSE $files logs --tail 40"
STOP="$COMPOSE $files down"
Expand Down
Loading
Loading