From 4fc0c3adb13688386c41d9894628f3ccf837b96c Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 10 Oct 2026 20:29:05 +0000 Subject: [PATCH 1/3] ci: count clicks and keystrokes per feature, with a baseline A Playwright survey (ci/clicks) drives a fresh Lens from the web installer through setup and 19 features, counting clicks, keys and passkey prompts per feature, module and page. The Clicks workflow fails when a journey takes more than baseline.json. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_018XZkebEbpvyi9y3qmDVD8V --- .github/workflows/clicks.yml | 60 ++ Makefile | 7 + ci/clicks/.gitignore | 2 + ci/clicks/README.md | 34 + ci/clicks/baseline.json | 1184 ++++++++++++++++++++++++++++++++++ ci/clicks/fake-install.sh | 8 + ci/clicks/fake-model.mjs | 59 ++ ci/clicks/journeys.mjs | 313 +++++++++ ci/clicks/meter.mjs | 127 ++++ ci/clicks/package-lock.json | 57 ++ ci/clicks/package.json | 12 + ci/clicks/report.mjs | 114 ++++ ci/clicks/run.sh | 45 ++ ci/clicks/survey.mjs | 179 +++++ 14 files changed, 2201 insertions(+) create mode 100644 .github/workflows/clicks.yml create mode 100644 ci/clicks/.gitignore create mode 100644 ci/clicks/README.md create mode 100644 ci/clicks/baseline.json create mode 100755 ci/clicks/fake-install.sh create mode 100644 ci/clicks/fake-model.mjs create mode 100644 ci/clicks/journeys.mjs create mode 100644 ci/clicks/meter.mjs create mode 100644 ci/clicks/package-lock.json create mode 100644 ci/clicks/package.json create mode 100644 ci/clicks/report.mjs create mode 100755 ci/clicks/run.sh create mode 100644 ci/clicks/survey.mjs diff --git a/.github/workflows/clicks.yml b/.github/workflows/clicks.yml new file mode 100644 index 000000000..c4fd80c28 --- /dev/null +++ b/.github/workflows/clicks.yml @@ -0,0 +1,60 @@ +name: Clicks + +# Drives a fresh Lens through setup and its main features and counts the clicks and keystrokes each takes +# (ci/clicks). A journey that takes more than ci/clicks/baseline.json says, or fails, turns this red; a change that +# takes fewer updates the baseline in the same pull request. +on: + push: + branches: [main] + pull_request: + paths: + - "nextjs-frontend/**" + - "fastapi_backend/app/**" + - "installer/**" + - "ci/clicks/**" + - ".github/workflows/clicks.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + clicks: + name: Clicks and keystrokes per feature + runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v7 + with: + cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }} + python-version: "3.12" + - uses: pnpm/action-setup@v6 + with: + package_json_file: nextjs-frontend/package.json + run_install: false + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: ${{ runner.environment == 'github-hosted' && 'pnpm' || '' }} + cache-dependency-path: nextjs-frontend/pnpm-lock.yaml + - name: Install + run: | + make env + (cd fastapi_backend && uv sync --frozen) + (cd nextjs-frontend && pnpm install --frozen-lockfile && pnpm build) + (cd ci/clicks && npm ci && npx playwright install --with-deps chromium) + - name: Survey + run: ci/clicks/run.sh --compare ci/clicks/baseline.json --fail-on-regression + - name: Summary + if: always() + run: test -f ci/clicks/results/clicks.md && sed -n '1,/^## Per page/p' ci/clicks/results/clicks.md >> "$GITHUB_STEP_SUMMARY" || true + - uses: actions/upload-artifact@v7 + if: always() + with: + name: clicks + path: ci/clicks/results/ diff --git a/Makefile b/Makefile index 24ce7c0e6..f32d7cae8 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,13 @@ docs-build: ## Build the documentation site, failing on broken links # Run it (Docker) PROD_COMPOSE=$(DOCKER_COMPOSE) -f docker-compose.prod.yml +.PHONY: clicks +clicks: env ## Count the clicks and keystrokes each journey takes on a fresh Lens (ci/clicks), compared with the baseline + cd $(BACKEND_DIR) && uv sync + cd $(FRONTEND_DIR) && pnpm install && pnpm run build + cd ci/clicks && npm ci + ci/clicks/run.sh --compare ci/clicks/baseline.json + .PHONY: run dev stop setup-code logs run: .env ## Build once and run the whole stack, no hot reload (the fast way to use it): http://localhost:3000 diff --git a/ci/clicks/.gitignore b/ci/clicks/.gitignore new file mode 100644 index 000000000..d1f24e07f --- /dev/null +++ b/ci/clicks/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +results/ diff --git a/ci/clicks/README.md b/ci/clicks/README.md new file mode 100644 index 000000000..4bb9c4364 --- /dev/null +++ b/ci/clicks/README.md @@ -0,0 +1,34 @@ +# Clicks and keystrokes per feature + +A survey of what Lens asks of people: it drives a fresh Lens in Chromium from the install wizard through setup and +each main feature, the way a person would with the defaults, and counts the clicks, keystrokes and passkey touches +each takes, per feature, per module and per page. `baseline.json` is the last accepted run; CI (the Clicks +workflow) fails a pull request whose journeys take more, so friction only goes down. + +```sh +make clicks # builds, then runs the survey and compares with the baseline +ci/clicks/run.sh --only chat-ask,note-create # some journeys (ids in journeys.mjs) +``` + +`run.sh` starts a Lens of its own on a temporary database and data folder (ports 3000, 8000, 3990 and 11434 must be +free), a stand-in OpenAI-compatible model on 11434 (`fake-model.mjs`, so the installer and the setup wizard find a +local model and the assistant answers), and the web installer with a stand-in for install.sh (`fake-install.sh`). +Results go to `results/clicks.json` and `results/clicks.md`. + +## What counts + +- **Clicks**: each mouse click. Typing into a field that doesn't have the focus adds the click that focuses it. + Choosing from a native select is two; a file chooser is two (the button, then the file). +- **Keys**: each character typed, plus each key or shortcut pressed (Enter, Ctrl+K, Ctrl+S count one). A paste is one. +- **Prompts**: passkey touches. +- A step whose control is covered by something else (the floating Ask button) is still one click, marked + "covered" in the steps. +- **Per page**: reach is clicks from Home through the sidebar (sections and tabs are two); controls are the buttons, + links and fields in the page's main area on a fresh install, not counting navigation. + +Journeys run in order on the same Lens, each from Home unless it continues the one before (setup does). + +## When a change cuts clicks + +Run the survey, check `results/clicks.md` shows the journey going down, and copy `results/clicks.json` over +`baseline.json` in the same pull request. A new journey goes at the end of `journeys.mjs` with an id that stays. diff --git a/ci/clicks/baseline.json b/ci/clicks/baseline.json new file mode 100644 index 000000000..97dcfc0d1 --- /dev/null +++ b/ci/clicks/baseline.json @@ -0,0 +1,1184 @@ +{ + "surveyed": "2026-10-10T20:28:52.714Z", + "commit": "", + "viewport": "1400x900", + "journeys": [ + { + "module": "Setup", + "feature": "Install with the web installer (defaults, local model found)", + "note": "The install line in a terminal, then the browser installer. The browser opens on its own on a desktop; on a server the link is opened by hand (counted). A computer missing Docker also asks for its password.", + "clicks": 8, + "keys": 2, + "prompts": 0, + "steps": [ + { + "page": "(outside the browser)", + "kind": "terminal", + "what": "paste the install line and press Enter", + "clicks": 0, + "keys": 2, + "prompts": 0 + }, + { + "page": "(outside the browser)", + "kind": "link", + "what": "open the installer link", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Continue (this computer)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Continue (how it runs)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "pick the model server found on this computer", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Continue (AI model)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Continue (extras)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Install Lens", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "installer:/", + "kind": "click", + "what": "Open Lens", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "install", + "ms": 1925, + "ok": true + }, + { + "module": "Setup", + "feature": "Create the first admin (passkey)", + "note": "From the setup link, which fills in the setup code.", + "clicks": 2, + "keys": 17, + "prompts": 1, + "steps": [ + { + "page": "/setup", + "kind": "type", + "what": "email", + "clicks": 1, + "keys": 17, + "prompts": 0 + }, + { + "page": "/setup", + "kind": "click", + "what": "Create admin with a passkey", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/setup", + "kind": "passkey", + "what": "create the passkey", + "clicks": 0, + "keys": 0, + "prompts": 1 + } + ], + "id": "first-admin", + "ms": 515, + "ok": true + }, + { + "module": "Setup", + "feature": "First-run setup wizard, every step with its defaults", + "note": "The namespaces come from the install's configuration, so the wizard opens on the model provider. Finish with defaults (one click) skips the remaining steps.", + "clicks": 4, + "keys": 0, + "prompts": 0, + "steps": [ + { + "page": "/welcome", + "kind": "click", + "what": "Save and continue (Connect a model provider)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/welcome", + "kind": "click", + "what": "Save and continue (Storage)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/welcome", + "kind": "click", + "what": "Save and continue (Apps and AI assistants)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/welcome", + "kind": "click", + "what": "Save and finish (Telemetry)", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "setup-wizard", + "ms": 690, + "ok": true + }, + { + "module": "Setup", + "feature": "Ask the first question from the welcome tour", + "note": "", + "clicks": 0, + "keys": 44, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "type", + "what": "question", + "clicks": 0, + "keys": 42, + "prompts": 0 + }, + { + "page": "/", + "kind": "key", + "what": "Enter to ask", + "clicks": 0, + "keys": 1, + "prompts": 0 + }, + { + "page": "/chat", + "kind": "key", + "what": "Enter again: the question waits in Chat's composer", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "first-question", + "ms": 2409, + "ok": true + }, + { + "module": "Import", + "feature": "Import a pasted transcript", + "note": "", + "clicks": 3, + "keys": 1, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Import (top bar)", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/import", + "kind": "click", + "what": "Paste tab", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/import", + "kind": "paste", + "what": "paste the transcript", + "clicks": 0, + "keys": 1, + "prompts": 0 + }, + { + "page": "/import", + "kind": "click", + "what": "Import (covered by something else on the page)", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "import-paste", + "ms": 7044, + "ok": true + }, + { + "module": "Library", + "feature": "Open a recording from the Library", + "note": "", + "clicks": 2, + "keys": 0, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Library", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/library", + "kind": "click", + "what": "the recording", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "library-open", + "ms": 2819, + "ok": true + }, + { + "module": "Search", + "feature": "Search for a word and see where it was said", + "note": "", + "clicks": 2, + "keys": 10, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Search", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/search", + "kind": "type", + "what": "the word", + "clicks": 1, + "keys": 9, + "prompts": 0 + }, + { + "page": "/search", + "kind": "key", + "what": "Enter to search", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "search-word", + "ms": 3260, + "ok": true + }, + { + "module": "Navigation", + "feature": "Jump to a recording with Ctrl+K", + "note": "", + "clicks": 0, + "keys": 9, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "key", + "what": "Ctrl+K", + "clicks": 0, + "keys": 1, + "prompts": 0 + }, + { + "page": "/", + "kind": "type", + "what": "part of its title", + "clicks": 0, + "keys": 7, + "prompts": 0 + }, + { + "page": "/", + "kind": "key", + "what": "Enter to open", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "jump-to", + "ms": 2357, + "ok": true + }, + { + "module": "Chat", + "feature": "Ask the assistant in Chat", + "note": "", + "clicks": 3, + "keys": 34, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Chat", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/chat", + "kind": "click", + "what": "New conversation", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/chat", + "kind": "type", + "what": "question", + "clicks": 1, + "keys": 33, + "prompts": 0 + }, + { + "page": "/chat", + "kind": "key", + "what": "Enter to send", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "chat-ask", + "ms": 3638, + "ok": true + }, + { + "module": "Notes", + "feature": "Write a note", + "note": "", + "clicks": 4, + "keys": 36, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Notes", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/notes", + "kind": "click", + "what": "New note", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/notes/:id", + "kind": "replace", + "what": "title", + "clicks": 1, + "keys": 9, + "prompts": 0 + }, + { + "page": "/notes/:id", + "kind": "type", + "what": "first line", + "clicks": 1, + "keys": 27, + "prompts": 0 + } + ], + "id": "note-create", + "ms": 3515, + "ok": true + }, + { + "module": "Topics", + "feature": "Add a topic", + "note": "", + "clicks": 2, + "keys": 9, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Topics", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/topics", + "kind": "click", + "what": "Add topic", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/topics", + "kind": "type", + "what": "label", + "clicks": 0, + "keys": 8, + "prompts": 0 + }, + { + "page": "/topics", + "kind": "key", + "what": "Enter to add", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "topic-add", + "ms": 3026, + "ok": true + }, + { + "module": "Pipelines", + "feature": "Make a pipeline from the standard steps", + "note": "", + "clicks": 3, + "keys": 14, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Pipelines", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/pipelines", + "kind": "click", + "what": "New pipeline", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/pipelines/new", + "kind": "type", + "what": "name", + "clicks": 0, + "keys": 14, + "prompts": 0 + }, + { + "page": "/pipelines/new", + "kind": "click", + "what": "Create pipeline", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "pipeline-create", + "ms": 2909, + "ok": true + }, + { + "module": "Routines", + "feature": "Make a nightly routine (sync sources, the default action)", + "note": "", + "clicks": 3, + "keys": 12, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Routines", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/routines", + "kind": "click", + "what": "New routine", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/routines/new", + "kind": "type", + "what": "name", + "clicks": 0, + "keys": 12, + "prompts": 0 + }, + { + "page": "/routines/new", + "kind": "click", + "what": "Create routine", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "routine-create", + "ms": 2735, + "ok": true + }, + { + "module": "Extensions", + "feature": "Add the example skill to the assistant", + "note": "", + "clicks": 3, + "keys": 0, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Extensions", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/extensions", + "kind": "click", + "what": "New", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/extensions/new", + "kind": "click", + "what": "Add to the assistant", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "extension-add", + "ms": 2637, + "ok": true + }, + { + "module": "Sensors", + "feature": "Make a webhook sensor", + "note": "", + "clicks": 3, + "keys": 0, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Sensors", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/sensors", + "kind": "click", + "what": "Add sensor", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/sensors", + "kind": "click", + "what": "Make webhook", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "sensor-webhook", + "ms": 2687, + "ok": true + }, + { + "module": "Settings", + "feature": "Change the model's timeout", + "note": "", + "clicks": 3, + "keys": 5, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Settings", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/settings/speakers", + "kind": "click", + "what": "LLM provider", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/settings/llm", + "kind": "replace", + "what": "timeout", + "clicks": 1, + "keys": 4, + "prompts": 0 + }, + { + "page": "/settings/llm", + "kind": "key", + "what": "Ctrl+S to save", + "clicks": 0, + "keys": 1, + "prompts": 0 + } + ], + "id": "settings-change", + "ms": 3646, + "ok": true + }, + { + "module": "Admin", + "feature": "Add a person as a viewer", + "note": "", + "clicks": 5, + "keys": 18, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Admin", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/admin/people", + "kind": "click", + "what": "Create account", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/admin/people", + "kind": "type", + "what": "email", + "clicks": 0, + "keys": 18, + "prompts": 0 + }, + { + "page": "/admin/people", + "kind": "click", + "what": "Create account", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/admin/people", + "kind": "click", + "what": "Copy the sign-in link", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/admin/people", + "kind": "click", + "what": "Done", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "person-add", + "ms": 3268, + "ok": true + }, + { + "module": "Account", + "feature": "Make an API token", + "note": "", + "clicks": 4, + "keys": 6, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "Account menu", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/", + "kind": "click", + "what": "API tokens", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/account/tokens", + "kind": "click", + "what": "New token", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/account/tokens", + "kind": "type", + "what": "name", + "clicks": 0, + "keys": 6, + "prompts": 0 + }, + { + "page": "/account/tokens", + "kind": "click", + "what": "Create token", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "api-token", + "ms": 2869, + "ok": true + }, + { + "module": "Navigation", + "feature": "Switch to another namespace", + "note": "", + "clicks": 2, + "keys": 0, + "prompts": 0, + "steps": [ + { + "page": "/", + "kind": "click", + "what": "the namespace switcher", + "clicks": 1, + "keys": 0, + "prompts": 0 + }, + { + "page": "/", + "kind": "click", + "what": "podcasts", + "clicks": 1, + "keys": 0, + "prompts": 0 + } + ], + "id": "namespace-switch", + "ms": 2348, + "ok": true + } + ], + "pages": [ + { + "page": "/", + "label": "Home", + "reach": 1, + "controls": 4 + }, + { + "page": "/library", + "label": "Library", + "reach": 1, + "controls": 4 + }, + { + "page": "/search", + "label": "Search", + "reach": 1, + "controls": 11 + }, + { + "page": "/chat", + "label": "Chat", + "reach": 1, + "controls": 9 + }, + { + "page": "/notes", + "label": "Notes", + "reach": 1, + "controls": 2 + }, + { + "page": "/entities", + "label": "Entities", + "reach": 1, + "controls": 10 + }, + { + "page": "/topics", + "label": "Topics", + "reach": 1, + "controls": 4 + }, + { + "page": "/graph", + "label": "Graph", + "reach": 1, + "controls": 19 + }, + { + "page": "/podcasts", + "label": "Podcasts", + "reach": 1, + "controls": 1 + }, + { + "page": "/reports", + "label": "Reports", + "reach": 1, + "controls": 4 + }, + { + "page": "/pipelines", + "label": "Pipelines", + "reach": 1, + "controls": 16 + }, + { + "page": "/workflows", + "label": "Pipelines › Workflows", + "reach": 2, + "controls": 7 + }, + { + "page": "/workflows/nodes", + "label": "Pipelines › Custom nodes", + "reach": 2, + "controls": 6 + }, + { + "page": "/content-types", + "label": "Pipelines › Content types", + "reach": 2, + "controls": 66 + }, + { + "page": "/templates", + "label": "Pipelines › Templates", + "reach": 2, + "controls": 9 + }, + { + "page": "/routines", + "label": "Routines", + "reach": 1, + "controls": 20 + }, + { + "page": "/routines/changes", + "label": "Routines › Proposed changes", + "reach": 2, + "controls": 5 + }, + { + "page": "/routines/history", + "label": "Routines › History", + "reach": 2, + "controls": 3 + }, + { + "page": "/extensions", + "label": "Extensions", + "reach": 1, + "controls": 6 + }, + { + "page": "/sensors", + "label": "Sensors", + "reach": 1, + "controls": 9 + }, + { + "page": "/sources", + "label": "Sensors › Files, email and calendars", + "reach": 2, + "controls": 5 + }, + { + "page": "/sensors/logins", + "label": "Sensors › Hub logins", + "reach": 2, + "controls": 5 + }, + { + "page": "/settings/speakers", + "label": "Settings", + "reach": 1, + "controls": 5 + }, + { + "page": "/settings/transcription", + "label": "Settings › Transcription", + "reach": 2, + "controls": 4 + }, + { + "page": "/settings/speaker-separation", + "label": "Settings › Speaker separation", + "reach": 2, + "controls": 9 + }, + { + "page": "/settings/speech-providers", + "label": "Settings › Speech providers", + "reach": 2, + "controls": 18 + }, + { + "page": "/settings/voice-ids", + "label": "Settings › Voice IDs", + "reach": 2, + "controls": 6 + }, + { + "page": "/settings/analysis", + "label": "Settings › Analysis", + "reach": 2, + "controls": 3 + }, + { + "page": "/settings/llm", + "label": "Settings › LLM provider", + "reach": 2, + "controls": 10 + }, + { + "page": "/settings/local-model", + "label": "Settings › Local model", + "reach": 2, + "controls": 25 + }, + { + "page": "/settings/ai", + "label": "Settings › AI assistant", + "reach": 2, + "controls": 48 + }, + { + "page": "/settings/search", + "label": "Settings › Search", + "reach": 2, + "controls": 17 + }, + { + "page": "/settings/reports", + "label": "Settings › Reports and graph", + "reach": 2, + "controls": 5 + }, + { + "page": "/settings/video", + "label": "Settings › Video, OCR, faces and objects", + "reach": 2, + "controls": 19 + }, + { + "page": "/settings/workers", + "label": "Settings › Workers", + "reach": 2, + "controls": 18 + }, + { + "page": "/settings/sign-in", + "label": "Settings › Sign-in", + "reach": 2, + "controls": 2 + }, + { + "page": "/settings/encryption", + "label": "Settings › Encryption", + "reach": 2, + "controls": 3 + }, + { + "page": "/settings/components", + "label": "Settings › Components", + "reach": 2, + "controls": 3 + }, + { + "page": "/settings/access", + "label": "Settings › Access & embedding", + "reach": 2, + "controls": 6 + }, + { + "page": "/settings/remote-access", + "label": "Settings › Remote access", + "reach": 2, + "controls": 4 + }, + { + "page": "/settings/mail", + "label": "Settings › Email", + "reach": 2, + "controls": 8 + }, + { + "page": "/settings/bridge", + "label": "Settings › Chat rooms", + "reach": 2, + "controls": 13 + }, + { + "page": "/settings/notifications", + "label": "Settings › Notifications", + "reach": 2, + "controls": 5 + }, + { + "page": "/settings/telemetry", + "label": "Settings › Telemetry", + "reach": 2, + "controls": 10 + }, + { + "page": "/settings/fedora", + "label": "Settings › Fedora repository", + "reach": 2, + "controls": 10 + }, + { + "page": "/settings/sensors", + "label": "Settings › Sensors", + "reach": 2, + "controls": 15 + }, + { + "page": "/settings/uploads", + "label": "Settings › Uploads", + "reach": 2, + "controls": 23 + }, + { + "page": "/settings/storage", + "label": "Settings › Storage", + "reach": 2, + "controls": 3 + }, + { + "page": "/settings/documents", + "label": "Settings › Documents", + "reach": 2, + "controls": 10 + }, + { + "page": "/settings/tokens", + "label": "Settings › API keys", + "reach": 2, + "controls": 7 + }, + { + "page": "/settings/iiif", + "label": "Settings › IIIF & metadata", + "reach": 2, + "controls": 20 + }, + { + "page": "/settings/startup", + "label": "Settings › Set at startup", + "reach": 2, + "controls": 0 + }, + { + "page": "/admin/people", + "label": "Admin", + "reach": 1, + "controls": 11 + }, + { + "page": "/admin/namespaces", + "label": "Admin › Namespaces", + "reach": 2, + "controls": 7 + }, + { + "page": "/admin/audit", + "label": "Admin › Audit log", + "reach": 2, + "controls": 28 + }, + { + "page": "/admin/health", + "label": "Admin › System health", + "reach": 2, + "controls": 10 + } + ] +} diff --git a/ci/clicks/fake-install.sh b/ci/clicks/fake-install.sh new file mode 100755 index 000000000..94840e485 --- /dev/null +++ b/ci/clicks/fake-install.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Stands in for install.sh when the survey drives the web installer: says what install.sh says, without installing, +# and points at the Lens the survey already started. +b=$(printf '\033[1m'); r=$(printf '\033[0m') +echo "${b}Checking this computer${r}" +echo "${b}Getting Lens${r}" +echo "${b}Building and starting${r}" +echo "Lens is running: ${LENS_SURVEY_SETUP_URL:-http://localhost:3000/setup}" diff --git a/ci/clicks/fake-model.mjs b/ci/clicks/fake-model.mjs new file mode 100644 index 000000000..cd28af9ca --- /dev/null +++ b/ci/clicks/fake-model.mjs @@ -0,0 +1,59 @@ +// A stand-in OpenAI-compatible model server, so the journeys that need a model (the install wizard finding one, the +// setup wizard testing one, the assistant answering) run without a real one. Every answer is the same short reply. +import http from "node:http"; + +const REPLY = "This is the survey's stand-in model answering."; +const MODELS = ["survey-chat", "survey-embed"]; + +export function startFakeModel(port) { + const server = http.createServer((req, res) => { + let body = ""; + req.on("data", (c) => (body += c)); + req.on("end", () => { + const url = req.url.replace(/^\/v1/, ""); + const json = (code, data) => { + res.writeHead(code, { "Content-Type": "application/json" }); + res.end(JSON.stringify(data)); + }; + if (req.method === "GET" && url.startsWith("/models")) + return json(200, { object: "list", data: MODELS.map((id) => ({ id, object: "model" })) }); + if (req.method === "GET" && url.startsWith("/api/tags")) return json(200, { models: MODELS.map((name) => ({ name })) }); + let payload = {}; + try { + payload = JSON.parse(body || "{}"); + } catch { + /* an empty answer below */ + } + if (url.startsWith("/embeddings")) { + const inputs = Array.isArray(payload.input) ? payload.input : [payload.input ?? ""]; + return json(200, { + object: "list", + model: "survey-embed", + data: inputs.map((_, index) => ({ object: "embedding", index, embedding: Array.from({ length: 8 }, (_, i) => (i + 1) / 8) })), + }); + } + if (url.startsWith("/chat/completions")) { + const model = payload.model || MODELS[0]; + if (payload.stream) { + res.writeHead(200, { "Content-Type": "text/event-stream" }); + const chunk = (delta, finish = null) => + `data: ${JSON.stringify({ id: "survey", object: "chat.completion.chunk", model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`; + res.write(chunk({ role: "assistant", content: REPLY })); + res.write(chunk({}, "stop")); + return res.end("data: [DONE]\n\n"); + } + // structured output: an empty object satisfies most schemas' optional fields + const content = payload.response_format ? "{}" : REPLY; + return json(200, { + id: "survey", + object: "chat.completion", + model, + choices: [{ index: 0, message: { role: "assistant", content }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + } + json(404, { error: "not found" }); + }); + }); + return new Promise((resolve) => server.listen(port, "127.0.0.1", () => resolve(server))); +} diff --git a/ci/clicks/journeys.mjs b/ci/clicks/journeys.mjs new file mode 100644 index 000000000..cd8a78bd5 --- /dev/null +++ b/ci/clicks/journeys.mjs @@ -0,0 +1,313 @@ +// What the survey does, in order: each journey is one feature, done the way a person would with the defaults, from +// where they'd usually start. They run one after another on the same fresh Lens, so later ones can use what earlier +// ones made (the admin account, an imported transcript). Keep ids stable: the baseline compares by id. + +const INSTALLER = "http://localhost:3990"; + +/** A page in the primary navigation (the sidebar). */ +const nav = (page, label) => + page + .getByRole("navigation", { name: "Primary" }) + .getByRole("link", { name: new RegExp(`^${label}( \\d+)?$`) }) // a count can follow the label (Library 1) + .first(); + +/** The model-backed answer the stand-in model gives (fake-model.mjs). */ +const ANSWER = /stand-in model answering/; + +const TRANSCRIPT = [ + "Host A: Welcome back. Today we talk about local models on small computers.", + "Host B: A Raspberry Pi can run Lens when the models are in the cloud.", + "Host A: And everything stays on your own hardware.", +].join("\n"); + +export const JOURNEYS = [ + // --- Setup ------------------------------------------------------------------------------------------------------- + { + id: "install", + module: "Setup", + feature: "Install with the web installer (defaults, local model found)", + note: "The install line in a terminal, then the browser installer. The browser opens on its own on a desktop; on a server the link is opened by hand (counted). A computer missing Docker also asks for its password.", + needs: "installer", + async run({ page, meter, base }) { + meter.record("terminal", "paste the install line and press Enter", 0, 2); + meter.openLink("open the installer link"); + await page.goto(`${INSTALLER}/?t=survey-token`); + await meter.click(page.locator("#next"), "Continue (this computer)"); + await page.getByRole("heading", { name: "How Lens runs" }).waitFor(); + await meter.click(page.locator("#next"), "Continue (how it runs)"); + const found = page.getByRole("button", { name: /Ollama · \d+ models/ }); + await found.waitFor(); + await meter.click(found, "pick the model server found on this computer"); + await meter.click(page.locator("#next"), "Continue (AI model)"); + await page.getByRole("heading", { name: "Extras" }).waitFor(); + await meter.click(page.locator("#next"), "Continue (extras)"); + await page.getByRole("heading", { name: "Ready to install" }).waitFor(); + await meter.click(page.locator("#next"), "Install Lens"); + const open = page.getByRole("button", { name: "Open Lens" }); + await open.waitFor({ timeout: 60000 }); + await meter.click(open, "Open Lens"); + await page.waitForURL(`${base}/setup**`); + }, + }, + { + id: "first-admin", + module: "Setup", + feature: "Create the first admin (passkey)", + note: "From the setup link, which fills in the setup code.", + async run({ page, meter, base, code, installer }) { + if (!installer) { + meter.openLink("open the setup link the server printed"); + await page.goto(`${base}/setup?code=${code}`); + } + await page.getByRole("heading", { name: "Set up this server" }).waitFor(); + await meter.type(page.getByRole("textbox", { name: "Email" }), "admin@example.com", "email"); + await meter.click(page.getByRole("button", { name: /Create admin/ }), "Create admin with a passkey"); + meter.passkey("create the passkey"); + await page.waitForURL(`${base}/welcome`); + }, + }, + { + id: "setup-wizard", + module: "Setup", + feature: "First-run setup wizard, every step with its defaults", + note: "The namespaces come from the install's configuration, so the wizard opens on the model provider. Finish with defaults (one click) skips the remaining steps.", + async run({ page, meter, base }) { + await page.getByRole("heading", { name: "Connect a model provider" }).waitFor(); + await page.getByRole("button", { name: /Ollama 2 models/ }).waitFor(); + for (const [heading, button] of [ + ["Connect a model provider", "Save and continue"], + ["Storage", "Save and continue"], + ["Apps and AI assistants", "Save and continue"], + ["Telemetry", "Save and finish"], + ]) { + await page.getByRole("heading", { name: heading, exact: true }).waitFor(); + await meter.click(page.getByRole("button", { name: button }), `${button} (${heading})`); + } + await page.waitForURL(`${base}/`); + }, + }, + { + id: "first-question", + module: "Setup", + feature: "Ask the first question from the welcome tour", + async run({ page, meter }) { + const dialog = page.getByRole("dialog", { name: "Welcome to Lens" }); + await dialog.waitFor(); + await meter.type(dialog.getByRole("textbox", { name: "Ask the assistant" }), "What did the hosts say about Raspberry Pi?", "question"); + await meter.press("Enter", "Enter to ask"); + // The tour opens Chat with the question in the composer, not sent + const composer = page.getByRole("textbox", { name: "Your question" }); + await composer.waitFor(); + if (!(await page.getByText(ANSWER).first().isVisible().catch(() => false))) { + await page.waitForTimeout(1500); + if ((await composer.inputValue()).trim()) await meter.press("Enter", "Enter again: the question waits in Chat's composer"); + } + await page.getByText(ANSWER).first().waitFor({ timeout: 60000 }); + }, + }, + + // --- Everyday use: each starts on Home, as after signing in ----------------------------------------------------- + { + id: "import-paste", + module: "Import", + feature: "Import a pasted transcript", + from: "/", + async run({ page, meter }) { + await meter.click(page.getByRole("banner").getByRole("link", { name: "Import" }), "Import (top bar)"); + await meter.click(page.getByRole("tab", { name: "Paste" }), "Paste tab"); + await meter.paste(page.getByRole("textbox", { name: "Paste a transcript" }), TRANSCRIPT, "paste the transcript"); + await page.getByText("3 turns").waitFor(); + await meter.click(page.getByRole("main").getByRole("button", { name: "Import", exact: true }), "Import"); + await page.getByText(/import sent/).waitFor({ timeout: 30000 }); + }, + }, + { + id: "library-open", + module: "Library", + feature: "Open a recording from the Library", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Library"), "Library"); + const item = page.getByRole("main").getByRole("link", { name: /Welcome back/ }).first(); + await item.waitFor(); + await meter.click(item, "the recording"); + await page.waitForURL(/\/(recordings|resources)\//); + }, + }, + { + id: "search-word", + module: "Search", + feature: "Search for a word and see where it was said", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Search"), "Search"); + await meter.type(page.getByRole("searchbox", { name: "Search transcripts" }), "raspberry", "the word"); + await meter.press("Enter", "Enter to search"); + await page.getByRole("main").getByText(/Raspberry Pi can run Lens/).first().waitFor(); + }, + }, + { + id: "jump-to", + module: "Navigation", + feature: "Jump to a recording with Ctrl+K", + from: "/", + async run({ page, meter }) { + await meter.press("Control+k", "Ctrl+K"); + await meter.type(page.getByRole("combobox", { name: /Search recordings and pages/ }), "welcome", "part of its title"); + await page.getByRole("option", { name: /Welcome back/ }).first().waitFor(); + await meter.press("Enter", "Enter to open"); + await page.waitForURL(/\/(recordings|resources)\//); + }, + }, + { + id: "chat-ask", + module: "Chat", + feature: "Ask the assistant in Chat", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Chat"), "Chat"); + await meter.click(page.getByRole("button", { name: "New conversation" }), "New conversation"); + await meter.type(page.getByRole("textbox", { name: "Your question" }), "Who said what about Raspberry Pi?", "question"); + await meter.press("Enter", "Enter to send"); + await page.getByText(ANSWER).first().waitFor({ timeout: 60000 }); + }, + }, + { + id: "note-create", + module: "Notes", + feature: "Write a note", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Notes"), "Notes"); + await meter.click(page.getByRole("main").getByRole("button", { name: "New note" }).first(), "New note"); + await page.waitForURL(/\/notes\/\d+/); + const title = page.getByRole("textbox", { name: "Title" }); + await meter.replace(title, "Pi setup", "title"); + const body = page.locator("main affine-page-root [contenteditable=true]").first(); + const editor = (await body.count()) ? body : page.locator("main .inline-editor").last(); + await meter.type(editor, "Use cloud models on the Pi.", "first line"); + await page.getByText("Saved").first().waitFor(); + }, + }, + { + id: "topic-add", + module: "Topics", + feature: "Add a topic", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Topics"), "Topics"); + await meter.click(page.getByRole("main").getByRole("button", { name: "Add topic" }).first(), "Add topic"); + await meter.type(page.getByRole("textbox", { name: "Label", exact: true }), "Hardware", "label"); + await meter.press("Enter", "Enter to add"); + await page.getByRole("main").getByText("Hardware").first().waitFor(); + }, + }, + { + id: "pipeline-create", + module: "Pipelines", + feature: "Make a pipeline from the standard steps", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Pipelines"), "Pipelines"); + await meter.click(page.getByRole("main").getByRole("link", { name: "New pipeline" }), "New pipeline"); + await meter.type(page.getByRole("textbox", { name: "Pipeline name" }), "Quick pipeline", "name"); + await meter.click(page.getByRole("button", { name: "Create pipeline" }), "Create pipeline"); + await page.waitForURL(/\/pipelines\/\d+/); + }, + }, + { + id: "routine-create", + module: "Routines", + feature: "Make a nightly routine (sync sources, the default action)", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Routines"), "Routines"); + await meter.click(page.getByRole("main").getByRole("link", { name: "New routine" }), "New routine"); + await meter.type(page.getByRole("textbox", { name: "Name" }), "Nightly sync", "name"); + await meter.click(page.getByRole("button", { name: "Create routine" }), "Create routine"); + await page.waitForURL(/\/routines\/\d+/); + }, + }, + { + id: "extension-add", + module: "Extensions", + feature: "Add the example skill to the assistant", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Extensions"), "Extensions"); + await meter.click(page.getByRole("main").getByRole("link", { name: "New", exact: true }), "New"); + await meter.click(page.getByRole("button", { name: "Add to the assistant" }), "Add to the assistant"); + await page.waitForURL(/\/extensions\/\d+/); + }, + }, + { + id: "sensor-webhook", + module: "Sensors", + feature: "Make a webhook sensor", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Sensors"), "Sensors"); + await meter.click(page.getByRole("main").getByRole("button", { name: "Add sensor" }).first(), "Add sensor"); + await meter.click(page.getByRole("dialog").getByRole("button", { name: "Make webhook" }), "Make webhook"); + await page.getByText(/token/i).first().waitFor(); + }, + }, + { + id: "settings-change", + module: "Settings", + feature: "Change the model's timeout", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Settings"), "Settings"); + await meter.click(page.getByRole("link", { name: "LLM provider" }), "LLM provider"); + await meter.replace(page.getByRole("textbox", { name: "Timeout (seconds)" }), "120", "timeout"); + await meter.press("Control+s", "Ctrl+S to save"); + await page.getByText(/Saved|Last saved/).first().waitFor(); + }, + }, + { + id: "person-add", + module: "Admin", + feature: "Add a person as a viewer", + from: "/", + async run({ page, meter }) { + await meter.click(nav(page, "Admin"), "Admin"); + await meter.click(page.getByRole("button", { name: "Create account" }), "Create account"); + const d = page.getByRole("dialog"); + await meter.type(d.getByRole("textbox", { name: "Email" }), "viewer@example.com", "email"); + await meter.click(d.getByRole("button", { name: "Create account" }), "Create account"); + // The sign-in link is shown once, to send to them + await meter.click(d.getByRole("button", { name: /Copy/ }), "Copy the sign-in link"); + await meter.click(d.getByRole("button", { name: "Done" }), "Done"); + await page.getByRole("main").getByText("viewer@example.com").first().waitFor(); + }, + }, + { + id: "api-token", + module: "Account", + feature: "Make an API token", + from: "/", + async run({ page, meter }) { + await meter.click(page.getByRole("button", { name: "Account menu" }), "Account menu"); + await meter.click(page.getByRole("menuitem", { name: /API tokens/ }), "API tokens"); + await meter.click(page.getByRole("main").getByRole("button", { name: "New token" }).first(), "New token"); + const d = page.getByRole("dialog"); + await meter.type(d.getByRole("textbox", { name: "Name" }), "script", "name"); + await meter.click(d.getByRole("button", { name: "Create token" }), "Create token"); + await page.getByText(/copy/i).first().waitFor(); + }, + }, + { + id: "namespace-switch", + module: "Navigation", + feature: "Switch to another namespace", + from: "/", + async run({ page, meter }) { + await meter.click(page.getByRole("button", { name: /^Namespace: / }), "the namespace switcher"); + await meter.click(page.getByRole("option", { name: /^podcasts/ }), "podcasts"); + await page.getByRole("button", { name: /^Namespace: podcasts/ }).waitFor(); + }, + }, +]; + +export { TRANSCRIPT, ANSWER }; diff --git a/ci/clicks/meter.mjs b/ci/clicks/meter.mjs new file mode 100644 index 000000000..2af7d9be2 --- /dev/null +++ b/ci/clicks/meter.mjs @@ -0,0 +1,127 @@ +// Counts what a person does in the browser: clicks, keystrokes and passkey prompts, attributed to the page they're on. +// +// The rules, so runs compare: +// - click: one mouse click. Choosing from a native