Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions .github/workflows/install.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ on:
- nextjs-frontend/package.json
- nextjs-frontend/pnpm-lock.yaml
- packaging/host/**
- ci/scorecard/measure_install.py
- .github/workflows/install.yml
push:
branches: [main]
Expand All @@ -31,6 +32,7 @@ on:
- docker-compose.prod.yml
- microsandbox/**
- packaging/host/**
- ci/scorecard/measure_install.py
- .github/workflows/install.yml
schedule:
- cron: "17 4 * * 1"
Expand Down Expand Up @@ -230,6 +232,60 @@ jobs:
docker compose -f docker-compose.prod.yml down -v || true
fi

# The scorecard's install numbers (ci/scorecard): how long the one-line install takes to answer, and the RAM and disk
# Lens then takes idle, on arm64 as a Raspberry Pi is. GitHub's arm64 runner (4 cores, 16 GB, Docker already there)
# is faster than a Pi, so the time is a floor; RAM and disk are what Lens takes there.
arm64:
name: One-line install on arm64, timed
if: github.event.action != 'closed'
runs-on: ubuntu-24.04-arm
timeout-minutes: 90
env:
LENS_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
LENS_REF: ${{ github.event.pull_request.head.ref || github.ref_name }}
LENS_DIR: /tmp/lens-arm64
LENS_NO_OPEN: "1"
LENS_MACHINE: GitHub ubuntu-24.04-arm runner
steps:
- uses: actions/checkout@v7

- name: Install, timed until the setup page answers
run: |
date +%s > started
sh < install.sh | tee install.log
test "${PIPESTATUS[0]}" = 0 || { echo "install.sh failed"; exit 1; }
url="$(grep -Eo 'http://[^ ]+/setup\?code=[A-Za-z0-9]{12}' install.log | head -1)"
test -n "$url" || { echo "install.sh didn't hand over a setup address"; exit 1; }
curl -fsS -o /dev/null "$url"
date +%s > answered
echo "Answered after $(( $(cat answered) - $(cat started) ))s"

- name: RAM and disk, idle
run: |
# the stack settles first: workers start and the first jobs run
sleep 120
python3 ci/scorecard/measure_install.py --started "$(cat started)" --answered "$(cat answered)" \
--dir "$LENS_DIR" --out install-scorecard.json
{ echo "### Install on arm64"; echo '```json'; cat install-scorecard.json; echo '```'; } >> "$GITHUB_STEP_SUMMARY"

- uses: actions/upload-artifact@v7
with:
name: install-scorecard
path: install-scorecard.json

- name: What the containers said
if: failure()
run: cd "$LENS_DIR" && docker compose -f docker-compose.prod.yml ps -a && docker compose -f docker-compose.prod.yml logs --tail 200

- name: Stop
if: always()
run: |
sudo systemctl disable --now lens-upgrade.path 2>/dev/null || true
if [ -d "$LENS_DIR" ]; then
cd "$LENS_DIR"
docker compose -f docker-compose.prod.yml down -v || true
fi

microsandbox:
name: One-line install in a microsandbox
if: github.event.action != 'closed'
Expand Down
149 changes: 149 additions & 0 deletions ci/scorecard/measure_install.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Measure an install for the scorecard: how long it took to answer, and the RAM and disk Lens takes when idle.

The Install workflow's arm64 job runs it after the one-line install, once Lens has had a while to settle, and keeps
the JSON it writes as the install-scorecard artifact, which ci/scorecard/scorecard.py reads from main.

Usage: measure_install.py --started EPOCH --answered EPOCH --dir LENS_DIR [--out install-scorecard.json]

- install_to_first_answer_s: from the install line starting to the setup page answering.
- idle_ram_mb: the memory the install's containers use (docker stats), added up.
- disk_mb: Docker's images, containers and volumes (docker system df), plus the install folder. The build cache is
kept apart (build_cache_mb), as Docker can drop it.
"""

from __future__ import annotations

import argparse
import json
import os
import platform
import re
import subprocess
import sys
from pathlib import Path

# docker stats prints binary units (MiB); docker system df decimal ones (MB)
UNITS = {
"b": 1,
"kb": 1e3,
"mb": 1e6,
"gb": 1e9,
"tb": 1e12,
"kib": 1024,
"mib": 1024**2,
"gib": 1024**3,
"tib": 1024**4,
}


def to_mb(size: str) -> float:
"""'1.5GiB', '512MB' or '0B' in megabytes (10^6 bytes)."""
m = re.fullmatch(r"\s*([\d.]+)\s*([a-zA-Z]+)\s*", size)
if not m or m.group(2).lower() not in UNITS:
raise ValueError(f"not a size: {size!r}")
return float(m.group(1)) * UNITS[m.group(2).lower()] / 1e6


def ram_mb(stats: list[dict]) -> dict[str, float]:
"""Each container's memory from docker stats' JSON lines (MemUsage is 'used / limit')."""
return {s["Name"]: round(to_mb(s["MemUsage"].split("/")[0]), 1) for s in stats}


def disk_mb(df: list[dict]) -> dict[str, float]:
"""Docker's disk by type, from docker system df's JSON lines."""
return {d["Type"]: round(to_mb(d["Size"]), 1) for d in df}


def docker_json(*args: str) -> list[dict]:
out = subprocess.run(
["docker", *args, "--format", "{{json .}}"],
capture_output=True,
text=True,
check=True,
)
return [json.loads(line) for line in out.stdout.splitlines() if line.strip()]


def folder_mb(path: Path) -> float:
total = 0
for root, _, files in os.walk(path):
for name in files:
p = os.path.join(root, name)
if not os.path.islink(p):
total += os.lstat(p).st_size
return round(total / 1e6, 1)


def memory_gb() -> float | None:
try:
kb = re.search(
r"^MemTotal:\s+(\d+) kB", Path("/proc/meminfo").read_text(), re.MULTILINE
)
except OSError:
return None
return round(int(kb.group(1)) / 1024**2, 1) if kb else None


def target(lens_dir: Path) -> str | None:
try:
env = (lens_dir / ".env").read_text()
except OSError:
return None
m = re.search(r"^LENS_TARGET=(\S+)$", env, re.MULTILINE)
return m.group(1) if m else None


def main(argv=None) -> int:
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument(
"--started",
type=int,
required=True,
help="when the install line started (seconds since 1970)",
)
p.add_argument(
"--answered", type=int, required=True, help="when the setup page first answered"
)
p.add_argument(
"--dir", type=Path, required=True, help="the install folder (LENS_DIR)"
)
p.add_argument("--out", type=Path, default=Path("install-scorecard.json"))
args = p.parse_args(argv)

ids = subprocess.run(
["docker", "compose", "-f", "docker-compose.prod.yml", "ps", "-q"],
cwd=args.dir,
capture_output=True,
text=True,
check=True,
).stdout.split()
containers = ram_mb(docker_json("stats", "--no-stream", *ids)) if ids else {}
docker = disk_mb(docker_json("system", "df"))
folder = folder_mb(args.dir)
result = {
"install_to_first_answer_s": args.answered - args.started,
"idle_ram_mb": round(sum(containers.values()), 1),
"disk_mb": round(
sum(docker.get(k, 0) for k in ("Images", "Containers", "Local Volumes"))
+ folder,
1,
),
"build_cache_mb": docker.get("Build Cache", 0),
"containers_mb": containers,
"docker_disk_mb": docker,
"install_folder_mb": folder,
"target": target(args.dir),
"machine": os.environ.get("LENS_MACHINE") or platform.node(),
"arch": platform.machine(),
"cores": os.cpu_count(),
"memory_gb": memory_gb(),
"commit": os.environ.get("GITHUB_SHA"),
}
args.out.write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading