From 1f92f6e3c6d2b0631922e24affeb09ec082b0bda Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 02:36:01 +0000 Subject: [PATCH] ci(install): time the one-line install on arm64 for the scorecard A new Install workflow job installs Lens on GitHub's arm64 runner, times the install line until the setup page answers, then lets the stack settle and measures its containers' memory and its disk (ci/scorecard/measure_install.py), kept as the install-scorecard artifact. make scorecard reads the newest one from main for install_to_first_answer_s, idle_ram_mb and disk_mb, which said "not measured" until now. Part of #481. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01SDzZtD95f3XeGHNfinM1qv --- .github/workflows/install.yml | 56 ++++++++++++ ci/scorecard/measure_install.py | 149 ++++++++++++++++++++++++++++++++ ci/scorecard/scorecard.py | 124 +++++++++++++++++++------- ci/scorecard/test_scorecard.py | 95 ++++++++++++++++++++ docs/contributing.md | 2 +- 5 files changed, 394 insertions(+), 32 deletions(-) create mode 100755 ci/scorecard/measure_install.py diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index e81be34c8..7cd68e210 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -21,6 +21,7 @@ on: - nextjs-frontend/package.json - nextjs-frontend/pnpm-lock.yaml - packaging/host/** + - ci/scorecard/measure_install.py - .github/workflows/install.yml push: branches: [main] @@ -31,6 +32,7 @@ on: - docker-compose.prod.yml - microsandbox/** - packaging/host/** + - ci/scorecard/measure_install.py - .github/workflows/install.yml schedule: - cron: "17 4 * * 1" @@ -230,6 +232,60 @@ jobs: docker compose -f docker-compose.prod.yml down -v || true fi + # The scorecard's install numbers (ci/scorecard): how long the one-line install takes to answer, and the RAM and disk + # Lens then takes idle, on arm64 as a Raspberry Pi is. GitHub's arm64 runner (4 cores, 16 GB, Docker already there) + # is faster than a Pi, so the time is a floor; RAM and disk are what Lens takes there. + arm64: + name: One-line install on arm64, timed + if: github.event.action != 'closed' + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + env: + LENS_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + LENS_REF: ${{ github.event.pull_request.head.ref || github.ref_name }} + LENS_DIR: /tmp/lens-arm64 + LENS_NO_OPEN: "1" + LENS_MACHINE: GitHub ubuntu-24.04-arm runner + steps: + - uses: actions/checkout@v7 + + - name: Install, timed until the setup page answers + run: | + date +%s > started + sh < install.sh | tee install.log + test "${PIPESTATUS[0]}" = 0 || { echo "install.sh failed"; exit 1; } + url="$(grep -Eo 'http://[^ ]+/setup\?code=[A-Za-z0-9]{12}' install.log | head -1)" + test -n "$url" || { echo "install.sh didn't hand over a setup address"; exit 1; } + curl -fsS -o /dev/null "$url" + date +%s > answered + echo "Answered after $(( $(cat answered) - $(cat started) ))s" + + - name: RAM and disk, idle + run: | + # the stack settles first: workers start and the first jobs run + sleep 120 + python3 ci/scorecard/measure_install.py --started "$(cat started)" --answered "$(cat answered)" \ + --dir "$LENS_DIR" --out install-scorecard.json + { echo "### Install on arm64"; echo '```json'; cat install-scorecard.json; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v7 + with: + name: install-scorecard + path: install-scorecard.json + + - name: What the containers said + if: failure() + run: cd "$LENS_DIR" && docker compose -f docker-compose.prod.yml ps -a && docker compose -f docker-compose.prod.yml logs --tail 200 + + - name: Stop + if: always() + run: | + sudo systemctl disable --now lens-upgrade.path 2>/dev/null || true + if [ -d "$LENS_DIR" ]; then + cd "$LENS_DIR" + docker compose -f docker-compose.prod.yml down -v || true + fi + microsandbox: name: One-line install in a microsandbox if: github.event.action != 'closed' diff --git a/ci/scorecard/measure_install.py b/ci/scorecard/measure_install.py new file mode 100755 index 000000000..b3ae4a564 --- /dev/null +++ b/ci/scorecard/measure_install.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Measure an install for the scorecard: how long it took to answer, and the RAM and disk Lens takes when idle. + +The Install workflow's arm64 job runs it after the one-line install, once Lens has had a while to settle, and keeps +the JSON it writes as the install-scorecard artifact, which ci/scorecard/scorecard.py reads from main. + +Usage: measure_install.py --started EPOCH --answered EPOCH --dir LENS_DIR [--out install-scorecard.json] + +- install_to_first_answer_s: from the install line starting to the setup page answering. +- idle_ram_mb: the memory the install's containers use (docker stats), added up. +- disk_mb: Docker's images, containers and volumes (docker system df), plus the install folder. The build cache is + kept apart (build_cache_mb), as Docker can drop it. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import subprocess +import sys +from pathlib import Path + +# docker stats prints binary units (MiB); docker system df decimal ones (MB) +UNITS = { + "b": 1, + "kb": 1e3, + "mb": 1e6, + "gb": 1e9, + "tb": 1e12, + "kib": 1024, + "mib": 1024**2, + "gib": 1024**3, + "tib": 1024**4, +} + + +def to_mb(size: str) -> float: + """'1.5GiB', '512MB' or '0B' in megabytes (10^6 bytes).""" + m = re.fullmatch(r"\s*([\d.]+)\s*([a-zA-Z]+)\s*", size) + if not m or m.group(2).lower() not in UNITS: + raise ValueError(f"not a size: {size!r}") + return float(m.group(1)) * UNITS[m.group(2).lower()] / 1e6 + + +def ram_mb(stats: list[dict]) -> dict[str, float]: + """Each container's memory from docker stats' JSON lines (MemUsage is 'used / limit').""" + return {s["Name"]: round(to_mb(s["MemUsage"].split("/")[0]), 1) for s in stats} + + +def disk_mb(df: list[dict]) -> dict[str, float]: + """Docker's disk by type, from docker system df's JSON lines.""" + return {d["Type"]: round(to_mb(d["Size"]), 1) for d in df} + + +def docker_json(*args: str) -> list[dict]: + out = subprocess.run( + ["docker", *args, "--format", "{{json .}}"], + capture_output=True, + text=True, + check=True, + ) + return [json.loads(line) for line in out.stdout.splitlines() if line.strip()] + + +def folder_mb(path: Path) -> float: + total = 0 + for root, _, files in os.walk(path): + for name in files: + p = os.path.join(root, name) + if not os.path.islink(p): + total += os.lstat(p).st_size + return round(total / 1e6, 1) + + +def memory_gb() -> float | None: + try: + kb = re.search( + r"^MemTotal:\s+(\d+) kB", Path("/proc/meminfo").read_text(), re.MULTILINE + ) + except OSError: + return None + return round(int(kb.group(1)) / 1024**2, 1) if kb else None + + +def target(lens_dir: Path) -> str | None: + try: + env = (lens_dir / ".env").read_text() + except OSError: + return None + m = re.search(r"^LENS_TARGET=(\S+)$", env, re.MULTILINE) + return m.group(1) if m else None + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument( + "--started", + type=int, + required=True, + help="when the install line started (seconds since 1970)", + ) + p.add_argument( + "--answered", type=int, required=True, help="when the setup page first answered" + ) + p.add_argument( + "--dir", type=Path, required=True, help="the install folder (LENS_DIR)" + ) + p.add_argument("--out", type=Path, default=Path("install-scorecard.json")) + args = p.parse_args(argv) + + ids = subprocess.run( + ["docker", "compose", "-f", "docker-compose.prod.yml", "ps", "-q"], + cwd=args.dir, + capture_output=True, + text=True, + check=True, + ).stdout.split() + containers = ram_mb(docker_json("stats", "--no-stream", *ids)) if ids else {} + docker = disk_mb(docker_json("system", "df")) + folder = folder_mb(args.dir) + result = { + "install_to_first_answer_s": args.answered - args.started, + "idle_ram_mb": round(sum(containers.values()), 1), + "disk_mb": round( + sum(docker.get(k, 0) for k in ("Images", "Containers", "Local Volumes")) + + folder, + 1, + ), + "build_cache_mb": docker.get("Build Cache", 0), + "containers_mb": containers, + "docker_disk_mb": docker, + "install_folder_mb": folder, + "target": target(args.dir), + "machine": os.environ.get("LENS_MACHINE") or platform.node(), + "arch": platform.machine(), + "cores": os.cpu_count(), + "memory_gb": memory_gb(), + "commit": os.environ.get("GITHUB_SHA"), + } + args.out.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/scorecard/scorecard.py b/ci/scorecard/scorecard.py index 7ec2e026c..a2fa0d915 100755 --- a/ci/scorecard/scorecard.py +++ b/ci/scorecard/scorecard.py @@ -13,7 +13,8 @@ - CI: the share of pull requests whose first push had a failing check, failed runs on main, the median time from ask to merge and the median workflow minutes per merged pull request, over the last --days. - Outside interest: stars, forks, and issues and pull requests from people who aren't collaborators. -- Install to first answer, and idle RAM and disk on arm64 (as on a Raspberry Pi): not measured yet. +- Install: the time from the install line to Lens answering, and the RAM and disk it takes idle, on arm64 (as a + Raspberry Pi is), from the Install workflow's timed install on main. Usage: ci/scorecard/scorecard.py print the line @@ -464,24 +465,44 @@ def newest_from_main(gh: GitHub, name: str) -> dict | None: return max(found, key=lambda a: a.get("created_at") or "") if found else None +class Unavailable(Exception): + """Why a number can't be measured.""" + + +def read_artifact(gh: GitHub, name: str, member: str) -> tuple[str, dict]: + """A file from the newest artifact of that name from main, and the artifact.""" + artifact = newest_from_main(gh, name) + if artifact is None: + raise Unavailable(f"no {name} artifact from main (CI keeps them for 90 days)") + if not gh.token: + raise Unavailable( + "downloading CI artifacts needs a GitHub token (GITHUB_TOKEN)" + ) + try: + data = gh.request(artifact["archive_download_url"], accept="*/*") + text = zipfile.ZipFile(io.BytesIO(data)).read(member).decode("utf-8", "replace") + except (urllib.error.URLError, KeyError, zipfile.BadZipFile) as e: + raise Unavailable(f"couldn't read the {name} artifact: {e}") from e + return text, artifact + + +def made_at(artifact: dict) -> dict: + run = artifact.get("workflow_run") or {} + return { + "commit": (run.get("head_sha") or "")[:12], + "created": artifact.get("created_at"), + } + + def coverage(gh: GitHub, name: str, member: str, parse) -> dict: cmd = f"GET /repos/{gh.repo}/actions/artifacts?name={name}, the newest from main, {member}" try: - artifact = newest_from_main(gh, name) - if artifact is None: - return not_measured( - f"no {name} artifact from main (CI keeps them for a while)", cmd - ) - if not gh.token: - return not_measured( - "downloading CI artifacts needs a GitHub token (GITHUB_TOKEN)", cmd - ) - z = zipfile.ZipFile( - io.BytesIO(gh.request(artifact["archive_download_url"], accept="*/*")) - ) - covered, lines = parse(z.read(member).decode("utf-8", errors="replace")) - except (urllib.error.URLError, KeyError, ValueError, zipfile.BadZipFile) as e: - return not_measured(f"couldn't read the {name} artifact: {e}", cmd) + text, artifact = read_artifact(gh, name, member) + covered, lines = parse(text) + except Unavailable as e: + return not_measured(str(e), cmd) + except ValueError as e: + return not_measured(f"couldn't read {member}: {e}", cmd) return measured( round(100 * covered / lines, 2), "% of lines", @@ -489,11 +510,62 @@ def coverage(gh: GitHub, name: str, member: str, parse) -> dict: better="up", lines=lines, covered=covered, - commit=(artifact.get("workflow_run") or {}).get("head_sha", "")[:12], - created=artifact.get("created_at"), + **made_at(artifact), ) +INSTALL = ["install_to_first_answer_s", "idle_ram_mb", "disk_mb"] + + +def install_metrics(gh: GitHub) -> dict: + """From the Install workflow's timed one-line install on arm64 (ci/scorecard/measure_install.py).""" + cmd = ( + f"GET /repos/{gh.repo}/actions/artifacts?name=install-scorecard, the newest from main: the Install " + "workflow's arm64 job, ci/scorecard/measure_install.py" + ) + try: + text, artifact = read_artifact( + gh, "install-scorecard", "install-scorecard.json" + ) + data = json.loads(text) + on = { + k: data.get(k) for k in ("machine", "arch", "cores", "memory_gb", "target") + } + on.update(made_at(artifact)) + return { + "install_to_first_answer_s": measured( + data["install_to_first_answer_s"], + "seconds", + cmd + " (from the install line to the setup page answering)", + better="down", + **on, + ), + "idle_ram_mb": measured( + data["idle_ram_mb"], + "MB", + cmd + " (docker stats of its containers, idle)", + better="down", + containers=data.get("containers_mb"), + **on, + ), + "disk_mb": measured( + data["disk_mb"], + "MB", + cmd + " (images, containers and volumes, and the install folder)", + better="down", + build_cache_mb=data.get("build_cache_mb"), + **on, + ), + } + except Unavailable as e: + return {k: not_measured(str(e), cmd) for k in INSTALL} + except (ValueError, KeyError) as e: + return { + k: not_measured(f"couldn't read install-scorecard.json: {e}", cmd) + for k in INSTALL + } + + def community(gh: GitHub) -> dict: repo = gh.get(f"/repos/{gh.repo}") out = { @@ -540,6 +612,7 @@ def github_metrics(gh: GitHub, since: datetime, until: datetime) -> dict: ) }, lambda: community(gh), + lambda: install_metrics(gh), ] names = [ ["red_check_rate", "ask_to_merge_hours", "ci_minutes_per_merged_pr"], @@ -553,6 +626,7 @@ def github_metrics(gh: GitHub, since: datetime, until: datetime) -> dict: "outside_pull_requests", "open_user_reported_issues", ], + INSTALL, ] out = {} for part, keys in zip(parts, names): @@ -576,22 +650,11 @@ def offline_github(why: str) -> dict: "outside_issues", "outside_pull_requests", "open_user_reported_issues", + *INSTALL, ] return {k: not_measured(why) for k in keys} -# --- Install and the Pi --- - - -def install_metrics() -> dict: - why = "no timed install on an arm64 runner yet (#481)" - return { - "install_to_first_answer_s": not_measured(why), - "idle_ram_mb": not_measured(why), - "disk_mb": not_measured(why), - } - - # --- The line and the comparison --- @@ -615,7 +678,6 @@ def scorecard(args, now: datetime | None = None) -> dict: else: gh = GitHub(repo, os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")) metrics.update(github_metrics(gh, since, now)) - metrics.update(install_metrics()) return { "at": iso(now), "repo": repo, diff --git a/ci/scorecard/test_scorecard.py b/ci/scorecard/test_scorecard.py index 64f11af2d..a08e3ee97 100644 --- a/ci/scorecard/test_scorecard.py +++ b/ci/scorecard/test_scorecard.py @@ -6,11 +6,13 @@ import json import tempfile import unittest +import zipfile from contextlib import redirect_stderr, redirect_stdout from datetime import datetime, timezone from pathlib import Path from unittest import mock +import measure_install as mi import scorecard as sc @@ -300,3 +302,96 @@ def test_offline_line_then_log_compares_and_appends(self): if __name__ == "__main__": unittest.main() + + +class FakeGitHub(sc.GitHub): + """Answers the artifacts list and downloads from memory.""" + + def __init__(self, artifacts: dict[str, dict[str, str]], token="t"): + super().__init__("o/r", token) + self.artifacts = artifacts + + def pages(self, path, params, key=None): + name = params.get("name") + if name not in self.artifacts: + return iter([]) + run = {"head_branch": "main", "head_sha": "abcdef1234567890"} + return iter( + [ + { + "expired": False, + "created_at": "t", + "workflow_run": run, + "archive_download_url": name, + }, + ] + ) + + def request(self, url, accept="application/vnd.github+json"): + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + for member, text in self.artifacts[url].items(): + z.writestr(member, text) + return buf.getvalue() + + +class Install(unittest.TestCase): + def test_reads_the_timed_install_from_main(self): + made = { + "install_to_first_answer_s": 812, + "idle_ram_mb": 1450.3, + "disk_mb": 6120.0, + "build_cache_mb": 900.0, + "containers_mb": {"backend": 700.0}, + "machine": "GitHub ubuntu-24.04-arm", + "arch": "aarch64", + "target": "full", + } + gh = FakeGitHub( + {"install-scorecard": {"install-scorecard.json": json.dumps(made)}} + ) + m = sc.install_metrics(gh) + self.assertEqual(m["install_to_first_answer_s"]["value"], 812) + self.assertEqual(m["idle_ram_mb"]["containers"], {"backend": 700.0}) + self.assertEqual(m["disk_mb"]["build_cache_mb"], 900.0) + self.assertEqual(m["disk_mb"]["arch"], "aarch64") + self.assertEqual(m["disk_mb"]["commit"], "abcdef123456") + self.assertEqual(m["disk_mb"]["better"], "down") + + def test_not_measured_without_an_artifact_or_a_token(self): + m = sc.install_metrics(FakeGitHub({})) + self.assertIn("no install-scorecard artifact", m["idle_ram_mb"]["not_measured"]) + m = sc.install_metrics(FakeGitHub({"install-scorecard": {}}, token=None)) + self.assertIn("needs a GitHub token", m["disk_mb"]["not_measured"]) + + def test_coverage_from_an_artifact(self): + xml = '' + gh = FakeGitHub({"backend-coverage": {"coverage.xml": xml}}) + m = sc.coverage(gh, "backend-coverage", "coverage.xml", sc.cobertura_rate) + self.assertEqual( + (m["value"], m["lines"], m["commit"]), (75.0, 400, "abcdef123456") + ) + + +class MeasureInstall(unittest.TestCase): + def test_sizes(self): + self.assertAlmostEqual(mi.to_mb("1.5GiB"), 1610.6127, places=3) + self.assertEqual(mi.to_mb("512MB"), 512) + self.assertEqual(mi.to_mb("0B"), 0) + self.assertAlmostEqual(mi.to_mb("12.2kB"), 0.0122) + with self.assertRaises(ValueError): + mi.to_mb("lots") + + def test_docker_stats_and_df(self): + stats = [ + {"Name": "lens-backend-1", "MemUsage": "700MiB / 15.6GiB"}, + {"Name": "lens-db-1", "MemUsage": "1.2GiB / 15.6GiB"}, + ] + self.assertEqual( + mi.ram_mb(stats), {"lens-backend-1": 734.0, "lens-db-1": 1288.5} + ) + df = [ + {"Type": "Images", "Size": "3.2GB"}, + {"Type": "Build Cache", "Size": "0B"}, + ] + self.assertEqual(mi.disk_mb(df), {"Images": 3200.0, "Build Cache": 0.0}) diff --git a/docs/contributing.md b/docs/contributing.md index 2f1667da0..01239b6f1 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -160,7 +160,7 @@ line carries the commit, `git describe` tag and release it was taken at. | Red-check rate, ask to merge, CI minutes per merged pull request | Pull requests opened in the last `--days` (7), bots left out: the share whose first push had a failing workflow run (a re-run counts its first attempt), the median hours from the earliest of the pull request and the issues it closes to its merge, and the median wall time of its workflow runs | | Failed runs on `main` | Workflow runs on pushes to `main` in the same window that failed or timed out | | Stars, forks, and issues and pull requests from outside | The repository, and issues and pull requests whose authors aren't its owner, members or collaborators | -| Install to first answer, and idle RAM and disk on arm64 (as on a Raspberry Pi) | Not measured yet: there's no timed install on an arm64 runner | +| Install to first answer, and idle RAM and disk on arm64 (as a Raspberry Pi is) | The newest `install-scorecard` artifact from main: the Install workflow's arm64 job times the one-line install until the setup page answers, waits two minutes, then adds up its containers' memory (`docker stats`) and Docker's images, containers and volumes plus the install folder (`ci/scorecard/measure_install.py`). It runs weekly and when the install changes. GitHub's arm64 runner is faster than a Pi, so the time is a floor | ```bash make scorecard # print the line