From 70e02af8c89a53c5cd6b16f245d16f601e6fc5a8 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Tue, 20 Oct 2026 02:08:56 +0000 Subject: [PATCH 1/2] M9.6: GA4 telemetry, opt-in, informing goals and never a gate mission telemetry init/consent/revoke/auth/setup/debug-hit/report/share/show/check. Consent is a human event on the exact terms; setup goes through the Admin API (property, user-data acknowledgement, web stream, MP secret outside docs/, key events); the debug hit validates journey events; fixed Data API reports read settled data; the minimum-sample rule decides usage's share. Tests run against a local fake of the GA APIs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Ro7iYo3scQt6vAuAWdYpV --- docs/ROADMAP.md | 4 +- docs/TRACKING.md | 2 +- plugin/mission/schemas/telemetry.json | 75 +++ plugin/mission/telemetry.py | 808 +++++++++++++++++++++++++ scripts/test/fixtures/fake-ga.py | 249 ++++++++ scripts/test/mission-telemetry.test.py | 325 ++++++++++ 6 files changed, 1460 insertions(+), 3 deletions(-) create mode 100644 plugin/mission/schemas/telemetry.json create mode 100644 plugin/mission/telemetry.py create mode 100644 scripts/test/fixtures/fake-ga.py create mode 100644 scripts/test/mission-telemetry.test.py diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index fba104ff..51f86b9e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -408,7 +408,7 @@ Each task flips only its own row here and in TRACKING.md. | M9.3 | Journeys and metrics: steps, pass criteria, events, metric query definitions, source refs; `diff` blocks removing or weakening without approval; `run` through a pinned runner; all-green-at-baseline refused | M9.0 | `plugin/mission/journeys.py`, `schemas/journey.json`, `scripts/test/mission-journeys.test.py` | yes | open | | M9.4 | Ethics: review per mission and goal, three tiers, gray consent bound to the goal's sha256, hard line never unlocked | M9.0 | `plugin/mission/ethics.py`, `schemas/ethics.json`, `scripts/test/mission-ethics.test.py` | yes | open | | M9.5 | Judges: rubric setup check and an agreement verdict from recorded runs | M9.0 | `plugin/mission/judge.py`, `schemas/rubric.json`, `scripts/test/mission-judge.test.py` | yes | open | -| M9.6 | Telemetry (GA4): opt-in consent, Admin API setup, Measurement Protocol debug hit, fixed Data API reports (+ realtime), minimum-sample rule, static emit check; network faked in tests | M9.0 | `plugin/mission/telemetry.py`, `schemas/telemetry.json`, `scripts/test/mission-telemetry.test.py`, `scripts/test/fixtures/fake-ga.py` | yes | open | +| M9.6 | Telemetry (GA4): opt-in consent, Admin API setup, Measurement Protocol debug hit, fixed Data API reports (+ realtime), minimum-sample rule, static emit check; network faked in tests | M9.0 | `plugin/mission/telemetry.py`, `schemas/telemetry.json`, `scripts/test/mission-telemetry.test.py`, `scripts/test/fixtures/fake-ga.py` | yes | done: `mission-telemetry.test.py` (12 checks, fake GA) | | M9.7 | Principles in two variants with tension pairs; `principles show --mode`, hashed and injected into goals | M9.0 | `plugin/mission/principles.{md,py}`, `scripts/test/mission-principles.test.py` | yes | open | | M9.8 | Loop: `tick` (the state machine above), `due` (zero-token cadence check), `cron` (host routine or crontab line), budget caps from the token ledger | M9.0 | `plugin/mission/{tick,due,cron}.py`, `scripts/test/mission-tick.test.py` | yes | open | | M9.9 | Roles and wiring: the six personas, hook matchers, model routes, host install, eval suites, role counts in docs; `gate-mission-scope` (role path guards) and `gate-mission-artifact` (a role's artifact passes its check at stop) | M9.0 | `plugin/agents/` (6 new files), `plugin/hooks/hooks.json`, `plugin/hosts/`, `plugin/evals/`, `plugin/bin/gate-mission-*`, role counts in README / OVERVIEW / plugin docs | yes (the only track touching shared lists) | open | @@ -416,7 +416,7 @@ Each task flips only its own row here and in TRACKING.md. | M9.11 | Goals and the Aladdin handoff: top gaps → a goal → intents (`intent` source `mission`), ethics and principles applied, mode → autonomy; `events trace` reaches goal → mission | M9.4, M9.7, M9.10 | `plugin/mission/goal.py`, `schemas/goal.json`, `plugin/bin/intent` (source list), `plugin/bin/events` (trace) | after its needs | blocked | | M9.12 | The `/aloop:mission` skill: the supervisor's playbook for the tick loop | M9.8, M9.9 | `plugin/skills/mission/` | after its needs | blocked | | M9.13 | End to end: an offline scripted to-do-app mission (mission → baseline → eval → goal → intent, zero tokens), then a live `make e2e-mission YES=1` | M9.10–M9.12 | `scripts/test/mission-e2e.test.py`, `scripts/smoke/e2e-mission.sh` | after its needs | blocked | -| M9.14 | GA4 live on the owner's account (one Google sign-in) | M9.6 | — | — | needs the owner | +| M9.14 | GA4 live on the owner's account (one Google sign-in) | M9.6 | — | — | needs the owner: `aloop mission telemetry auth` prints the steps, then `init`, `consent`, `setup`, `debug-hit` | | M9.15 | Native voice mode | the owner's scope (spec cut off) | — | — | open question | | M9.16 | Docs: OVERVIEW, README, CHANGELOG, site | M9.13 | shared docs | — | blocked | | M9.17 | Approval provenance: an approval names the captured human prompt (the UserPromptSubmit log), so a supervisor cannot approve its own mission change. Reason: found while building M9.0, an approval is only as strong as `--by` | M9.0 | `plugin/bin/_mission.py` (`approve`), its test | yes | open | diff --git a/docs/TRACKING.md b/docs/TRACKING.md index 94174e6e..c6c79cd0 100644 --- a/docs/TRACKING.md +++ b/docs/TRACKING.md @@ -65,7 +65,7 @@ Status of every requirement in [ROADMAP.md](ROADMAP.md), in the four states §8 | M9.3 | Journeys and metrics: events per step, query definitions, no removal or weakening without approval | NOT_IMPLEMENTED | §4.12 | | M9.4 | Ethics: three tiers; gray consent bound to the goal; hard line never unlocked | NOT_IMPLEMENTED | §4.12 | | M9.5 | LLM judges: pinned model, hashed rubric, several runs, agreement threshold | NOT_IMPLEMENTED | §4.12 | -| M9.6 | Telemetry (GA4): opt-in, Admin API setup, debug hit, fixed reports, minimum sample; never a gate | NOT_IMPLEMENTED | §4.12 | +| M9.6 | Telemetry (GA4): opt-in, Admin API setup, debug hit, fixed reports, minimum sample; never a gate | IMPLEMENTED_TESTED | `scripts/test/mission-telemetry.test.py` | | M9.7 | Standing principles in two variants (autonomous, spoon-fed) | NOT_IMPLEMENTED | §4.12 | | M9.8 | Mission loop: `tick` from facts, zero-token `due`, cron, budget caps | NOT_IMPLEMENTED | §4.12 | | M9.9 | Six fixed mission roles, wired; role path guards; artifact check at stop | NOT_IMPLEMENTED | §4.12 | diff --git a/plugin/mission/schemas/telemetry.json b/plugin/mission/schemas/telemetry.json new file mode 100644 index 00000000..a853b319 --- /dev/null +++ b/plugin/mission/schemas/telemetry.json @@ -0,0 +1,75 @@ +{ + "title": "telemetry — the opt-in GA4 usage record (ROADMAP §4.12, M9.6): docs/agents/mission/telemetry/.json", + "type": "object", + "required": ["telemetry_id", "status", "created_at", "provider", "terms", "settings", "property", "acknowledged", "stream", "secret", "key_events", "debug_hit"], + "additionalProperties": false, + "properties": { + "telemetry_id": {"type": "string", "pattern": "^tlm_"}, + "status": {"enum": ["off", "consented", "configured", "revoked"]}, + "created_at": {"type": "string", "minLength": 1}, + "provider": {"const": "ga4"}, + "mission_id": {"type": ["string", "null"]}, + "terms": { + "type": "object", + "required": ["uri", "purpose", "collects", "agents_see", "acknowledgement", "scopes", "others"], + "additionalProperties": false, + "properties": { + "uri": {"type": "string", "pattern": "^https?://[^\\s/]+"}, + "purpose": {"type": "string", "minLength": 1}, + "collects": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "agents_see": {"type": "string", "minLength": 1}, + "acknowledgement": {"type": "string", "minLength": 1}, + "scopes": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://www\\.googleapis\\.com/auth/analytics"}}, + "others": {"type": "string", "minLength": 1} + } + }, + "settings": { + "type": "object", + "required": ["display_name", "time_zone", "currency"], + "additionalProperties": false, + "properties": { + "display_name": {"type": "string", "minLength": 1}, + "time_zone": {"type": "string", "minLength": 1}, + "currency": {"type": "string", "pattern": "^[A-Z]{3}$"} + } + }, + "account": {"type": ["string", "null"], "pattern": "^accounts/[0-9]+$"}, + "property": {"type": ["string", "null"], "pattern": "^properties/[0-9]+$"}, + "acknowledged": {"type": "boolean"}, + "stream": { + "type": ["object", "null"], + "required": ["name", "measurement_id", "default_uri"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "pattern": "^properties/[0-9]+/dataStreams/[0-9]+$"}, + "measurement_id": {"type": "string", "pattern": "^G-[A-Z0-9]+$"}, + "default_uri": {"type": "string", "minLength": 1} + } + }, + "secret": { + "type": ["object", "null"], + "required": ["name", "display_name"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "pattern": "/measurementProtocolSecrets/"}, + "display_name": {"type": "string"} + } + }, + "key_events": {"type": "array", "items": {"type": "string", "pattern": "^[A-Za-z][A-Za-z0-9_]{0,39}$"}}, + "debug_hit": { + "type": ["object", "null"], + "required": ["at", "ok", "measurement_id", "events_sha256", "events", "messages"], + "additionalProperties": false, + "properties": { + "at": {"type": "string", "minLength": 1}, + "ok": {"type": "boolean"}, + "measurement_id": {"type": "string", "pattern": "^G-"}, + "events_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "events": {"type": "integer", "minimum": 1}, + "messages": {"type": "array", "items": {"type": "object"}} + } + }, + "approval": {"type": "object"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } +} diff --git a/plugin/mission/telemetry.py b/plugin/mission/telemetry.py new file mode 100644 index 00000000..12961d4d --- /dev/null +++ b/plugin/mission/telemetry.py @@ -0,0 +1,808 @@ +"""telemetry — usage from Google Analytics 4: opt-in, informs goals, never a gate (ROADMAP §4.12, M9.6). + +Real usage drives most of a later goal once there is enough of it, so it is captured from day +one through GA4, which the agents set up through Google's APIs: no server to host, nothing to +buy. Nothing reaches the network until a human consents to the exact terms below, and a +revoke stops it. Agents see aggregate counts from fixed report definitions only. + +Usage: + mission telemetry init --uri URL [--name NAME] [--time-zone TZ] [--force] + draft the terms (what is measured, why, what agents see, GA's user-data acknowledgement, + the OAuth scopes) as a sealed record, status off. Nothing leaves the machine. + --force re-drafts the terms of the existing record (consent is asked again). + mission telemetry consent --by NAME [--reason TEXT] + the human opts in to exactly these terms: telemetry.consent names their sha256 + mission telemetry revoke --by NAME [--reason TEXT] + opt out: setup, debug-hit and report refuse until a new consent + mission telemetry auth + the one-time Google sign-in, as pastable steps (the only human step) + mission telemetry setup [--account accounts/N] [--property properties/N] + Admin API, idempotent: find or create the property, acknowledge user-data collection, + a web data stream for the uri, a Measurement Protocol secret (kept in .aladdin/state/, + never in docs/) and a key event for each active journey's last step + mission telemetry debug-hit + send every active journey step's event to GA's validation server; the result is recorded + mission telemetry report [--realtime] [--json] + the fixed Data API reports: settled (28 days ending 3 days ago, so ≥48h old) or realtime + (the last 30 minutes, a quick signal only); written to docs/agents/mission/telemetry/reports/ + mission telemetry share [--json] + how much of the next goal usage may drive, from the latest settled report: 0 below + 50 users over 28 days or when GA thresholded the data, else 0.7 (research keeps 0.3) + mission telemetry show [--json] + mission telemetry check [--json] + the structural check: consent covers the current terms; once consented, every active + journey step's event is emitted by non-test source code; once configured, a passing + debug hit covers the current events. Usage numbers never change its exit. + Exit 0 ok (or off) · 2 block · 3 needs the human · 64 usage + +Environment: ALADDIN_GA_ACCESS_TOKEN (an OAuth access token), ALADDIN_GA_TOKEN_COMMAND (default +`gcloud auth application-default print-access-token`), ALADDIN_GA_BASE (a loopback fake, tests only). +""" +import glob +import json +import os +import re +import shlex +import subprocess +import sys +import urllib.error +import urllib.request +from urllib.parse import urlencode, urlsplit + +import _events +import _mission as M +import _paths + +SCOPES = ("https://www.googleapis.com/auth/analytics.edit", "https://www.googleapis.com/auth/analytics.readonly") +# Google's verbatim text for properties.acknowledgeUserDataCollection, which a Measurement Protocol +# secret needs; the human consents to it as part of the terms +ACKNOWLEDGEMENT = ("I acknowledge that I have the necessary privacy disclosures and rights from my end users for the " + "collection and processing of their data, including the association of such data with the " + "visitation information Google Analytics collects from my site and/or app property.") +HOSTS = {"admin": "https://analyticsadmin.googleapis.com", "data": "https://analyticsdata.googleapis.com", + "mp": "https://www.google-analytics.com"} +LOOPBACK = ("127.0.0.1", "localhost", "::1") +TOKEN_COMMAND = "gcloud auth application-default print-access-token" + +# the minimum-sample rule (§4.12 call): small numbers are noise, and GA hides them anyway +MIN_USERS, WINDOW_DAYS, USAGE_SHARE, RESEARCH_FLOOR = 50, 28, 0.7, 0.3 +SETTLED = {"startDate": "30daysAgo", "endDate": "3daysAgo"} # 28 days; the newest is ≥48h old, so settled +# the fixed report definitions (hashed into every report). Aggregate dimensions only: agents +# never see a user or client id +AGGREGATE_DIMENSIONS = {"eventName", "date", "newVsReturning", "deviceCategory"} +REPORTS = { + "users": {"api": "runReport", "body": {"dateRanges": [SETTLED], "metrics": [ + {"name": "totalUsers"}, {"name": "newUsers"}, {"name": "activeUsers"}]}}, + "journeys": {"api": "runReport", "events": True, "body": {"dateRanges": [SETTLED], "dimensions": [ + {"name": "eventName"}], "metrics": [{"name": "eventCount"}, {"name": "totalUsers"}]}}, + "realtime": {"api": "runRealtimeReport", "events": True, "body": {"dimensions": [{"name": "eventName"}], + "metrics": [{"name": "eventCount"}, {"name": "activeUsers"}]}}, +} +SETTLED_REPORTS = ("users", "journeys") + +# GA4 event names: a letter, then letters, digits or underscores, at most 40; reserved names and prefixes +EVENT_NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,39}$") +RESERVED = {"ad_activeview", "ad_click", "ad_exposure", "ad_impression", "ad_query", "ad_reward", "adunit_exposure", + "app_clear_data", "app_exception", "app_install", "app_remove", "app_store_refund", "app_update", + "app_upgrade", "dynamic_link_app_open", "dynamic_link_app_update", "dynamic_link_first_open", "error", + "first_open", "first_visit", "in_app_purchase", "notification_dismiss", "notification_foreground", + "notification_open", "notification_receive", "os_update", "session_start", "session_start_with_rollout", + "user_engagement"} +RESERVED_PREFIXES = ("_", "firebase_", "ga_", "google_", "gtag.") +MP_BATCH = 25 # events per Measurement Protocol request +DEBUG_CLIENT_ID = "aloop.debug" # the debug hit's client id: no user's +PROBE_EVENT = "aloop_probe" # sent when no journey has events yet, to prove the wiring +SECRET_NAME = "aloop" + +# the emit check reads code, not docs, data or tests +CODE_EXT = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".vue", ".svelte", ".astro", ".html", ".htm", ".py", + ".rb", ".erb", ".go", ".rs", ".java", ".kt", ".swift", ".dart", ".php", ".cs", ".ex", ".exs", ".m"} +SKIP_DIRS = {"docs", "test", "tests", "__tests__", "spec", "specs", "fixtures", "node_modules", "vendor", "dist", "build"} +TEST_FILE = re.compile(r"(^test_|_test\.[^.]+$|\.(test|spec)\.[^.]+$)") + +AUTH_STEPS = """One-time Google sign-in (the only human step; paste in a terminal on your Mac): + 1. A Google Analytics account (skip if you have one): https://analytics.google.com -> Start measuring. + 2. A Google Cloud project with the two Analytics APIs on: + gcloud projects create aloop-ga-$RANDOM --set-as-default # or: gcloud config set project YOUR_PROJECT + gcloud services enable analyticsadmin.googleapis.com analyticsdata.googleapis.com + 3. An OAuth client for that project: https://console.cloud.google.com/auth/clients -> Create client -> + Desktop app -> Download JSON, saved as ~/aloop-ga-client.json (the first time, the consent screen asks + for an app name and your email, and you add yourself as a test user). + 4. Sign in once: + gcloud auth application-default login --client-id-file="$HOME/aloop-ga-client.json" \\ + --scopes=https://www.googleapis.com/auth/analytics.edit,https://www.googleapis.com/auth/analytics.readonly,https://www.googleapis.com/auth/cloud-platform +Then: aloop mission telemetry setup""" + + +class Usage(Exception): + pass + + +class GAError(Exception): + def __init__(self, code, status, message): + super().__init__("%s %s: %s" % (code, status, message)) + self.code, self.status, self.message = code, status, message + + +def say(kind, msg): + print("%s[telemetry]: %s" % (kind, msg), file=sys.stderr) + + +# ── terms, consent and the record ──────────────────────────────────────────────────── +def terms_sha(doc): + """What a consent covers: this record's terms, nothing it learns later (property, stream).""" + return M.digest({"telemetry_id": doc.get("telemetry_id"), "terms": doc.get("terms")}, drop=()) + + +def current(cwd): + """(path, doc) of the project's telemetry record, or (None, None): one per project.""" + recs = M.artifacts(cwd, "telemetry") + return recs[-1] if recs else (None, None) + + +def consent_state(cwd, path, doc): + """none | revoked | stale (the last consent covers other terms) | ok, from the events only.""" + ref, last = M.rel(cwd, path), None + for e in _events.read(cwd): + if e.get("event") in ("telemetry.consent", "telemetry.revoked") and (e.get("data") or {}).get("subject") == ref: + last = e + if last is None: + return "none", None + if last["event"] == "telemetry.revoked": + return "revoked", last + return ("ok" if (last.get("data") or {}).get("sha256") == terms_sha(doc) else "stale"), last + + +OPT_IN = ("telemetry is opt-in and off: `aloop mission telemetry init --uri URL`, then a human runs " + "`aloop mission telemetry consent --by NAME`") + + +def consented(cwd): + """(path, doc, 0) when the human's consent covers the current terms; else asks and returns rc 3 + (or 2 for a record that fails its own check). Every network command starts here.""" + path, doc = current(cwd) + if doc is None: + say("ASK", OPT_IN) + return None, None, 3 + bad = M.problems(doc, "telemetry", path) + if bad: + for b in bad: + say("BLOCK", b) + return path, doc, 2 + state, ev = consent_state(cwd, path, doc) + if state == "ok": + return path, doc, 0 + if state == "revoked": + say("ASK", "telemetry was revoked by %s: a human runs `aloop mission telemetry consent --by NAME` to resume" + % (ev.get("data") or {}).get("by")) + elif state == "stale": + say("ASK", "the consent does not cover the current terms (they changed): a human reviews " + "`aloop mission telemetry show` and runs `aloop mission telemetry consent --by NAME`") + else: + say("ASK", "not consented yet: a human reviews `aloop mission telemetry show` and runs " + "`aloop mission telemetry consent --by NAME`") + return path, doc, 3 + + +def store(path, doc): + M.seal(doc) + M.save(path, doc) + + +def secret_file(cwd, telemetry_id): + """The Measurement Protocol secret lives in the git-ignored run-data dir, never in docs/.""" + return os.path.join(_paths.home(cwd, "state"), "telemetry", telemetry_id + ".json") + + +def read_secret(cwd, telemetry_id): + d, _ = M.load(secret_file(cwd, telemetry_id)) + return d if isinstance(d, dict) and d.get("api_secret") else None + + +def write_secret(cwd, telemetry_id, data): + p = secret_file(cwd, telemetry_id) + os.makedirs(os.path.dirname(p), exist_ok=True) + fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + os.fchmod(f.fileno(), 0o600) + json.dump(data, f) + return p + + +# ── journeys (M9.3's files, read by contract fields only) and the emit check ───────── +def journeys(cwd): + """Active journeys as [(name, [step event, …])], oldest first.""" + out = [] + for _, d in M.artifacts(cwd, "journey"): + if d.get("status") == "active": + evs = [s.get("event") for s in d.get("steps") or [] if isinstance(s, dict) and isinstance(s.get("event"), str)] + out.append((d.get("name") or d.get("journey_id"), evs)) + return out + + +def journey_events(cwd): + return sorted({e for _, evs in journeys(cwd) for e in evs}) + + +def events_sha(evs): + return M.digest({"events": sorted(evs)}, drop=()) + + +def name_problem(name): + if not EVENT_NAME.match(name): + return "must start with a letter and use only letters, digits and _ (at most 40)" + if name in RESERVED or name.startswith(RESERVED_PREFIXES): + return "is reserved by Google Analytics" + return None + + +def source_files(root): + try: + out = subprocess.run(["git", "-C", root, "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + capture_output=True, text=True, timeout=60) + files = [f for f in out.stdout.split("\0") if f] if out.returncode == 0 else None + except (OSError, subprocess.TimeoutExpired): + files = None + if files is None: + files = [os.path.relpath(os.path.join(d, f), root) for d, _, fs in os.walk(root) for f in fs] + for f in sorted(set(files)): + parts = f.replace(os.sep, "/").split("/") + if any(p.startswith(".") or p in SKIP_DIRS for p in parts[:-1]) or TEST_FILE.search(parts[-1]): + continue + if os.path.splitext(f)[1].lower() in CODE_EXT: + yield f + + +def emitted(cwd, names): + """{event: "file:line"} for each name found as a quoted literal ('x', "x" or `x`) in code.""" + root, found = _paths.project(cwd), {} + want = {n: re.compile(r"(['\"`])%s\1" % re.escape(n)) for n in names} + for f in source_files(root): + if len(found) == len(want): + break + try: + with open(os.path.join(root, f), errors="ignore") as fh: + for i, line in enumerate(fh, 1): + for n, rx in want.items(): + if n not in found and rx.search(line): + found[n] = "%s:%d" % (f, i) + except OSError: + continue + return found + + +# ── the network: three Google hosts, or one loopback fake ──────────────────────────── +def endpoint(api): + base = _paths.env("GA_BASE") + if not base: + return HOSTS[api] + u = urlsplit(base) + if u.scheme not in ("http", "https") or u.hostname not in LOOPBACK: + raise Usage("ALADDIN_GA_BASE must be a loopback URL (a test fake), not %r: a token never leaves for " + "another host" % base) + return base.rstrip("/") + "/" + api + + +def token(): + t = _paths.env("GA_ACCESS_TOKEN") + if t: + return t.strip() + try: + r = subprocess.run(shlex.split(_paths.env("GA_TOKEN_COMMAND") or TOKEN_COMMAND), + capture_output=True, text=True, timeout=60) + except (OSError, ValueError, subprocess.TimeoutExpired): + return None + out = r.stdout.strip() + return out if r.returncode == 0 and out and not any(c.isspace() for c in out) else None + + +def call(method, api, path, body=None, tok=None, query=None): + url = endpoint(api) + path + ("?" + urlencode(query) if query else "") + headers = {"Content-Type": "application/json"} + if tok: + headers["Authorization"] = "Bearer " + tok + req = urllib.request.Request(url, data=None if body is None else json.dumps(body).encode(), headers=headers, + method=method) + # a loopback fake is reached directly, never through a proxy + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) if _paths.env("GA_BASE") else \ + urllib.request.build_opener() + try: + with opener.open(req, timeout=60) as r: + raw = r.read() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as e: + try: + err = (json.loads(e.read() or b"{}") or {}).get("error") or {} + except ValueError: + err = {} + raise GAError(e.code, err.get("status", ""), err.get("message") or str(e.reason)) + except (urllib.error.URLError, OSError, ValueError) as e: + raise GAError(0, "UNREACHABLE", str(e)) + + +def listall(path, key, tok): + """Every item of a paged Admin API list.""" + out, page = [], None + while True: + q = {"pageSize": 200} + if page: + q["pageToken"] = page + r = call("GET", "admin", path, tok=tok, query=q) + out += r.get(key) or [] + page = r.get("nextPageToken") + if not page: + return out + + +def ga_failed(e): + """Map a GA error to an exit: credentials and API access are the human's (3), the rest block (2).""" + if e.code == 401: + say("ASK", "Google rejected the credentials (%s).\n%s" % (e.message, AUTH_STEPS)) + return 3 + if e.code == 403 and re.search(r"has not been used|is disabled|SERVICE_DISABLED|accessNotConfigured", e.message): + say("ASK", "an Analytics API is off in your Google Cloud project (%s). Paste:\n" + " gcloud services enable analyticsadmin.googleapis.com analyticsdata.googleapis.com" % e.message) + return 3 + if e.code == 403: + say("ASK", "the signed-in Google account lacks access (%s): give it the Editor role on the Analytics " + "account (Admin -> Account access management), or pass --account / --property you can edit" % e.message) + return 3 + say("BLOCK", "Google Analytics: %s" % e) + return 2 + + +def need_token(): + tok = token() + if not tok: + say("ASK", "no Google credentials yet.\n" + AUTH_STEPS) + return tok + + +# ── commands ───────────────────────────────────────────────────────────────────────── +def cmd_init(cwd, o): + uri = o.get("uri") or "" + if not re.match(r"^https?://[^\s/]+", uri): + say("BLOCK", "--uri must be the app's http(s) URL (what is measured is part of the terms)") + return 64 + path, doc = current(cwd) + if doc is not None and not o.get("force"): + say("BLOCK", "%s exists; `aloop mission telemetry init --uri URL --force` re-drafts its terms " + "(consent is asked again)" % M.rel(cwd, path)) + return 2 + mission, _ = M.load(M.path_of(cwd, "mission")) + mission = mission if isinstance(mission, dict) else {} + name = o.get("name") or ("aloop: " + (mission.get("statement") or urlsplit(uri).hostname))[:100] + terms = {"uri": uri, + "purpose": "usage of %s informs research and goal-setting for the mission; it is never a gate" % uri, + "collects": ["the app's Google Analytics 4 web measurement on this uri", + "one event per user-journey step (its name and engagement time)"], + "agents_see": "aggregate counts only (event name, users, new users) from fixed report definitions; " + "never a user or client id", + "acknowledgement": ACKNOWLEDGEMENT, "scopes": list(SCOPES), + "others": "once other people use the app, a consent banner (Consent Mode v2) is a journey requirement"} + settings = {"display_name": name, "time_zone": o.get("time_zone") or "Etc/GMT", "currency": "USD"} + if doc is None: + tid = M.new_id("telemetry") + path = M.path_of(cwd, "telemetry", tid) + doc = {"telemetry_id": tid, "status": "off", "created_at": M.now(), "provider": "ga4", + "mission_id": mission.get("mission_id"), "terms": terms, "settings": settings, "account": None, + "property": None, "acknowledged": False, "stream": None, "secret": None, "key_events": [], + "debug_hit": None} + else: + doc.update(terms=terms, settings=settings, status="off") + store(path, doc) + print(path) + say("ASK", "terms drafted (nothing was sent): a human reviews `aloop mission telemetry show` and runs " + "`aloop mission telemetry consent --by NAME`") + return 0 + + +def who(o, verb): + by, reason = (o.get("by") or "").strip(), (o.get("reason") or "").strip() + if not by: + raise Usage("%s names the human (--by NAME)" % verb) + if "\n" in reason or len(reason) > 200: + raise Usage("a reason is one line of at most 200 characters") + return by, reason + + +def cmd_consent(cwd, o): + path, doc = current(cwd) + if doc is None: + say("ASK", OPT_IN) + return 3 + by, reason = who(o, "consent") + bad = M.problems(doc, "telemetry", path) + for b in bad: + say("BLOCK", b) + if bad: + return 2 + eid = M.approve(cwd, "telemetry.consent", M.rel(cwd, path), terms_sha(doc), by, reason, + telemetry_id=doc["telemetry_id"], uri=doc["terms"]["uri"]) + doc["status"] = "configured" if doc.get("stream") else "consented" + doc["approval"] = {"event_id": eid, "by": by, "at": M.now()} + store(path, doc) + print("[telemetry] %s: %s consented to the terms (sha256 %s)" % (doc["telemetry_id"], by, terms_sha(doc)[:12]), + file=sys.stderr) + return 0 + + +def cmd_revoke(cwd, o): + path, doc = current(cwd) + if doc is None: + say("ASK", OPT_IN) + return 3 + by, reason = who(o, "revoke") + ref = M.rel(cwd, path) + M.emit(cwd, "telemetry.revoked", role="human", refs={"subject": ref}, subject=ref, by=by, reason=reason, + telemetry_id=doc.get("telemetry_id")) + doc["status"] = "revoked" + M.save(path, doc) # status is outside the digest: the seal still holds + print("[telemetry] %s revoked by %s: nothing is sent until a new consent" % (doc.get("telemetry_id"), by), + file=sys.stderr) + return 0 + + +def pick_property(doc, o, tok): + """(account, property, rc): the property given, recorded, found by display name, or created in + the one reachable account. Which account is the human's call when there are none or several (rc 3).""" + prop = o.get("property") or doc.get("property") + if prop: + call("GET", "admin", "/v1beta/" + prop, tok=tok) + return doc.get("account"), prop, 0 + summaries = listall("/v1beta/accountSummaries", "accountSummaries", tok) + if o.get("account"): + found = [a for a in summaries if a.get("account") == o["account"]] + if not found: + say("BLOCK", "%s is not an account the signed-in Google user can reach (%s)" + % (o["account"], ", ".join(a.get("account", "") for a in summaries) or "none")) + return None, None, 2 + elif not summaries: + say("ASK", "the signed-in Google user has no Analytics account: create one at https://analytics.google.com " + "(Admin -> Create account; Google needs you to accept its terms there), then rerun " + "`aloop mission telemetry setup`") + return None, None, 3 + elif len(summaries) > 1: + say("ASK", "several Analytics accounts: choose one, e.g. `aloop mission telemetry setup --account %s` (%s)" + % (summaries[0].get("account"), "; ".join("%s %s" % (a.get("account"), a.get("displayName", "")) + for a in summaries))) + return None, None, 3 + else: + found = summaries + acct, name = found[0]["account"], doc["settings"]["display_name"] + same = [p for p in found[0].get("propertySummaries") or [] if p.get("displayName") == name] + if same: + return acct, same[0]["property"], 0 + p = call("POST", "admin", "/v1beta/properties", {"parent": acct, "displayName": name, + "timeZone": doc["settings"]["time_zone"], + "currencyCode": doc["settings"]["currency"]}, tok) + return acct, p["name"], 0 + + +def cmd_setup(cwd, o): + path, doc, rc = consented(cwd) + if rc: + return rc + endpoint("admin") + tok = need_token() + if not tok: + return 3 + tid, uri = doc["telemetry_id"], doc["terms"]["uri"] + try: + acct, prop, rc = pick_property(doc, o, tok) + if rc: + return rc + if prop != doc.get("property"): + doc.update(account=acct, property=prop, acknowledged=False, stream=None, secret=None, key_events=[], + debug_hit=None) + store(path, doc) # progress survives a later failure: a rerun reuses it + if not doc.get("acknowledged"): + call("POST", "admin", "/v1beta/%s:acknowledgeUserDataCollection" % prop, + {"acknowledgement": doc["terms"]["acknowledgement"]}, tok) + doc["acknowledged"] = True + store(path, doc) + streams = listall("/v1beta/%s/dataStreams" % prop, "dataStreams", tok) + s = next((x for x in streams if x.get("type") == "WEB_DATA_STREAM" + and (x.get("webStreamData") or {}).get("defaultUri") == uri), None) + if s is None: + s = call("POST", "admin", "/v1beta/%s/dataStreams" % prop, + {"type": "WEB_DATA_STREAM", "displayName": doc["settings"]["display_name"], + "webStreamData": {"defaultUri": uri}}, tok) + stream = {"name": s["name"], "measurement_id": s["webStreamData"]["measurementId"], "default_uri": uri} + if stream != doc.get("stream"): + doc.update(stream=stream, secret=None, debug_hit=None) + store(path, doc) + local = read_secret(cwd, tid) + secrets = listall("/v1beta/%s/measurementProtocolSecrets" % s["name"], "measurementProtocolSecrets", tok) + sec = next((x for x in secrets if local and x.get("name") == local.get("name")), None) or \ + next((x for x in secrets if x.get("displayName") == SECRET_NAME and x.get("secretValue")), None) + if sec is None: + sec = call("POST", "admin", "/v1beta/%s/measurementProtocolSecrets" % s["name"], + {"displayName": SECRET_NAME}, tok) + value = sec.get("secretValue") or (local or {}).get("api_secret") + if not value: + raise GAError(0, "NO_SECRET", "Google returned no Measurement Protocol secret value") + write_secret(cwd, tid, {"telemetry_id": tid, "stream": s["name"], "name": sec["name"], "api_secret": value}) + doc["secret"] = {"name": sec["name"], "display_name": sec.get("displayName") or SECRET_NAME} + want = [evs[-1] for _, evs in journeys(cwd) if evs and not name_problem(evs[-1])] + have = {k.get("eventName") for k in listall("/v1beta/%s/keyEvents" % prop, "keyEvents", tok)} + for ev in sorted(set(want) - have): + call("POST", "admin", "/v1beta/%s/keyEvents" % prop, {"eventName": ev, "countingMethod": "ONCE_PER_EVENT"}, + tok) + doc["key_events"] = sorted(set(want)) + doc["status"] = "configured" + store(path, doc) + except GAError as e: + return ga_failed(e) + M.emit(cwd, "telemetry.setup", role="mission", refs={"telemetry": M.rel(cwd, path)}, action="setup", + telemetry_id=tid, property=prop, measurement_id=stream["measurement_id"], key_events=doc["key_events"]) + print("[telemetry] %s configured: %s, stream %s, %d key event(s)" + % (tid, prop, stream["measurement_id"], len(doc["key_events"])), file=sys.stderr) + return 0 + + +def cmd_debug_hit(cwd, o): + path, doc, rc = consented(cwd) + if rc: + return rc + endpoint("mp") + sec = read_secret(cwd, doc["telemetry_id"]) + if not doc.get("stream") or not sec: + say("BLOCK", "not set up on this machine yet: `aloop mission telemetry setup` first") + return 2 + evs = journey_events(cwd) + bad = ["event %r %s" % (e, name_problem(e)) for e in evs if name_problem(e)] + for b in bad: + say("BLOCK", b) + if bad: + return 2 + mid, msgs = doc["stream"]["measurement_id"], [] + send = evs or [PROBE_EVENT] + try: + for i in range(0, len(send), MP_BATCH): + body = {"client_id": DEBUG_CLIENT_ID, + "events": [{"name": e, "params": {"engagement_time_msec": 1}} for e in send[i:i + MP_BATCH]]} + r = call("POST", "mp", "/debug/mp/collect", body, query={"measurement_id": mid, "api_secret": sec["api_secret"]}) + msgs += r.get("validationMessages") or [] + except GAError as e: + return ga_failed(e) + doc["debug_hit"] = {"at": M.now(), "ok": not msgs, "measurement_id": mid, "events_sha256": events_sha(evs), + "events": len(send), "messages": msgs[:20]} + store(path, doc) + M.emit(cwd, "telemetry.setup", role="mission", refs={"telemetry": M.rel(cwd, path)}, action="debug_hit", + telemetry_id=doc["telemetry_id"], ok=not msgs, events=len(send)) + for m in msgs: + say("BLOCK", "GA validation: %s (%s) %s" % (m.get("validationCode"), m.get("fieldPath"), m.get("description"))) + if msgs: + return 2 + print("[telemetry] debug hit ok: %d event(s) valid for %s" % (len(send), mid), file=sys.stderr) + return 0 + + +def number(v): + try: + f = float(v) + except (TypeError, ValueError): + return v + return int(f) if f.is_integer() else f + + +def simplify(r): + """A Data API response as plain rows: [{dimension: value, metric: number}], plus its flags.""" + dims = [h["name"] for h in r.get("dimensionHeaders") or []] + mets = [h["name"] for h in r.get("metricHeaders") or []] + rows = [] + for row in r.get("rows") or []: + d = {k: v.get("value") for k, v in zip(dims, row.get("dimensionValues") or [])} + d.update({k: number(v.get("value")) for k, v in zip(mets, row.get("metricValues") or [])}) + rows.append(d) + meta = r.get("metadata") or {} + return {"rows": rows, "row_count": r.get("rowCount", len(rows)), + "thresholded": bool(meta.get("subjectToThresholding")), "data_loss": bool(meta.get("dataLossFromOtherRow"))} + + +def request(name, events): + """The exact body sent for a fixed report: its definition, filtered to the journey events.""" + body = json.loads(json.dumps(REPORTS[name]["body"])) + if REPORTS[name].get("events"): + body["dimensionFilter"] = {"filter": {"fieldName": "eventName", "inListFilter": {"values": sorted(events)}}} + return body + + +def share(users, thresholded, settled): + """The minimum-sample rule: how much of the next goal usage may drive. Code, not judgment.""" + if not settled: + return {"usage": 0, "research": 1, "reason": "unsettled (realtime) data never drives a goal"} + if thresholded: + return {"usage": 0, "research": 1, "reason": "GA4 thresholded the data: treated as below the minimum sample"} + if users < MIN_USERS: + return {"usage": 0, "research": 1, "reason": "%d users over %d days is below the minimum sample of %d" + % (users, WINDOW_DAYS, MIN_USERS)} + return {"usage": USAGE_SHARE, "research": RESEARCH_FLOOR, + "reason": "%d users over %d days: usage leads, research keeps %d%%" % (users, WINDOW_DAYS, RESEARCH_FLOOR * 100)} + + +def cmd_report(cwd, o): + path, doc, rc = consented(cwd) + if rc: + return rc + endpoint("data") + if not doc.get("property"): + say("BLOCK", "no property yet: `aloop mission telemetry setup` first") + return 2 + tok = need_token() + if not tok: + return 3 + realtime, evs, results = bool(o.get("realtime")), journey_events(cwd), {} + try: + for n in (("realtime",) if realtime else SETTLED_REPORTS): + if REPORTS[n].get("events") and not evs: + continue # only journey events are read; none yet, nothing to read + r = call("POST", "data", "/v1beta/%s:%s" % (doc["property"], REPORTS[n]["api"]), request(n, evs), tok) + results[n] = simplify(r) + except GAError as e: + return ga_failed(e) + sample = sh = None + if not realtime: + rows = results["users"]["rows"] + users = int(rows[0].get("totalUsers") or 0) if rows else 0 + thr = any(x["thresholded"] for x in results.values()) + sample = {"users": users, "min_users": MIN_USERS, "days": WINDOW_DAYS, "thresholded": thr, + "met": users >= MIN_USERS and not thr} + sh = share(users, thr, True) + rid = M.new_id("telemetry") + rep = {"report_id": rid, "telemetry_id": doc["telemetry_id"], "kind": "realtime" if realtime else "settled", + "fetched_at": M.now(), "property": doc["property"], "definitions_sha256": M.digest(REPORTS, drop=()), + "date_range": "last 30 minutes" if realtime else SETTLED, "events": evs, "results": results, + "sample": sample, "share": sh} + M.seal(rep) + p = M.save(M.mdir(cwd, "telemetry", "reports", rid + ".json"), rep) + M.emit(cwd, "telemetry.report", role="mission", refs={"report": M.rel(cwd, p)}, report_id=rid, kind=rep["kind"], + users=(sample or {}).get("users"), sample_met=(sample or {}).get("met")) + if o.get("json"): + print(json.dumps(dict(rep, path=M.rel(cwd, p)), indent=1)) + else: + print(M.rel(cwd, p)) + if sh: + print("[telemetry] usage share %s, research %s: %s" % (sh["usage"], sh["research"], sh["reason"]), + file=sys.stderr) + return 0 + + +def latest_share(cwd): + """The usage/research split for the next goal (M9.11 reads this): from the newest settled report + whose seal holds, and only while the consent covers the terms.""" + path, doc = current(cwd) + none = {"usage": 0, "research": 1, "report": None} + if doc is None or consent_state(cwd, path, doc)[0] != "ok": + return dict(none, reason="telemetry is off, revoked or not consented: research only") + for p in sorted(glob.glob(M.mdir(cwd, "telemetry", "reports", "*.json")), reverse=True): + rep, _ = M.load(p) + if not isinstance(rep, dict) or rep.get("kind") != "settled" or rep.get("sha256") != M.digest(rep): + continue + s = rep.get("sample") or {} + return dict(share(int(s.get("users") or 0), bool(s.get("thresholded")), True), report=rep.get("report_id"), + users=s.get("users")) + return dict(none, reason="no settled usage report yet: research only") + + +def cmd_share(cwd, o): + sh = latest_share(cwd) + if o.get("json"): + print(json.dumps(sh, indent=1)) + else: + print("usage %s · research %s · %s" % (sh["usage"], sh["research"], sh["reason"])) + return 0 + + +def evaluate(cwd): + """(exit, problems, facts) of the structural check. It never reads usage numbers.""" + path, doc = current(cwd) + if doc is None: + return 0, [], {"status": "off", "note": "telemetry off (opt-in): nothing to check"} + bad = M.problems(doc, "telemetry", path) + facts = {"telemetry_id": doc.get("telemetry_id"), "path": M.rel(cwd, path), "status": doc.get("status")} + if bad: + return 2, bad, facts + state, ev = consent_state(cwd, path, doc) + facts["consent"] = state + if state == "none": + return 0, [], dict(facts, note="telemetry off (opt-in): drafted, not consented") + if state == "revoked": + return 0, [], dict(facts, note="telemetry revoked (opt-out) by %s" % (ev.get("data") or {}).get("by")) + if state == "stale": + return 3, ["the consent does not cover the current terms: a human runs " + "`aloop mission telemetry consent --by NAME`"], facts + js, problems = journeys(cwd), [] + evs = sorted({e for _, x in js for e in x}) + for e in evs: + if name_problem(e): + problems.append("journey event %r %s" % (e, name_problem(e))) + found = emitted(cwd, evs) + missing = [{"journey": j, "event": e} for j, x in js for e in x if e not in found] + for m in missing: + problems.append("journey %r step event %r is not emitted by any non-test source file (instrumentation is " + "part of done)" % (m["journey"], m["event"])) + facts.update(events=evs, emitted=found, missing=missing) + if doc.get("stream"): + dh = doc.get("debug_hit") + if not dh: + problems.append("no debug hit yet: `aloop mission telemetry debug-hit`") + elif not dh.get("ok"): + problems.append("the last debug hit failed GA validation: fix it and rerun `aloop mission telemetry debug-hit`") + elif dh.get("measurement_id") != doc["stream"]["measurement_id"] or dh.get("events_sha256") != events_sha(evs): + problems.append("the debug hit is stale (the journey events or stream changed): " + "`aloop mission telemetry debug-hit`") + return (2 if problems else 0), problems, facts + + +def cmd_check(cwd, o): + rc, bad, facts = evaluate(cwd) + if o.get("json"): + print(json.dumps(dict(facts, exit=rc, problems=bad), indent=1)) + for b in bad: + say("ASK" if rc == 3 else "BLOCK", b) + if rc == 0: + print("[telemetry] %s" % (facts.get("note") or "ok: consented, instrumented%s" % ( + ", debug hit fresh" if facts.get("status") == "configured" else "")), file=sys.stderr) + return rc + + +def cmd_show(cwd, o): + path, doc = current(cwd) + if doc is None: + say("ASK", OPT_IN) + return 3 + state = consent_state(cwd, path, doc)[0] + if o.get("json"): + print(json.dumps({"record": doc, "consent": state, "terms_sha256": terms_sha(doc), "share": latest_share(cwd)}, + indent=1)) + return 0 + t = doc["terms"] + print("Telemetry %s (%s, consent: %s)\nMeasures: %s\nPurpose: %s\nCollects: %s\nAgents see: %s\nOthers: %s\n" + "Google's acknowledgement: %s\nScopes: %s" % (doc["telemetry_id"], doc.get("status"), state, t["uri"], + t["purpose"], "; ".join(t["collects"]), t["agents_see"], + t["others"], t["acknowledgement"], " ".join(t["scopes"]))) + if doc.get("property"): + print("Property: %s · stream %s · key events: %s" % (doc["property"], (doc.get("stream") or {}).get( + "measurement_id"), ", ".join(doc.get("key_events") or []) or "none")) + return 0 + + +def cmd_auth(cwd, o): + print(AUTH_STEPS) + return 0 + + +CMDS = {"init": cmd_init, "consent": cmd_consent, "revoke": cmd_revoke, "auth": cmd_auth, "setup": cmd_setup, + "debug-hit": cmd_debug_hit, "report": cmd_report, "share": cmd_share, "show": cmd_show, "check": cmd_check} +FLAGS = ("json", "force", "realtime") + + +def parse(argv): + opts, pos, k = {}, [], None + for a in argv: + if k: + opts[k], k = a, None + elif a.startswith("--"): + k = a[2:].replace("-", "_") + if k in FLAGS: + opts[k], k = True, None + else: + pos.append(a) + if k: + opts[k] = "" + return opts, pos + + +def main(argv, cwd): + fn = CMDS.get(argv[0]) + if fn is None: + say("BLOCK", "unknown command %r (mission telemetry --help)" % argv[0]) + return 64 + o, _ = parse(argv[1:]) + try: + return fn(cwd, o) + except Usage as e: + say("BLOCK", str(e)) + return 64 diff --git a/scripts/test/fixtures/fake-ga.py b/scripts/test/fixtures/fake-ga.py new file mode 100644 index 00000000..8dafa3b6 --- /dev/null +++ b/scripts/test/fixtures/fake-ga.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""A tiny, deterministic fake of the Google Analytics APIs the mission layer's telemetry area +calls (ROADMAP §4.12, M9.6), for tests: no network, no Google account, no tokens. + + /admin/v1beta/… Admin API: accountSummaries, properties, :acknowledgeUserDataCollection, + dataStreams, measurementProtocolSecrets, keyEvents + /data/v1beta/… Data API: :runReport, :runRealtimeReport + /mp/debug/mp/collect the Measurement Protocol validation server + +Point the client at it with ALADDIN_GA_BASE=http://127.0.0.1:. The port is the first +line on stdout. Admin and Data calls need `Authorization: Bearer $FAKE_GA_TOKEN` (default +`fake-token`); the Measurement Protocol takes none. Test controls: + POST /_control {json} merge into the scenario: accounts (0, 1, 2…), users, new_users, + thresholded, events {name: [count, users]}, realtime {name: [count, users]} + GET /_log every request so far: [{method, path, query, auth, body}] +""" +import json +import os +import re +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit + +TOKEN = os.environ.get("FAKE_GA_TOKEN", "fake-token") +ACK = ("I acknowledge that I have the necessary privacy disclosures and rights from my end users for the " + "collection and processing of their data, including the association of such data with the visitation " + "information Google Analytics collects from my site and/or app property.") +NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,39}$") +RESERVED = {"ad_activeview", "ad_click", "ad_exposure", "ad_impression", "ad_query", "ad_reward", "adunit_exposure", + "app_clear_data", "app_exception", "app_install", "app_remove", "app_store_refund", "app_update", + "app_upgrade", "dynamic_link_app_open", "dynamic_link_app_update", "dynamic_link_first_open", "error", + "first_open", "first_visit", "in_app_purchase", "notification_dismiss", "notification_foreground", + "notification_open", "notification_receive", "os_update", "session_start", "session_start_with_rollout", + "user_engagement"} + +LOCK = threading.Lock() +SCENARIO = {"accounts": 1, "users": 0, "new_users": 0, "thresholded": False, "events": {}, "realtime": {}} +DB = {"n": 1000, "properties": {}, "acked": set(), "streams": {}, "secrets": {}, "key_events": {}} +LOG = [] + + +def nid(): + DB["n"] += 1 + return str(DB["n"]) + + +def accounts(): + out = [] + for i in range(int(SCENARIO["accounts"])): + acct = "accounts/%d" % (100 + i) + props = [{"property": p, "displayName": d["displayName"]} for p, d in sorted(DB["properties"].items()) + if d["parent"] == acct] + out.append({"name": "accountSummaries/%d" % (100 + i), "account": acct, "displayName": "Account %d" % i, + "propertySummaries": props}) + return out + + +def err(code, status, message): + return code, {"error": {"code": code, "message": message, "status": status}} + + +def admin(method, parts, body): + if parts == ["accountSummaries"] and method == "GET": + return 200, {"accountSummaries": accounts()} + if parts == ["properties"] and method == "POST": + if not any(a["account"] == body.get("parent") for a in accounts()): + return err(403, "PERMISSION_DENIED", "The caller does not have permission") + for k in ("displayName", "timeZone"): + if not body.get(k): + return err(400, "INVALID_ARGUMENT", "%s is required" % k) + name = "properties/" + nid() + DB["properties"][name] = dict(body, name=name, propertyType="PROPERTY_TYPE_ORDINARY") + return 200, DB["properties"][name] + if len(parts) < 2 or parts[0] != "properties": + return err(404, "NOT_FOUND", "unknown admin path") + prop = "properties/" + parts[1].split(":")[0] + if prop not in DB["properties"]: + return err(403, "PERMISSION_DENIED", "The caller does not have permission") + if len(parts) == 2 and parts[1].endswith(":acknowledgeUserDataCollection") and method == "POST": + if body.get("acknowledgement") != ACK: + return err(400, "INVALID_ARGUMENT", "acknowledgement does not match the required text") + DB["acked"].add(prop) + return 200, {} + if len(parts) == 2 and method == "GET": + return 200, DB["properties"][prop] + rest = parts[2:] + if rest == ["dataStreams"]: + if method == "GET": + return 200, {"dataStreams": [s for s in DB["streams"].values() if s["name"].startswith(prop + "/")]} + if body.get("type") != "WEB_DATA_STREAM" or not (body.get("webStreamData") or {}).get("defaultUri"): + return err(400, "INVALID_ARGUMENT", "a web data stream needs webStreamData.defaultUri") + sid = nid() + s = {"name": "%s/dataStreams/%s" % (prop, sid), "type": "WEB_DATA_STREAM", "displayName": body.get("displayName"), + "webStreamData": {"defaultUri": body["webStreamData"]["defaultUri"], "measurementId": "G-FAKE" + sid}} + DB["streams"][s["name"]] = s + return 200, s + if len(rest) == 3 and rest[0] == "dataStreams" and rest[2] == "measurementProtocolSecrets": + stream = "%s/dataStreams/%s" % (prop, rest[1]) + if stream not in DB["streams"]: + return err(404, "NOT_FOUND", "no such data stream") + if method == "GET": + return 200, {"measurementProtocolSecrets": [s for s in DB["secrets"].values() if s["name"].startswith(stream + "/")]} + if prop not in DB["acked"]: + return err(403, "PERMISSION_DENIED", "User data collection acknowledgement is required") + sec = {"name": "%s/measurementProtocolSecrets/%s" % (stream, nid()), "displayName": body.get("displayName"), + "secretValue": "fake-secret-" + str(DB["n"])} + DB["secrets"][sec["name"]] = sec + return 200, sec + if rest == ["keyEvents"]: + if method == "GET": + return 200, {"keyEvents": [k for k in DB["key_events"].values() if k["name"].startswith(prop + "/")]} + if body.get("countingMethod") not in ("ONCE_PER_EVENT", "ONCE_PER_SESSION"): + return err(400, "INVALID_ARGUMENT", "countingMethod is required") + if any(k["eventName"] == body.get("eventName") for k in DB["key_events"].values() if k["name"].startswith(prop + "/")): + return err(409, "ALREADY_EXISTS", "key event exists") + k = {"name": "%s/keyEvents/%s" % (prop, nid()), "eventName": body.get("eventName"), + "countingMethod": body["countingMethod"]} + DB["key_events"][k["name"]] = k + return 200, k + return err(404, "NOT_FOUND", "unknown admin path") + + +def wanted(body): + f = ((body.get("dimensionFilter") or {}).get("filter") or {}) + return set((f.get("inListFilter") or {}).get("values") or []) if f.get("fieldName") == "eventName" else None + + +def report(body, source): + dims = [d["name"] for d in body.get("dimensions") or []] + mets = [m["name"] for m in body.get("metrics") or []] + rows = [] + if dims == ["eventName"]: + keep = wanted(body) + for name, (count, users) in sorted(SCENARIO[source].items()): + if keep is None or name in keep: + vals = {"eventCount": count, "totalUsers": users, "activeUsers": users} + rows.append({"dimensionValues": [{"value": name}], + "metricValues": [{"value": str(vals.get(m, 0))} for m in mets]}) + elif not dims: + vals = {"totalUsers": SCENARIO["users"], "activeUsers": SCENARIO["users"], "newUsers": SCENARIO["new_users"]} + rows.append({"metricValues": [{"value": str(vals.get(m, 0))} for m in mets]}) + else: + return err(400, "INVALID_ARGUMENT", "fake-ga supports no dimensions or eventName only") + out = {"dimensionHeaders": [{"name": d} for d in dims], + "metricHeaders": [{"name": m, "type": "TYPE_INTEGER"} for m in mets], + "rows": rows, "rowCount": len(rows), "kind": "analyticsData#" + ("runReport" if source == "events" else "runRealtimeReport")} + if source == "events": + out["metadata"] = {"currencyCode": "USD", "timeZone": "Etc/UTC"} + if SCENARIO["thresholded"]: + out["metadata"]["subjectToThresholding"] = True + return 200, out + + +def data(method, parts, body): + if method != "POST" or len(parts) != 2 or parts[0] != "properties": + return err(404, "NOT_FOUND", "unknown data path") + pid, _, verb = parts[1].partition(":") + if "properties/" + pid not in DB["properties"]: + return err(403, "PERMISSION_DENIED", "User does not have sufficient permissions for this property") + if verb == "runReport": + if not body.get("dateRanges"): + return err(400, "INVALID_ARGUMENT", "dateRanges is required") + return report(body, "events") + if verb == "runRealtimeReport": + return report(body, "realtime") + return err(404, "NOT_FOUND", "unknown data method") + + +def mp_debug(query, body): + msgs = [] + mid, secret = (query.get("measurement_id") or [""])[0], (query.get("api_secret") or [""])[0] + streams = [s for s in DB["streams"].values() if s["webStreamData"]["measurementId"] == mid] + if not streams: + msgs.append({"fieldPath": "measurement_id", "description": "Unknown measurement_id %s" % mid, + "validationCode": "VALUE_INVALID"}) + elif not any(s["secretValue"] == secret and s["name"].startswith(streams[0]["name"] + "/") for s in DB["secrets"].values()): + msgs.append({"fieldPath": "api_secret", "description": "api_secret is not valid for this stream", + "validationCode": "VALUE_INVALID"}) + if not body.get("client_id"): + msgs.append({"fieldPath": "client_id", "description": "client_id is required", "validationCode": "VALUE_REQUIRED"}) + events = body.get("events") or [] + if not 1 <= len(events) <= 25: + msgs.append({"fieldPath": "events", "description": "1 to 25 events per request", "validationCode": "VALUE_INVALID"}) + for i, e in enumerate(events): + n = e.get("name", "") + if not NAME.match(n): + msgs.append({"fieldPath": "events", "description": "Event at index: [%d] has invalid name [%s]." % (i, n), + "validationCode": "NAME_INVALID"}) + elif n in RESERVED or n.startswith(("_", "firebase_", "ga_", "google_", "gtag.")): + msgs.append({"fieldPath": "events", "description": "Event at index: [%d] has name [%s] which is reserved." % (i, n), + "validationCode": "NAME_RESERVED"}) + return 200, {"validationMessages": msgs} + + +class H(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def reply(self, code, obj): + b = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(b))) + self.end_headers() + self.wfile.write(b) + + def handle_any(self, method): + u = urlsplit(self.path) + n = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(n) if n else b"" + try: + body = json.loads(raw) if raw else {} + except ValueError: + return self.reply(400, {"error": {"code": 400, "message": "body is not JSON", "status": "INVALID_ARGUMENT"}}) + with LOCK: + if u.path == "/_log": + return self.reply(200, LOG) + if u.path == "/_control": + SCENARIO.update(body) + return self.reply(200, SCENARIO) + auth = self.headers.get("Authorization") or "" + LOG.append({"method": method, "path": u.path, "query": u.query, "auth": auth, "body": body}) + parts = [p for p in u.path.split("/") if p] + if parts[:2] == ["mp", "debug"] and parts[2:] == ["mp", "collect"] and method == "POST": + return self.reply(*mp_debug(parse_qs(u.query), body)) + if parts[:1] in (["admin"], ["data"]): + if auth != "Bearer " + TOKEN: + return self.reply(*err(401, "UNAUTHENTICATED", "Request had invalid authentication credentials.")) + if parts[1:2] != ["v1beta"]: + return self.reply(*err(404, "NOT_FOUND", "unknown version")) + fn = admin if parts[0] == "admin" else data + return self.reply(*fn(method, parts[2:], body)) + return self.reply(*err(404, "NOT_FOUND", "unknown path %s" % u.path)) + + def do_GET(self): + self.handle_any("GET") + + def do_POST(self): + self.handle_any("POST") + + +if __name__ == "__main__": + srv = ThreadingHTTPServer(("127.0.0.1", 0), H) + print(srv.server_address[1], flush=True) + try: + srv.serve_forever() + except KeyboardInterrupt: + sys.exit(0) diff --git a/scripts/test/mission-telemetry.test.py b/scripts/test/mission-telemetry.test.py new file mode 100644 index 00000000..f1fa4035 --- /dev/null +++ b/scripts/test/mission-telemetry.test.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""ROADMAP §4.12 (M9.6): usage telemetry through Google Analytics 4. It is opt-in: nothing +reaches the network until a human consents to exact terms, and a revoke stops it. Setup goes +through the Admin API (property, web stream, user-data acknowledgement, Measurement Protocol +secret, a key event per journey), a debug hit proves the wiring, fixed Data API reports read +settled data, and the minimum-sample rule decides how much of a goal usage may drive. Usage +informs goals and is never a gate; the check is structural (consent, every journey step's +event is emitted, a fresh debug hit). All network is a local fake (fixtures/fake-ga.py). + python3 scripts/test/mission-telemetry.test.py +""" +import importlib.util +import json +import os +import stat +import subprocess +import sys +import tempfile +import urllib.request + +ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "plugin")) +BIN = os.path.join(ROOT, "bin") +MI = os.path.join(BIN, "mission") +T = tempfile.mkdtemp(prefix="aloop-telemetry-") +PASS = [0] +sys.path.insert(0, BIN) +import _mission as M # noqa: E402 + +spec = importlib.util.spec_from_file_location("telemetry", os.path.join(ROOT, "mission", "telemetry.py")) +TM = importlib.util.module_from_spec(spec) +spec.loader.exec_module(TM) + +FAKE = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), "fixtures", "fake-ga.py")], + stdout=subprocess.PIPE, text=True) +BASE = "http://127.0.0.1:%s" % FAKE.stdout.readline().strip() +ENV = dict(os.environ, HOME=os.path.join(T, "home"), ALADDIN_GA_BASE=BASE, ALADDIN_GA_ACCESS_TOKEN="fake-token") +for k in ("AGENTIC_HOST", "AGENTIC_GA_ACCESS_TOKEN", "AGENTIC_GA_TOKEN_COMMAND", "AGENTIC_GA_BASE"): + ENV.pop(k, None) + + +def ok(name): + PASS[0] += 1 + print("ok -", name) + + +def run(*args, env=None): + return subprocess.run([MI, "telemetry"] + list(args), cwd=T, env=env or ENV, capture_output=True, text=True) + + +def fake(path, body=None): + req = urllib.request.Request(BASE + path, data=None if body is None else json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, method="GET" if body is None else "POST") + with urllib.request.build_opener(urllib.request.ProxyHandler({})).open(req, timeout=10) as r: + return json.load(r) + + +def calls(): + return fake("/_log") + + +def events(kind=None): + p = os.path.join(T, ".aladdin", "logs", "events.jsonl") + evs = [json.loads(l) for l in open(p)] if os.path.exists(p) else [] + return [e for e in evs if kind is None or e["event"] == kind] + + +def record(): + recs = M.artifacts(T, "telemetry") + assert len(recs) == 1, recs + return recs[0] + + +def journey(name, steps, status="active"): + jid = M.new_id("journey") + M.save(M.path_of(T, "journey", jid), {"journey_id": jid, "status": status, "created_at": M.now(), "name": name, + "steps": [{"name": s, "event": s} for s in steps]}) + return jid + + +def write(rel, text): + p = os.path.join(T, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + open(p, "w").write(text) + subprocess.run(["git", "add", rel], cwd=T, check=True) + + +try: + subprocess.run(["git", "init", "-q"], cwd=T, check=True) + + # 1) off by default: the check passes with nothing to check, and no command reaches the network + r = run("check") + assert r.returncode == 0 and "off (opt-in)" in r.stderr, (r.returncode, r.stderr) + for cmd in (["setup"], ["debug-hit"], ["report"], ["consent", "--by", "Adeel"]): + r = run(*cmd) + assert r.returncode == 3 and "ASK[telemetry]" in r.stderr and "telemetry init" in r.stderr, (cmd, r.stderr) + assert calls() == [], calls() + ok("telemetry is off until a human opts in: check passes, every network command asks, zero requests") + + # 2) init drafts exact terms (sealed, harness id, GA's verbatim acknowledgement); consent covers them + assert run("init").returncode == 64 # what is measured is part of the terms + assert run("init", "--uri", "ftp://todo").returncode == 64 + r = run("init", "--uri", "https://todo.example", "--time-zone", "Europe/Stockholm") + assert r.returncode == 0, r.stderr + path, doc = record() + assert M.problems(doc, "telemetry", path) == [], M.problems(doc, "telemetry", path) + assert doc["status"] == "off" and doc["telemetry_id"].startswith("tlm_") and doc["sha256"] == M.digest(doc) + assert doc["terms"]["acknowledgement"] == TM.ACKNOWLEDGEMENT and "never a gate" in doc["terms"]["purpose"] + assert doc["terms"]["uri"] == "https://todo.example" and set(TM.SCOPES) == set(doc["terms"]["scopes"]) + assert run("init", "--uri", "https://other.example").returncode == 2 # one record; terms change by re-init --force + r = run("setup") + assert r.returncode == 3 and "consent" in r.stderr, r.stderr + assert run("consent").returncode == 64 # who consents is required + r = run("consent", "--by", "Adeel", "--reason", "my own app, day one") + assert r.returncode == 0, r.stderr + path, doc = record() + ev = events("telemetry.consent") + assert len(ev) == 1 and ev[0]["role"] == "human" and ev[0]["data"]["by"] == "Adeel", ev + assert ev[0]["data"]["sha256"] == TM.terms_sha(doc) and ev[0]["data"]["subject"] == M.rel(T, path) + assert doc["status"] == "consented" and run("check").returncode == 0 and calls() == [] + ok("init seals the terms (uri, scopes, GA's acknowledgement); consent is a human event on exactly those terms") + + # 3) consent covers exact terms: a changed term asks again and setup refuses; reverting restores it + doc["terms"]["uri"] = "https://elsewhere.example" + M.seal(doc) + M.save(path, doc) + r = run("check") + assert r.returncode == 3 and "does not cover" in r.stderr, r.stderr + assert run("setup").returncode == 3 and calls() == [] + doc["terms"]["uri"] = "https://todo.example" + M.seal(doc) + M.save(path, doc) + assert run("check").returncode == 0 + raw = json.load(open(path)) + raw["property"] = "properties/999" # a hand edit after sealing + json.dump(raw, open(path, "w")) + r = run("check") + assert r.returncode == 2 and "sha256 does not match" in r.stderr, r.stderr + M.seal(doc) + M.save(path, doc) + ok("consent covers exact terms (changed terms ask again, hand edits block); no request was made") + + # 4) no Google credentials: setup asks the human once, with the pastable sign-in, and sends nothing + env = dict(ENV, ALADDIN_GA_TOKEN_COMMAND="false") + env.pop("ALADDIN_GA_ACCESS_TOKEN") + r = run("setup", env=env) + assert r.returncode == 3 and "gcloud auth application-default login" in r.stderr, r.stderr + assert "analytics.edit" in r.stderr and calls() == [] + r = run("auth") + assert r.returncode == 0 and "--client-id-file" in r.stdout and "analyticsadmin.googleapis.com" in r.stdout + r = run("setup", env=dict(ENV, ALADDIN_GA_BASE="https://evil.example")) + assert r.returncode == 64 and "loopback" in r.stderr and calls() == [], r.stderr + ok("no token: setup asks with the one-time sign-in; a non-loopback endpoint override is refused") + + # 5) setup through the Admin API: account choice is the human's when it is ambiguous or missing + fake("/_control", {"accounts": 0}) + r = run("setup") + assert r.returncode == 3 and "analytics.google.com" in r.stderr, r.stderr + fake("/_control", {"accounts": 2}) + r = run("setup") + assert r.returncode == 3 and "--account accounts/100" in r.stderr and "accounts/101" in r.stderr, r.stderr + assert all(c["method"] == "GET" for c in calls()) # asking never creates anything + fake("/_control", {"accounts": 1}) + signup = journey("sign up", ["signup_started", "signup_done"]) + journey("add a todo", ["todo_add_opened", "todo_added"]) + journey("old idea", ["old_step"], status="retired") + r = run("setup") + assert r.returncode == 0, r.stderr + path, doc = record() + assert doc["status"] == "configured" and doc["property"].startswith("properties/") and M.problems(doc, "telemetry", path) == [] + assert doc["stream"]["measurement_id"].startswith("G-") and doc["stream"]["default_uri"] == "https://todo.example" + assert sorted(doc["key_events"]) == ["signup_done", "todo_added"], doc["key_events"] + log = calls() + posts = [c["path"].split("/v1beta/")[1] for c in log if c["method"] == "POST"] + assert posts[0] == "properties" and posts[1].endswith(":acknowledgeUserDataCollection"), posts + assert [p.rsplit("/", 1)[-1] for p in posts[2:]] == ["dataStreams", "measurementProtocolSecrets", "keyEvents", "keyEvents"], posts + prop_body = [c["body"] for c in log if c["path"].endswith("/v1beta/properties") and c["method"] == "POST"][0] + assert prop_body["parent"] == "accounts/100" and prop_body["timeZone"] == "Europe/Stockholm", prop_body + ack = [c["body"] for c in log if c["path"].endswith(":acknowledgeUserDataCollection")][0] + assert ack == {"acknowledgement": TM.ACKNOWLEDGEMENT} + assert all(c["auth"] == "Bearer fake-token" for c in log) + secret = [c for c in log if c["path"].endswith("measurementProtocolSecrets") and c["method"] == "POST"] + assert len(secret) == 1 + sfile = TM.secret_file(T, doc["telemetry_id"]) + assert os.path.isfile(sfile) and stat.S_IMODE(os.stat(sfile).st_mode) == 0o600, sfile + assert M.rel(T, sfile).startswith(".aladdin" + os.sep), sfile + value = json.load(open(sfile))["api_secret"] + for d, _, fs in os.walk(os.path.join(T, "docs")): + for f in fs: + assert value not in open(os.path.join(d, f)).read(), "the MP secret leaked into %s" % f + assert [e["data"].get("action") for e in events("telemetry.setup")] == ["setup"] + ok("setup: property, acknowledgement, web stream, MP secret, a key event per active journey; secret 0600 outside docs/") + + # 6) setup is idempotent: a rerun finds everything and creates nothing + n = len(calls()) + r = run("setup") + assert r.returncode == 0, r.stderr + again = calls()[n:] + assert again and not [c for c in again if c["method"] == "POST" and not c["path"].endswith(":acknowledgeUserDataCollection")], again + assert record()[1]["property"] == doc["property"] + ok("a second setup reuses the property, stream, secret and key events (no duplicate creates)") + + # 7) the static emit check: every active journey step's event must be emitted by source code + r = run("check") + assert r.returncode == 2 and "signup_started" in r.stderr and "not emitted" in r.stderr, r.stderr + write("src/app.js", "gtag('event', 'signup_started');\ngtag('event', \"signup_done\");\ntrack(`todo_add_opened`);\n") + write("src/app.test.js", "expect(sent).toContain('todo_added')\n") # a test mentioning it does not count + write("docs/notes.md", "todo_added is emitted when … \n") # nor do docs + r = run("check", "--json") + assert r.returncode == 2, r.stderr + rep = json.loads(r.stdout) + assert [m["event"] for m in rep["missing"]] == ["todo_added"] and rep["missing"][0]["journey"] == "add a todo", rep + assert "old_step" not in r.stderr # retired journeys are not checked + write("src/todo.js", "export const add = () => gtag('event', 'todo_added');\n") + r = run("check", "--json") + assert r.returncode == 2 and "debug hit" in r.stderr, r.stderr # instrumented, not yet proven + assert json.loads(r.stdout)["emitted"]["todo_added"] == "src/todo.js:1", r.stdout + jid = journey("bad names", ["session_start", "2fast", "x" * 41]) + r = run("check") + assert r.returncode == 2 and "session_start" in r.stderr and "reserved" in r.stderr and "2fast" in r.stderr, r.stderr + os.remove(M.path_of(T, "journey", jid)) + ok("emit check: each active journey step's event appears in non-test source; GA name rules apply") + + # 8) the debug hit: journey events go to GA's validation server (no OAuth token sent), and stay fresh + n = len(calls()) + r = run("debug-hit") + assert r.returncode == 0, r.stderr + hits = calls()[n:] + assert len(hits) == 1 and hits[0]["path"] == "/mp/debug/mp/collect" and hits[0]["auth"] == "", hits + assert "measurement_id=G-" in hits[0]["query"] and "api_secret=" in hits[0]["query"] + assert sorted(e["name"] for e in hits[0]["body"]["events"]) == sorted(TM.journey_events(T)), hits[0]["body"] + assert hits[0]["body"]["client_id"] == TM.DEBUG_CLIENT_ID + r = run("check") + assert r.returncode == 0, r.stderr + journey("share a list", ["list_shared"]) + write("src/share.js", "gtag('event', 'list_shared')\n") + r = run("check") + assert r.returncode == 2 and "stale" in r.stderr, r.stderr # a new event needs a new hit + assert run("debug-hit").returncode == 0 and run("check").returncode == 0 + sf = json.load(open(sfile)) + json.dump(dict(sf, api_secret="wrong"), open(sfile, "w")) + r = run("debug-hit") + assert r.returncode == 2 and "api_secret" in r.stderr, r.stderr # GA's validation messages surface + assert run("check").returncode == 2 # a failed hit is not a pass + json.dump(sf, open(sfile, "w")) + assert run("debug-hit").returncode == 0 and run("check").returncode == 0 + ok("debug hit: every journey event validated by GA's debug endpoint; a new event or a failed hit blocks") + + # 9) fixed reports on settled data; the minimum-sample rule decides usage's share; never a gate + fake("/_control", {"users": 12, "new_users": 9, "events": {"signup_done": [14, 9], "todo_added": [40, 7], "page_view": [99, 12]}}) + n = len(calls()) + r = run("report", "--json") + assert r.returncode == 0, r.stderr + rep = json.loads(r.stdout) + sent = [c["body"] for c in calls()[n:]] + assert all(b["dateRanges"] == [{"startDate": "30daysAgo", "endDate": "3daysAgo"}] for b in sent), sent + assert {json.dumps(TM.request("users", []), sort_keys=True), json.dumps(TM.request("journeys", TM.journey_events(T)), sort_keys=True)} \ + == {json.dumps(b, sort_keys=True) for b in sent} + assert rep["kind"] == "settled" and rep["sample"] == {"users": 12, "min_users": 50, "days": 28, "thresholded": False, "met": False} + assert rep["share"]["usage"] == 0 and rep["share"]["research"] == 1, rep["share"] + assert "page_view" not in json.dumps(rep["results"]["journeys"]), rep # only journey events are read + assert rep["definitions_sha256"] == M.digest(TM.REPORTS, drop=()) + assert os.path.dirname(rep["path"]) == os.path.join("docs", "agents", "mission", "telemetry", "reports") + assert run("check").returncode == 0 # below the sample: still passes + fake("/_control", {"users": 80, "new_users": 50}) + rep = json.loads(run("report", "--json").stdout) + assert rep["sample"]["met"] and rep["share"] == {"usage": 0.7, "research": 0.3, "reason": rep["share"]["reason"]}, rep + fake("/_control", {"thresholded": True}) + rep = json.loads(run("report", "--json").stdout) + assert rep["sample"]["thresholded"] and not rep["sample"]["met"] and rep["share"]["usage"] == 0, rep + fake("/_control", {"thresholded": False, "realtime": {"todo_added": [3, 2]}}) + r = run("report", "--realtime", "--json") + rt = json.loads(r.stdout) + assert r.returncode == 0 and rt["kind"] == "realtime" and rt["share"] is None, rt # realtime never drives a goal + assert [c["path"] for c in calls()][-1].endswith(":runRealtimeReport") + r = run("share", "--json") + sh = json.loads(r.stdout) + assert r.returncode == 0 and sh["usage"] == 0 and sh["report"] != rt["report_id"], sh # latest SETTLED report + assert len(events("telemetry.report")) == 4 and run("check").returncode == 0 + ok("reports: fixed definitions on settled data (≥48h), usage leads only past 50 users/28d, thresholding = below") + + # 10) the minimum-sample rule is code with fixed boundaries; report definitions stay aggregate-only + assert TM.share(49, False, True)["usage"] == 0 and TM.share(50, False, True)["usage"] == 0.7 + assert TM.share(500, True, True)["usage"] == 0 and TM.share(500, False, False)["usage"] == 0 + assert TM.share(10 ** 6, False, True)["research"] >= 0.3 + for name, d in TM.REPORTS.items(): + dims = [x["name"] for x in d["body"].get("dimensions", [])] + assert set(dims) <= TM.AGGREGATE_DIMENSIONS, (name, dims) + assert not TM.AGGREGATE_DIMENSIONS & {"customUser:id", "userId", "clientId", "city"} + ok("share(): 49 users → 0, 50 → 0.7, thresholded or unsettled → 0, research ≥ 30%; reports read aggregates only") + + # 11) a tampered report is ignored by share(); revoke stops all network and turns the check off + reports = sorted(os.listdir(M.mdir(T, "telemetry", "reports"))) + assert len(reports) == 4 + sh_before = json.loads(run("share", "--json").stdout)["report"] + p = M.mdir(T, "telemetry", "reports", sh_before + ".json") + raw = json.load(open(p)) + raw["sample"]["users"] = 10 ** 6 + json.dump(raw, open(p, "w")) + assert json.loads(run("share", "--json").stdout)["report"] != sh_before + assert run("revoke").returncode == 64 + r = run("revoke", "--by", "Adeel", "--reason", "pausing") + assert r.returncode == 0, r.stderr + assert record()[1]["status"] == "revoked" and len(events("telemetry.revoked")) == 1 + n = len(calls()) + for cmd in (["setup"], ["debug-hit"], ["report"]): + r = run(*cmd) + assert r.returncode == 3 and "revoked" in r.stderr, (cmd, r.stderr) + assert calls()[n:] == [] + r = run("check") + assert r.returncode == 0 and "revoked" in r.stderr, r.stderr + assert run("consent", "--by", "Adeel").returncode == 0 and record()[1]["status"] == "configured" + assert run("check").returncode == 0 + ok("revoke: setup, debug-hit and report refuse with no request; a fresh consent resumes; tampered reports skipped") + + # 12) discovery and help: the area is found by name, its docstring is its help + r = subprocess.run([MI, "--list"], cwd=T, env=ENV, capture_output=True, text=True) + assert "telemetry" in r.stdout.split(), r.stdout + r = run("--help") + assert r.returncode == 0 and "never a gate" in r.stdout and "mission telemetry setup" in r.stdout, r.stdout + assert run("nope").returncode == 64 + ok("`mission telemetry` is discovered; --help is the module docstring; unknown subcommands are usage errors") +finally: + FAKE.terminate() + FAKE.wait(timeout=10) + +print("mission-telemetry: %d checks passed" % PASS[0]) From 18a194ce5a9d0b265e0f302c3d1388ef2feef930 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Tue, 20 Oct 2026 02:43:50 +0000 Subject: [PATCH 2/2] M9.6: fix review findings in GA4 telemetry Consent counts only human events and must match the sealed terms; reports and the debug hit are scoped to the recorded stream; redirects are refused so the token never leaves loopback; --help and unknown flags never touch the network; the emit check scans the worker's own worktree; revoke deletes the local secret and names the live tag. Sign-in moves to service-account impersonation (no 7-day refresh). The suite grows to 13 checks against the fake GA. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012Ro7iYo3scQt6vAuAWdYpV --- docs/ROADMAP.md | 4 +- plugin/mission/schemas/telemetry.json | 3 +- plugin/mission/telemetry.py | 431 +++++++++++++++++-------- scripts/test/fixtures/fake-ga.py | 57 ++-- scripts/test/mission-telemetry.test.py | 317 ++++++++++++------ 5 files changed, 546 insertions(+), 266 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 51f86b9e..bedc2712 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -408,7 +408,7 @@ Each task flips only its own row here and in TRACKING.md. | M9.3 | Journeys and metrics: steps, pass criteria, events, metric query definitions, source refs; `diff` blocks removing or weakening without approval; `run` through a pinned runner; all-green-at-baseline refused | M9.0 | `plugin/mission/journeys.py`, `schemas/journey.json`, `scripts/test/mission-journeys.test.py` | yes | open | | M9.4 | Ethics: review per mission and goal, three tiers, gray consent bound to the goal's sha256, hard line never unlocked | M9.0 | `plugin/mission/ethics.py`, `schemas/ethics.json`, `scripts/test/mission-ethics.test.py` | yes | open | | M9.5 | Judges: rubric setup check and an agreement verdict from recorded runs | M9.0 | `plugin/mission/judge.py`, `schemas/rubric.json`, `scripts/test/mission-judge.test.py` | yes | open | -| M9.6 | Telemetry (GA4): opt-in consent, Admin API setup, Measurement Protocol debug hit, fixed Data API reports (+ realtime), minimum-sample rule, static emit check; network faked in tests | M9.0 | `plugin/mission/telemetry.py`, `schemas/telemetry.json`, `scripts/test/mission-telemetry.test.py`, `scripts/test/fixtures/fake-ga.py` | yes | done: `mission-telemetry.test.py` (12 checks, fake GA) | +| M9.6 | Telemetry (GA4): opt-in consent, Admin API setup, Measurement Protocol debug hit, fixed Data API reports (+ realtime), minimum-sample rule, static emit check; network faked in tests | M9.0 | `plugin/mission/telemetry.py`, `schemas/telemetry.json`, `scripts/test/mission-telemetry.test.py`, `scripts/test/fixtures/fake-ga.py` | yes | done: `mission-telemetry.test.py` (13 checks, fake GA); consent counts only `role=human` events, so its provenance rides M9.17 | | M9.7 | Principles in two variants with tension pairs; `principles show --mode`, hashed and injected into goals | M9.0 | `plugin/mission/principles.{md,py}`, `scripts/test/mission-principles.test.py` | yes | open | | M9.8 | Loop: `tick` (the state machine above), `due` (zero-token cadence check), `cron` (host routine or crontab line), budget caps from the token ledger | M9.0 | `plugin/mission/{tick,due,cron}.py`, `scripts/test/mission-tick.test.py` | yes | open | | M9.9 | Roles and wiring: the six personas, hook matchers, model routes, host install, eval suites, role counts in docs; `gate-mission-scope` (role path guards) and `gate-mission-artifact` (a role's artifact passes its check at stop) | M9.0 | `plugin/agents/` (6 new files), `plugin/hooks/hooks.json`, `plugin/hosts/`, `plugin/evals/`, `plugin/bin/gate-mission-*`, role counts in README / OVERVIEW / plugin docs | yes (the only track touching shared lists) | open | @@ -419,7 +419,7 @@ Each task flips only its own row here and in TRACKING.md. | M9.14 | GA4 live on the owner's account (one Google sign-in) | M9.6 | — | — | needs the owner: `aloop mission telemetry auth` prints the steps, then `init`, `consent`, `setup`, `debug-hit` | | M9.15 | Native voice mode | the owner's scope (spec cut off) | — | — | open question | | M9.16 | Docs: OVERVIEW, README, CHANGELOG, site | M9.13 | shared docs | — | blocked | -| M9.17 | Approval provenance: an approval names the captured human prompt (the UserPromptSubmit log), so a supervisor cannot approve its own mission change. Reason: found while building M9.0, an approval is only as strong as `--by` | M9.0 | `plugin/bin/_mission.py` (`approve`), its test | yes | open | +| M9.17 | Approval provenance: an approval names the captured human prompt (the UserPromptSubmit log), so a supervisor cannot approve its own mission change. Reason: found while building M9.0, an approval is only as strong as `--by`; telemetry consent (M9.6) inherits the same gap | M9.0 | `plugin/bin/_mission.py` (`approve`), its test | yes | open | **Parallel tracks** (one thread each, from M9.0 on master): (1) Research + Judges, M9.1 + M9.5; (2) Baseline + Journeys → Assess/Evaluate, M9.2 + M9.3 → M9.10; (3) Ethics + Principles → Goals, diff --git a/plugin/mission/schemas/telemetry.json b/plugin/mission/schemas/telemetry.json index a853b319..bb218e44 100644 --- a/plugin/mission/schemas/telemetry.json +++ b/plugin/mission/schemas/telemetry.json @@ -11,7 +11,7 @@ "mission_id": {"type": ["string", "null"]}, "terms": { "type": "object", - "required": ["uri", "purpose", "collects", "agents_see", "acknowledgement", "scopes", "others"], + "required": ["uri", "purpose", "collects", "agents_see", "acknowledgement", "credential", "scopes", "others"], "additionalProperties": false, "properties": { "uri": {"type": "string", "pattern": "^https?://[^\\s/]+"}, @@ -19,6 +19,7 @@ "collects": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, "agents_see": {"type": "string", "minLength": 1}, "acknowledgement": {"type": "string", "minLength": 1}, + "credential": {"type": "string", "minLength": 1}, "scopes": {"type": "array", "minItems": 1, "items": {"type": "string", "pattern": "^https://www\\.googleapis\\.com/auth/analytics"}}, "others": {"type": "string", "minLength": 1} } diff --git a/plugin/mission/telemetry.py b/plugin/mission/telemetry.py index 12961d4d..6e028c7c 100644 --- a/plugin/mission/telemetry.py +++ b/plugin/mission/telemetry.py @@ -3,36 +3,41 @@ Real usage drives most of a later goal once there is enough of it, so it is captured from day one through GA4, which the agents set up through Google's APIs: no server to host, nothing to buy. Nothing reaches the network until a human consents to the exact terms below, and a -revoke stops it. Agents see aggregate counts from fixed report definitions only. +revoke stops the agents' use of it. Agents see aggregate counts of the consented stream only. Usage: mission telemetry init --uri URL [--name NAME] [--time-zone TZ] [--force] draft the terms (what is measured, why, what agents see, GA's user-data acknowledgement, - the OAuth scopes) as a sealed record, status off. Nothing leaves the machine. - --force re-drafts the terms of the existing record (consent is asked again). + the credential) as a sealed record. Nothing leaves the machine. --force re-drafts the + terms of the existing record; changed terms need a new consent. mission telemetry consent --by NAME [--reason TEXT] the human opts in to exactly these terms: telemetry.consent names their sha256 mission telemetry revoke --by NAME [--reason TEXT] - opt out: setup, debug-hit and report refuse until a new consent + opt out: setup, debug-hit and report refuse, usage stops driving goals, and the local + Measurement Protocol secret is deleted. The app's own GA tag is the app's code: check + names the files that still carry it. mission telemetry auth the one-time Google sign-in, as pastable steps (the only human step) mission telemetry setup [--account accounts/N] [--property properties/N] - Admin API, idempotent: find or create the property, acknowledge user-data collection, - a web data stream for the uri, a Measurement Protocol secret (kept in .aladdin/state/, - never in docs/) and a key event for each active journey's last step + Admin API, idempotent: find or create the property (--account picks or replaces it), + acknowledge user-data collection, a web data stream for the uri, a Measurement + Protocol secret (kept in .aladdin/state/, never in docs/) and a key event for each + active journey's last step mission telemetry debug-hit - send every active journey step's event to GA's validation server; the result is recorded + confirm the local secret is live on the stream (Admin API; Google's validation server + does not check secrets), then send every active journey step's event to that server mission telemetry report [--realtime] [--json] - the fixed Data API reports: settled (28 days ending 3 days ago, so ≥48h old) or realtime - (the last 30 minutes, a quick signal only); written to docs/agents/mission/telemetry/reports/ + the fixed Data API reports, scoped to the stream: settled (28 days ending 3 days ago, + so ≥48h old) or realtime (a quick signal only); written to docs/agents/mission/telemetry/reports/ mission telemetry share [--json] how much of the next goal usage may drive, from the latest settled report: 0 below 50 users over 28 days or when GA thresholded the data, else 0.7 (research keeps 0.3) mission telemetry show [--json] mission telemetry check [--json] the structural check: consent covers the current terms; once consented, every active - journey step's event is emitted by non-test source code; once configured, a passing - debug hit covers the current events. Usage numbers never change its exit. + journey step's event is emitted by non-test code in this checkout; once set up, the + app carries the stream's measurement id, the key events match the journeys and a + passing debug hit covers the current events. Usage numbers never change its exit. Exit 0 ok (or off) · 2 block · 3 needs the human · 64 usage Environment: ALADDIN_GA_ACCESS_TOKEN (an OAuth access token), ALADDIN_GA_TOKEN_COMMAND (default @@ -59,6 +64,9 @@ ACKNOWLEDGEMENT = ("I acknowledge that I have the necessary privacy disclosures and rights from my end users for the " "collection and processing of their data, including the association of such data with the " "visitation information Google Analytics collects from my site and/or app property.") +CREDENTIAL = ("a service account the human adds to their Analytics account as Editor; agents mint its tokens with " + "gcloud, carrying only the scopes below. Signing in also leaves the human's gcloud credentials " + "(cloud-platform) on this machine, as any gcloud use does") HOSTS = {"admin": "https://analyticsadmin.googleapis.com", "data": "https://analyticsdata.googleapis.com", "mp": "https://www.google-analytics.com"} LOOPBACK = ("127.0.0.1", "localhost", "::1") @@ -67,8 +75,8 @@ # the minimum-sample rule (§4.12 call): small numbers are noise, and GA hides them anyway MIN_USERS, WINDOW_DAYS, USAGE_SHARE, RESEARCH_FLOOR = 50, 28, 0.7, 0.3 SETTLED = {"startDate": "30daysAgo", "endDate": "3daysAgo"} # 28 days; the newest is ≥48h old, so settled -# the fixed report definitions (hashed into every report). Aggregate dimensions only: agents -# never see a user or client id +# the fixed report definitions (hashed into every report); request() scopes each to the stream. +# Aggregate dimensions only: agents never see a user or client id AGGREGATE_DIMENSIONS = {"eventName", "date", "newVsReturning", "deviceCategory"} REPORTS = { "users": {"api": "runReport", "body": {"dateRanges": [SETTLED], "metrics": [ @@ -80,37 +88,44 @@ } SETTLED_REPORTS = ("users", "journeys") -# GA4 event names: a letter, then letters, digits or underscores, at most 40; reserved names and prefixes +# GA4 event names, conservatively: an ASCII letter, then letters, digits or _, at most 40; the +# Measurement Protocol's reserved names (plus the app-only ones) and Firebase's reserved prefixes EVENT_NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,39}$") RESERVED = {"ad_activeview", "ad_click", "ad_exposure", "ad_impression", "ad_query", "ad_reward", "adunit_exposure", "app_clear_data", "app_exception", "app_install", "app_remove", "app_store_refund", "app_update", "app_upgrade", "dynamic_link_app_open", "dynamic_link_app_update", "dynamic_link_first_open", "error", - "first_open", "first_visit", "in_app_purchase", "notification_dismiss", "notification_foreground", - "notification_open", "notification_receive", "os_update", "session_start", "session_start_with_rollout", - "user_engagement"} -RESERVED_PREFIXES = ("_", "firebase_", "ga_", "google_", "gtag.") -MP_BATCH = 25 # events per Measurement Protocol request + "firebase_campaign", "firebase_in_app_message_action", "firebase_in_app_message_dismiss", + "firebase_in_app_message_impression", "first_open", "first_visit", "in_app_purchase", + "notification_dismiss", "notification_foreground", "notification_open", "notification_receive", + "notification_send", "os_update", "screen_view", "session_start", "user_engagement"} +RESERVED_PREFIXES = ("_", "firebase_", "ga_", "google_") +MP_BATCH = 25 # events per Measurement Protocol request (the validator does not enforce it) DEBUG_CLIENT_ID = "aloop.debug" # the debug hit's client id: no user's PROBE_EVENT = "aloop_probe" # sent when no journey has events yet, to prove the wiring SECRET_NAME = "aloop" -# the emit check reads code, not docs, data or tests +# the emit check reads this checkout's code, not docs, data or tests CODE_EXT = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".vue", ".svelte", ".astro", ".html", ".htm", ".py", - ".rb", ".erb", ".go", ".rs", ".java", ".kt", ".swift", ".dart", ".php", ".cs", ".ex", ".exs", ".m"} -SKIP_DIRS = {"docs", "test", "tests", "__tests__", "spec", "specs", "fixtures", "node_modules", "vendor", "dist", "build"} -TEST_FILE = re.compile(r"(^test_|_test\.[^.]+$|\.(test|spec)\.[^.]+$)") - -AUTH_STEPS = """One-time Google sign-in (the only human step; paste in a terminal on your Mac): - 1. A Google Analytics account (skip if you have one): https://analytics.google.com -> Start measuring. - 2. A Google Cloud project with the two Analytics APIs on: - gcloud projects create aloop-ga-$RANDOM --set-as-default # or: gcloud config set project YOUR_PROJECT - gcloud services enable analyticsadmin.googleapis.com analyticsdata.googleapis.com - 3. An OAuth client for that project: https://console.cloud.google.com/auth/clients -> Create client -> - Desktop app -> Download JSON, saved as ~/aloop-ga-client.json (the first time, the consent screen asks - for an app name and your email, and you add yourself as a test user). - 4. Sign in once: - gcloud auth application-default login --client-id-file="$HOME/aloop-ga-client.json" \\ - --scopes=https://www.googleapis.com/auth/analytics.edit,https://www.googleapis.com/auth/analytics.readonly,https://www.googleapis.com/auth/cloud-platform + ".rb", ".erb", ".go", ".rs", ".java", ".kt", ".swift", ".dart", ".php", ".cs", ".cshtml", ".razor", + ".ex", ".exs", ".heex", ".m", ".ejs", ".hbs", ".handlebars", ".njk", ".twig", ".liquid", ".pug", + ".haml", ".slim", ".jinja", ".j2", ".mustache"} +SKIP_DIRS = {"docs", "test", "tests", "__tests__", "__mocks__", "spec", "specs", "fixtures", "cypress", "e2e", + "playwright", "testdata", "node_modules", "vendor", "dist", "build"} +TEST_FILE = re.compile(r"(^test_|^tests\.py$|^conftest\.py$|_(test|spec)\.[^.]+$|\.(test|spec|cy|e2e)\.[^.]+$" + r"|(Test|Tests|Spec)\.[^.]+$)") +SCAN_LIMIT = 2 * 1024 * 1024 # bytes read per file + +AUTH_STEPS = """One-time Google sign-in (the only human step). Paste in a terminal with gcloud installed: + gcloud auth login + P=aloop-ga-$RANDOM; gcloud projects create $P && gcloud config set project $P + gcloud services enable analyticsadmin.googleapis.com analyticsdata.googleapis.com iam.googleapis.com iamcredentials.googleapis.com + gcloud iam service-accounts create aloop-ga --display-name="aloop GA" + SA=aloop-ga@$P.iam.gserviceaccount.com + gcloud iam service-accounts add-iam-policy-binding $SA --member="user:$(gcloud config get-value account)" --role=roles/iam.serviceAccountTokenCreator + gcloud auth application-default login --impersonate-service-account=$SA --scopes=https://www.googleapis.com/auth/analytics.edit,https://www.googleapis.com/auth/analytics.readonly + echo $SA +Then in a browser: https://analytics.google.com -> Admin -> Account access management -> + -> Add users -> +paste that service-account email, role Editor -> Add. (No Analytics account yet? Admin -> Create account first.) Then: aloop mission telemetry setup""" @@ -118,6 +133,10 @@ class Usage(Exception): pass +class Help(Exception): + pass + + class GAError(Exception): def __init__(self, code, status, message): super().__init__("%s %s: %s" % (code, status, message)) @@ -141,16 +160,35 @@ def current(cwd): def consent_state(cwd, path, doc): - """none | revoked | stale (the last consent covers other terms) | ok, from the events only.""" + """none | revoked | stale (the last consent covers other terms) | ok, from human events only: + an event any other role wrote is not a consent (approval provenance itself is M9.17).""" ref, last = M.rel(cwd, path), None for e in _events.read(cwd): - if e.get("event") in ("telemetry.consent", "telemetry.revoked") and (e.get("data") or {}).get("subject") == ref: + if e.get("event") in ("telemetry.consent", "telemetry.revoked") and e.get("role") == "human" \ + and (e.get("data") or {}).get("subject") == ref: last = e if last is None: return "none", None if last["event"] == "telemetry.revoked": return "revoked", last - return ("ok" if (last.get("data") or {}).get("sha256") == terms_sha(doc) else "stale"), last + sha = terms_sha(doc) + if (last.get("data") or {}).get("sha256") == sha and M.approved(cwd, "telemetry.consent", ref, sha): + return "ok", last + return "stale", last + + +def bound(doc): + """Set up for the current terms: a property and a stream measuring exactly the consented uri.""" + return bool(doc.get("property") and (doc.get("stream") or {}).get("default_uri") == doc["terms"]["uri"]) + + +def derive_status(cwd, path, doc): + state = consent_state(cwd, path, doc)[0] + if state == "revoked": + return "revoked" + if state != "ok": + return "off" + return "configured" if bound(doc) else "consented" OPT_IN = ("telemetry is opt-in and off: `aloop mission telemetry init --uri URL`, then a human runs " @@ -209,6 +247,20 @@ def write_secret(cwd, telemetry_id, data): return p +def local_time_zone(): + """The machine's IANA zone (the property's reporting zone), or Google's example zone.""" + tz = os.environ.get("TZ", "").lstrip(":") + if "/" not in tz: + try: + tz = os.path.realpath("/etc/localtime").split("zoneinfo/", 1)[1] + except (OSError, IndexError): + tz = "" + if tz.startswith(("posix/", "right/")): + tz = tz.split("/", 1)[1] + ok = re.match(r"^[A-Z][A-Za-z_]+/[A-Za-z0-9_+\-/]+$", tz) and not tz.startswith("Etc/") + return tz if ok else "America/Los_Angeles" + + # ── journeys (M9.3's files, read by contract fields only) and the emit check ───────── def journeys(cwd): """Active journeys as [(name, [step event, …])], oldest first.""" @@ -224,6 +276,11 @@ def journey_events(cwd): return sorted({e for _, evs in journeys(cwd) for e in evs}) +def key_events(cwd): + """The key event of each active journey: its last step's event.""" + return sorted({evs[-1] for _, evs in journeys(cwd) if evs and not name_problem(evs[-1])}) + + def events_sha(evs): return M.digest({"events": sorted(evs)}, drop=()) @@ -236,7 +293,21 @@ def name_problem(name): return None -def source_files(root): +def checkout(cwd): + """The source tree being checked: cwd's own checkout (a worker's linked worktree included), + while journeys and the record stay in the main worktree's docs/agents/mission/.""" + try: + r = subprocess.run(["git", "-C", cwd, "rev-parse", "--show-toplevel"], capture_output=True, text=True, + timeout=30) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip() + except (OSError, subprocess.TimeoutExpired): + pass + return _paths.project(cwd) + + +def source_files(root, code_only=True): + """Non-test, non-doc files of the checkout (tracked or new, never git-ignored).""" try: out = subprocess.run(["git", "-C", root, "ls-files", "-z", "--cached", "--others", "--exclude-standard"], capture_output=True, text=True, timeout=60) @@ -249,35 +320,47 @@ def source_files(root): parts = f.replace(os.sep, "/").split("/") if any(p.startswith(".") or p in SKIP_DIRS for p in parts[:-1]) or TEST_FILE.search(parts[-1]): continue - if os.path.splitext(f)[1].lower() in CODE_EXT: + if not code_only or os.path.splitext(f)[1].lower() in CODE_EXT: yield f -def emitted(cwd, names): - """{event: "file:line"} for each name found as a quoted literal ('x', "x" or `x`) in code.""" - root, found = _paths.project(cwd), {} - want = {n: re.compile(r"(['\"`])%s\1" % re.escape(n)) for n in names} - for f in source_files(root): - if len(found) == len(want): +def scan(cwd, patterns, code_only=True): + """{key: "file:line"} for each pattern's first hit in the checkout's files.""" + root, found = checkout(cwd), {} + for f in source_files(root, code_only): + if len(found) == len(patterns): break try: with open(os.path.join(root, f), errors="ignore") as fh: - for i, line in enumerate(fh, 1): - for n, rx in want.items(): - if n not in found and rx.search(line): - found[n] = "%s:%d" % (f, i) + text = fh.read(SCAN_LIMIT) except OSError: continue + for i, line in enumerate(text.splitlines(), 1): + for k, rx in patterns.items(): + if k not in found and rx.search(line): + found[k] = "%s:%d" % (f, i) return found +def emitted(cwd, names): + """{event: "file:line"} for each name found as a quoted literal ('x', "x" or `x`) in code.""" + return scan(cwd, {n: re.compile(r"(['\"`])%s\1" % re.escape(n)) for n in names}) + + +def tagged(cwd, measurement_id): + """Where the checkout carries the stream's measurement id (code, config or env file), or None.""" + hit = scan(cwd, {"mid": re.compile(r"(? Account access management), or pass --account / --property you can edit" % e.message) + say("ASK", "the signed-in Google identity lacks access (%s): add it to the Analytics account as Editor " + "(Admin -> Account access management), or pass --account / --property it can edit" % e.message) return 3 say("BLOCK", "Google Analytics: %s" % e) return 2 @@ -362,12 +452,11 @@ def need_token(): def cmd_init(cwd, o): uri = o.get("uri") or "" if not re.match(r"^https?://[^\s/]+", uri): - say("BLOCK", "--uri must be the app's http(s) URL (what is measured is part of the terms)") - return 64 + raise Usage("--uri must be the app's http(s) URL (what is measured is part of the terms)") path, doc = current(cwd) if doc is not None and not o.get("force"): say("BLOCK", "%s exists; `aloop mission telemetry init --uri URL --force` re-drafts its terms " - "(consent is asked again)" % M.rel(cwd, path)) + "(changed terms need a new consent)" % M.rel(cwd, path)) return 2 mission, _ = M.load(M.path_of(cwd, "mission")) mission = mission if isinstance(mission, dict) else {} @@ -376,11 +465,11 @@ def cmd_init(cwd, o): "purpose": "usage of %s informs research and goal-setting for the mission; it is never a gate" % uri, "collects": ["the app's Google Analytics 4 web measurement on this uri", "one event per user-journey step (its name and engagement time)"], - "agents_see": "aggregate counts only (event name, users, new users) from fixed report definitions; " - "never a user or client id", - "acknowledgement": ACKNOWLEDGEMENT, "scopes": list(SCOPES), + "agents_see": "aggregate counts of this uri's stream only (event name, users, new users) from fixed " + "report definitions; never a user or client id", + "acknowledgement": ACKNOWLEDGEMENT, "credential": CREDENTIAL, "scopes": list(SCOPES), "others": "once other people use the app, a consent banner (Consent Mode v2) is a journey requirement"} - settings = {"display_name": name, "time_zone": o.get("time_zone") or "Etc/GMT", "currency": "USD"} + settings = {"display_name": name, "time_zone": o.get("time_zone") or local_time_zone(), "currency": "USD"} if doc is None: tid = M.new_id("telemetry") path = M.path_of(cwd, "telemetry", tid) @@ -389,11 +478,15 @@ def cmd_init(cwd, o): "property": None, "acknowledged": False, "stream": None, "secret": None, "key_events": [], "debug_hit": None} else: - doc.update(terms=terms, settings=settings, status="off") + if (doc.get("terms") or {}).get("uri") != uri: # another uri: the old stream measures something else + doc.update(stream=None, secret=None, key_events=[], debug_hit=None) + doc.update(terms=terms, settings=settings) + doc["status"] = derive_status(cwd, path, doc) store(path, doc) print(path) - say("ASK", "terms drafted (nothing was sent): a human reviews `aloop mission telemetry show` and runs " - "`aloop mission telemetry consent --by NAME`") + if doc["status"] == "off": + say("ASK", "terms drafted (nothing was sent): a human reviews `aloop mission telemetry show` and runs " + "`aloop mission telemetry consent --by NAME`") return 0 @@ -419,7 +512,7 @@ def cmd_consent(cwd, o): return 2 eid = M.approve(cwd, "telemetry.consent", M.rel(cwd, path), terms_sha(doc), by, reason, telemetry_id=doc["telemetry_id"], uri=doc["terms"]["uri"]) - doc["status"] = "configured" if doc.get("stream") else "consented" + doc["status"] = derive_status(cwd, path, doc) doc["approval"] = {"event_id": eid, "by": by, "at": M.now()} store(path, doc) print("[telemetry] %s: %s consented to the terms (sha256 %s)" % (doc["telemetry_id"], by, terms_sha(doc)[:12]), @@ -436,31 +529,47 @@ def cmd_revoke(cwd, o): ref = M.rel(cwd, path) M.emit(cwd, "telemetry.revoked", role="human", refs={"subject": ref}, subject=ref, by=by, reason=reason, telemetry_id=doc.get("telemetry_id")) + try: + os.remove(secret_file(cwd, doc["telemetry_id"])) + except OSError: + pass doc["status"] = "revoked" M.save(path, doc) # status is outside the digest: the seal still holds - print("[telemetry] %s revoked by %s: nothing is sent until a new consent" % (doc.get("telemetry_id"), by), + print("[telemetry] %s revoked by %s: agents stop setup, debug hits and usage reports, usage no longer drives " + "goals, and the local Measurement Protocol secret is deleted. The app's own GA tag keeps sending until " + "it is removed (`aloop mission telemetry check` names where it is)." % (doc["telemetry_id"], by), file=sys.stderr) return 0 def pick_property(doc, o, tok): - """(account, property, rc): the property given, recorded, found by display name, or created in - the one reachable account. Which account is the human's call when there are none or several (rc 3).""" - prop = o.get("property") or doc.get("property") - if prop: - call("GET", "admin", "/v1beta/" + prop, tok=tok) - return doc.get("account"), prop, 0 + """(account, property, rc): --property, else the recorded one (unless --account asks to pick + again), else found by display name or created in the one reachable account. Which account + is the human's call when there are none or several, or when the recorded one is gone (rc 3).""" + if o.get("property"): + call("GET", "admin", "/v1beta/" + o["property"], tok=tok) + return doc.get("account") if o["property"] == doc.get("property") else None, o["property"], 0 + if doc.get("property") and not o.get("account"): + try: + call("GET", "admin", "/v1beta/" + doc["property"], tok=tok) + return doc.get("account"), doc["property"], 0 + except GAError as e: + if e.code not in (403, 404) or SIGN_IN.search(e.message) or API_OFF.search(e.message): + raise + say("ASK", "the recorded property %s is not reachable (%s): rerun with `--account accounts/N` to find or " + "create one there, or `--property properties/N`" % (doc["property"], e.message)) + return None, None, 3 summaries = listall("/v1beta/accountSummaries", "accountSummaries", tok) if o.get("account"): found = [a for a in summaries if a.get("account") == o["account"]] if not found: - say("BLOCK", "%s is not an account the signed-in Google user can reach (%s)" + say("BLOCK", "%s is not an account the signed-in Google identity can reach (%s)" % (o["account"], ", ".join(a.get("account", "") for a in summaries) or "none")) return None, None, 2 elif not summaries: - say("ASK", "the signed-in Google user has no Analytics account: create one at https://analytics.google.com " - "(Admin -> Create account; Google needs you to accept its terms there), then rerun " - "`aloop mission telemetry setup`") + say("ASK", "the signed-in Google identity has no Analytics account: create one at https://analytics.google.com " + "(Admin -> Create account; Google needs you to accept its terms there) and add the service account " + "to it, then rerun `aloop mission telemetry setup`") return None, None, 3 elif len(summaries) > 1: say("ASK", "several Analytics accounts: choose one, e.g. `aloop mission telemetry setup --account %s` (%s)" @@ -514,23 +623,23 @@ def cmd_setup(cwd, o): store(path, doc) local = read_secret(cwd, tid) secrets = listall("/v1beta/%s/measurementProtocolSecrets" % s["name"], "measurementProtocolSecrets", tok) - sec = next((x for x in secrets if local and x.get("name") == local.get("name")), None) or \ - next((x for x in secrets if x.get("displayName") == SECRET_NAME and x.get("secretValue")), None) + sec = next((x for x in secrets if local and x.get("name") == local.get("name") and x.get("secretValue")), None) \ + or next((x for x in secrets if x.get("displayName") == SECRET_NAME and x.get("secretValue")), None) if sec is None: sec = call("POST", "admin", "/v1beta/%s/measurementProtocolSecrets" % s["name"], {"displayName": SECRET_NAME}, tok) - value = sec.get("secretValue") or (local or {}).get("api_secret") - if not value: + if not sec.get("secretValue"): raise GAError(0, "NO_SECRET", "Google returned no Measurement Protocol secret value") - write_secret(cwd, tid, {"telemetry_id": tid, "stream": s["name"], "name": sec["name"], "api_secret": value}) + write_secret(cwd, tid, {"telemetry_id": tid, "stream": s["name"], "name": sec["name"], + "api_secret": sec["secretValue"]}) doc["secret"] = {"name": sec["name"], "display_name": sec.get("displayName") or SECRET_NAME} - want = [evs[-1] for _, evs in journeys(cwd) if evs and not name_problem(evs[-1])] + want = key_events(cwd) have = {k.get("eventName") for k in listall("/v1beta/%s/keyEvents" % prop, "keyEvents", tok)} for ev in sorted(set(want) - have): call("POST", "admin", "/v1beta/%s/keyEvents" % prop, {"eventName": ev, "countingMethod": "ONCE_PER_EVENT"}, tok) - doc["key_events"] = sorted(set(want)) - doc["status"] = "configured" + doc["key_events"] = want + doc["status"] = derive_status(cwd, path, doc) store(path, doc) except GAError as e: return ga_failed(e) @@ -547,8 +656,8 @@ def cmd_debug_hit(cwd, o): return rc endpoint("mp") sec = read_secret(cwd, doc["telemetry_id"]) - if not doc.get("stream") or not sec: - say("BLOCK", "not set up on this machine yet: `aloop mission telemetry setup` first") + if not bound(doc) or not sec: + say("BLOCK", "not set up for %s on this machine: `aloop mission telemetry setup` first" % doc["terms"]["uri"]) return 2 evs = journey_events(cwd) bad = ["event %r %s" % (e, name_problem(e)) for e in evs if name_problem(e)] @@ -556,14 +665,24 @@ def cmd_debug_hit(cwd, o): say("BLOCK", b) if bad: return 2 - mid, msgs = doc["stream"]["measurement_id"], [] - send = evs or [PROBE_EVENT] + tok = need_token() + if not tok: + return 3 + mid, msgs, send = doc["stream"]["measurement_id"], [], evs or [PROBE_EVENT] try: - for i in range(0, len(send), MP_BATCH): - body = {"client_id": DEBUG_CLIENT_ID, - "events": [{"name": e, "params": {"engagement_time_msec": 1}} for e in send[i:i + MP_BATCH]]} - r = call("POST", "mp", "/debug/mp/collect", body, query={"measurement_id": mid, "api_secret": sec["api_secret"]}) - msgs += r.get("validationMessages") or [] + # Google's validation server checks the payload, never the secret: prove the secret on the Admin API + live = listall("/v1beta/%s/measurementProtocolSecrets" % doc["stream"]["name"], "measurementProtocolSecrets", tok) + if not any(x.get("name") == sec.get("name") and x.get("secretValue") == sec["api_secret"] for x in live): + msgs.append({"fieldPath": "api_secret", "validationCode": "SECRET_NOT_LIVE", + "description": "the local Measurement Protocol secret is not live on %s (deleted or for " + "another stream): rerun `aloop mission telemetry setup`" % doc["stream"]["name"]}) + else: + for i in range(0, len(send), MP_BATCH): + body = {"client_id": DEBUG_CLIENT_ID, "validation_behavior": "ENFORCE_RECOMMENDATIONS", + "events": [{"name": e, "params": {"engagement_time_msec": 1}} for e in send[i:i + MP_BATCH]]} + r = call("POST", "mp", "/debug/mp/collect", body, + query={"measurement_id": mid, "api_secret": sec["api_secret"]}) + msgs += r.get("validationMessages") or [] except GAError as e: return ga_failed(e) doc["debug_hit"] = {"at": M.now(), "ok": not msgs, "measurement_id": mid, "events_sha256": events_sha(evs), @@ -601,11 +720,18 @@ def simplify(r): "thresholded": bool(meta.get("subjectToThresholding")), "data_loss": bool(meta.get("dataLossFromOtherRow"))} -def request(name, events): - """The exact body sent for a fixed report: its definition, filtered to the journey events.""" +def stream_id(doc): + return doc["stream"]["name"].rsplit("/", 1)[-1] + + +def request(name, events, stream): + """The exact body sent for a fixed report: its definition, scoped to the consented stream and, + for event reports, to the journey events.""" body = json.loads(json.dumps(REPORTS[name]["body"])) + exprs = [{"filter": {"fieldName": "streamId", "stringFilter": {"matchType": "EXACT", "value": stream}}}] if REPORTS[name].get("events"): - body["dimensionFilter"] = {"filter": {"fieldName": "eventName", "inListFilter": {"values": sorted(events)}}} + exprs.append({"filter": {"fieldName": "eventName", "inListFilter": {"values": sorted(events)}}}) + body["dimensionFilter"] = exprs[0] if len(exprs) == 1 else {"andGroup": {"expressions": exprs}} return body @@ -627,8 +753,8 @@ def cmd_report(cwd, o): if rc: return rc endpoint("data") - if not doc.get("property"): - say("BLOCK", "no property yet: `aloop mission telemetry setup` first") + if not bound(doc): + say("BLOCK", "no stream for %s yet: `aloop mission telemetry setup` first" % doc["terms"]["uri"]) return 2 tok = need_token() if not tok: @@ -638,7 +764,8 @@ def cmd_report(cwd, o): for n in (("realtime",) if realtime else SETTLED_REPORTS): if REPORTS[n].get("events") and not evs: continue # only journey events are read; none yet, nothing to read - r = call("POST", "data", "/v1beta/%s:%s" % (doc["property"], REPORTS[n]["api"]), request(n, evs), tok) + r = call("POST", "data", "/v1beta/%s:%s" % (doc["property"], REPORTS[n]["api"]), + request(n, evs, stream_id(doc)), tok) results[n] = simplify(r) except GAError as e: return ga_failed(e) @@ -652,9 +779,9 @@ def cmd_report(cwd, o): sh = share(users, thr, True) rid = M.new_id("telemetry") rep = {"report_id": rid, "telemetry_id": doc["telemetry_id"], "kind": "realtime" if realtime else "settled", - "fetched_at": M.now(), "property": doc["property"], "definitions_sha256": M.digest(REPORTS, drop=()), - "date_range": "last 30 minutes" if realtime else SETTLED, "events": evs, "results": results, - "sample": sample, "share": sh} + "fetched_at": M.now(), "property": doc["property"], "stream": doc["stream"]["name"], + "definitions_sha256": M.digest(REPORTS, drop=()), "date_range": "last 30 minutes" if realtime else SETTLED, + "events": evs, "results": results, "sample": sample, "share": sh} M.seal(rep) p = M.save(M.mdir(cwd, "telemetry", "reports", rid + ".json"), rep) M.emit(cwd, "telemetry.report", role="mission", refs={"report": M.rel(cwd, p)}, report_id=rid, kind=rep["kind"], @@ -671,19 +798,21 @@ def cmd_report(cwd, o): def latest_share(cwd): """The usage/research split for the next goal (M9.11 reads this): from the newest settled report - whose seal holds, and only while the consent covers the terms.""" + of the current record and stream whose seal holds, and only while the consent covers the terms.""" path, doc = current(cwd) none = {"usage": 0, "research": 1, "report": None} if doc is None or consent_state(cwd, path, doc)[0] != "ok": return dict(none, reason="telemetry is off, revoked or not consented: research only") + stream = (doc.get("stream") or {}).get("name") if bound(doc) else None for p in sorted(glob.glob(M.mdir(cwd, "telemetry", "reports", "*.json")), reverse=True): rep, _ = M.load(p) - if not isinstance(rep, dict) or rep.get("kind") != "settled" or rep.get("sha256") != M.digest(rep): + if not isinstance(rep, dict) or rep.get("kind") != "settled" or rep.get("sha256") != M.digest(rep) \ + or rep.get("telemetry_id") != doc["telemetry_id"] or not stream or rep.get("stream") != stream: continue s = rep.get("sample") or {} return dict(share(int(s.get("users") or 0), bool(s.get("thresholded")), True), report=rep.get("report_id"), users=s.get("users")) - return dict(none, reason="no settled usage report yet: research only") + return dict(none, reason="no settled usage report for this stream yet: research only") def cmd_share(cwd, o): @@ -709,7 +838,11 @@ def evaluate(cwd): if state == "none": return 0, [], dict(facts, note="telemetry off (opt-in): drafted, not consented") if state == "revoked": - return 0, [], dict(facts, note="telemetry revoked (opt-out) by %s" % (ev.get("data") or {}).get("by")) + mid = (doc.get("stream") or {}).get("measurement_id") + at = tagged(cwd, mid) if mid else None + return 0, [], dict(facts, still_tagged=at, note="telemetry revoked (opt-out) by %s%s" % ( + (ev.get("data") or {}).get("by"), "; the app still carries %s at %s (remove it to stop collection)" + % (mid, at) if at else "")) if state == "stale": return 3, ["the consent does not cover the current terms: a human runs " "`aloop mission telemetry consent --by NAME`"], facts @@ -724,13 +857,24 @@ def evaluate(cwd): problems.append("journey %r step event %r is not emitted by any non-test source file (instrumentation is " "part of done)" % (m["journey"], m["event"])) facts.update(events=evs, emitted=found, missing=missing) - if doc.get("stream"): + if doc.get("stream") and not bound(doc): + problems.append("the stream measures %s, not the consented %s: `aloop mission telemetry setup`" + % (doc["stream"]["default_uri"], doc["terms"]["uri"])) + elif bound(doc): + mid = doc["stream"]["measurement_id"] + facts["tagged"] = tagged(cwd, mid) + if not facts["tagged"]: + problems.append("the app does not carry the stream's measurement id %s (e.g. gtag('config', '%s')): " + "its events would reach no stream" % (mid, mid)) + if doc.get("key_events") != key_events(cwd): + problems.append("the key events %s do not match the active journeys' last steps %s: " + "`aloop mission telemetry setup`" % (doc.get("key_events"), key_events(cwd))) dh = doc.get("debug_hit") if not dh: problems.append("no debug hit yet: `aloop mission telemetry debug-hit`") elif not dh.get("ok"): - problems.append("the last debug hit failed GA validation: fix it and rerun `aloop mission telemetry debug-hit`") - elif dh.get("measurement_id") != doc["stream"]["measurement_id"] or dh.get("events_sha256") != events_sha(evs): + problems.append("the last debug hit failed: fix it and rerun `aloop mission telemetry debug-hit`") + elif dh.get("measurement_id") != mid or dh.get("events_sha256") != events_sha(evs): problems.append("the debug hit is stale (the journey events or stream changed): " "`aloop mission telemetry debug-hit`") return (2 if problems else 0), problems, facts @@ -744,7 +888,7 @@ def cmd_check(cwd, o): say("ASK" if rc == 3 else "BLOCK", b) if rc == 0: print("[telemetry] %s" % (facts.get("note") or "ok: consented, instrumented%s" % ( - ", debug hit fresh" if facts.get("status") == "configured" else "")), file=sys.stderr) + ", tagged, debug hit fresh" if facts.get("tagged") else "")), file=sys.stderr) return rc @@ -760,9 +904,9 @@ def cmd_show(cwd, o): return 0 t = doc["terms"] print("Telemetry %s (%s, consent: %s)\nMeasures: %s\nPurpose: %s\nCollects: %s\nAgents see: %s\nOthers: %s\n" - "Google's acknowledgement: %s\nScopes: %s" % (doc["telemetry_id"], doc.get("status"), state, t["uri"], - t["purpose"], "; ".join(t["collects"]), t["agents_see"], - t["others"], t["acknowledgement"], " ".join(t["scopes"]))) + "Google's acknowledgement: %s\nCredential: %s\nScopes: %s" % ( + doc["telemetry_id"], doc.get("status"), state, t["uri"], t["purpose"], "; ".join(t["collects"]), + t["agents_see"], t["others"], t["acknowledgement"], t["credential"], " ".join(t["scopes"]))) if doc.get("property"): print("Property: %s · stream %s · key events: %s" % (doc["property"], (doc.get("stream") or {}).get( "measurement_id"), ", ".join(doc.get("key_events") or []) or "none")) @@ -774,35 +918,44 @@ def cmd_auth(cwd, o): return 0 -CMDS = {"init": cmd_init, "consent": cmd_consent, "revoke": cmd_revoke, "auth": cmd_auth, "setup": cmd_setup, - "debug-hit": cmd_debug_hit, "report": cmd_report, "share": cmd_share, "show": cmd_show, "check": cmd_check} -FLAGS = ("json", "force", "realtime") +CMDS = {"init": (cmd_init, {"uri", "name", "time_zone", "force"}), "consent": (cmd_consent, {"by", "reason"}), + "revoke": (cmd_revoke, {"by", "reason"}), "auth": (cmd_auth, set()), + "setup": (cmd_setup, {"account", "property"}), "debug-hit": (cmd_debug_hit, set()), + "report": (cmd_report, {"realtime", "json"}), "share": (cmd_share, {"json"}), "show": (cmd_show, {"json"}), + "check": (cmd_check, {"json"})} +FLAGS = {"json", "force", "realtime"} -def parse(argv): - opts, pos, k = {}, [], None +def parse(argv, allowed): + """Options of one command: only its own, so a typo or a --help never runs a network command.""" + opts, k = {}, None for a in argv: if k: opts[k], k = a, None - elif a.startswith("--"): - k = a[2:].replace("-", "_") - if k in FLAGS: - opts[k], k = True, None + elif a in ("-h", "--help"): + raise Help() + elif not a.startswith("--") or a[2:].replace("-", "_") not in allowed: + raise Usage("unexpected %r (allowed: %s)" % (a, " ".join(sorted("--" + x.replace("_", "-") + for x in allowed)) or "no options")) + elif a[2:].replace("-", "_") in FLAGS: + opts[a[2:].replace("-", "_")] = True else: - pos.append(a) + k = a[2:].replace("-", "_") if k: - opts[k] = "" - return opts, pos + raise Usage("--%s needs a value" % k.replace("_", "-")) + return opts def main(argv, cwd): - fn = CMDS.get(argv[0]) - if fn is None: + if argv[0] not in CMDS: say("BLOCK", "unknown command %r (mission telemetry --help)" % argv[0]) return 64 - o, _ = parse(argv[1:]) + fn, allowed = CMDS[argv[0]] try: - return fn(cwd, o) + return fn(cwd, parse(argv[1:], allowed)) + except Help: + print(__doc__.strip()) + return 0 except Usage as e: say("BLOCK", str(e)) return 64 diff --git a/scripts/test/fixtures/fake-ga.py b/scripts/test/fixtures/fake-ga.py index 8dafa3b6..c2b4d5f3 100644 --- a/scripts/test/fixtures/fake-ga.py +++ b/scripts/test/fixtures/fake-ga.py @@ -5,13 +5,16 @@ /admin/v1beta/… Admin API: accountSummaries, properties, :acknowledgeUserDataCollection, dataStreams, measurementProtocolSecrets, keyEvents /data/v1beta/… Data API: :runReport, :runRealtimeReport - /mp/debug/mp/collect the Measurement Protocol validation server + /mp/debug/mp/collect the Measurement Protocol validation server (like Google's, it checks the + payload and never the measurement_id or api_secret) Point the client at it with ALADDIN_GA_BASE=http://127.0.0.1:. The port is the first line on stdout. Admin and Data calls need `Authorization: Bearer $FAKE_GA_TOKEN` (default `fake-token`); the Measurement Protocol takes none. Test controls: POST /_control {json} merge into the scenario: accounts (0, 1, 2…), users, new_users, thresholded, events {name: [count, users]}, realtime {name: [count, users]} + (all on the property's first stream), mp_messages [extra validation + messages], redirect (a path every admin/data call is 302-redirected to) GET /_log every request so far: [{method, path, query, auth, body}] """ import json @@ -20,7 +23,7 @@ import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from urllib.parse import parse_qs, urlsplit +from urllib.parse import urlsplit TOKEN = os.environ.get("FAKE_GA_TOKEN", "fake-token") ACK = ("I acknowledge that I have the necessary privacy disclosures and rights from my end users for the " @@ -35,7 +38,8 @@ "user_engagement"} LOCK = threading.Lock() -SCENARIO = {"accounts": 1, "users": 0, "new_users": 0, "thresholded": False, "events": {}, "realtime": {}} +SCENARIO = {"accounts": 1, "users": 0, "new_users": 0, "thresholded": False, "events": {}, "realtime": {}, + "mp_messages": [], "redirect": ""} DB = {"n": 1000, "properties": {}, "acked": set(), "streams": {}, "secrets": {}, "key_events": {}} LOG = [] @@ -121,17 +125,29 @@ def admin(method, parts, body): return err(404, "NOT_FOUND", "unknown admin path") -def wanted(body): - f = ((body.get("dimensionFilter") or {}).get("filter") or {}) - return set((f.get("inListFilter") or {}).get("values") or []) if f.get("fieldName") == "eventName" else None +def filters(expr): + """{fieldName: filter} of a dimensionFilter: one filter, or an andGroup of them.""" + if not expr: + return {} + if "filter" in expr: + return {expr["filter"]["fieldName"]: expr["filter"]} + out = {} + for e in (expr.get("andGroup") or {}).get("expressions") or []: + out.update(filters(e)) + return out -def report(body, source): +def report(prop, body, source): dims = [d["name"] for d in body.get("dimensions") or []] mets = [m["name"] for m in body.get("metrics") or []] + f = filters(body.get("dimensionFilter")) rows = [] - if dims == ["eventName"]: - keep = wanted(body) + first = sorted((s for s in DB["streams"].values() if s["name"].startswith(prop + "/")), key=lambda s: s["name"]) + sid = (f.get("streamId") or {}).get("stringFilter", {}).get("value") + if sid is not None and (not first or first[0]["name"].rsplit("/", 1)[-1] != sid): + pass # another stream: none of the scenario's traffic + elif dims == ["eventName"]: + keep = set((f.get("eventName") or {}).get("inListFilter", {}).get("values") or []) if "eventName" in f else None for name, (count, users) in sorted(SCENARIO[source].items()): if keep is None or name in keep: vals = {"eventCount": count, "totalUsers": users, "activeUsers": users} @@ -161,22 +177,14 @@ def data(method, parts, body): if verb == "runReport": if not body.get("dateRanges"): return err(400, "INVALID_ARGUMENT", "dateRanges is required") - return report(body, "events") + return report("properties/" + pid, body, "events") if verb == "runRealtimeReport": - return report(body, "realtime") + return report("properties/" + pid, body, "realtime") return err(404, "NOT_FOUND", "unknown data method") -def mp_debug(query, body): - msgs = [] - mid, secret = (query.get("measurement_id") or [""])[0], (query.get("api_secret") or [""])[0] - streams = [s for s in DB["streams"].values() if s["webStreamData"]["measurementId"] == mid] - if not streams: - msgs.append({"fieldPath": "measurement_id", "description": "Unknown measurement_id %s" % mid, - "validationCode": "VALUE_INVALID"}) - elif not any(s["secretValue"] == secret and s["name"].startswith(streams[0]["name"] + "/") for s in DB["secrets"].values()): - msgs.append({"fieldPath": "api_secret", "description": "api_secret is not valid for this stream", - "validationCode": "VALUE_INVALID"}) +def mp_debug(body): + msgs = list(SCENARIO["mp_messages"]) if not body.get("client_id"): msgs.append({"fieldPath": "client_id", "description": "client_id is required", "validationCode": "VALUE_REQUIRED"}) events = body.get("events") or [] @@ -223,7 +231,12 @@ def handle_any(self, method): LOG.append({"method": method, "path": u.path, "query": u.query, "auth": auth, "body": body}) parts = [p for p in u.path.split("/") if p] if parts[:2] == ["mp", "debug"] and parts[2:] == ["mp", "collect"] and method == "POST": - return self.reply(*mp_debug(parse_qs(u.query), body)) + return self.reply(*mp_debug(body)) + if parts[:1] in (["admin"], ["data"]) and SCENARIO["redirect"]: + self.send_response(302) + self.send_header("Location", SCENARIO["redirect"]) + self.send_header("Content-Length", "0") + return self.end_headers() if parts[:1] in (["admin"], ["data"]): if auth != "Bearer " + TOKEN: return self.reply(*err(401, "UNAUTHENTICATED", "Request had invalid authentication credentials.")) diff --git a/scripts/test/mission-telemetry.test.py b/scripts/test/mission-telemetry.test.py index f1fa4035..edacfba4 100644 --- a/scripts/test/mission-telemetry.test.py +++ b/scripts/test/mission-telemetry.test.py @@ -1,16 +1,18 @@ #!/usr/bin/env python3 """ROADMAP §4.12 (M9.6): usage telemetry through Google Analytics 4. It is opt-in: nothing -reaches the network until a human consents to exact terms, and a revoke stops it. Setup goes -through the Admin API (property, web stream, user-data acknowledgement, Measurement Protocol -secret, a key event per journey), a debug hit proves the wiring, fixed Data API reports read -settled data, and the minimum-sample rule decides how much of a goal usage may drive. Usage -informs goals and is never a gate; the check is structural (consent, every journey step's -event is emitted, a fresh debug hit). All network is a local fake (fixtures/fake-ga.py). +reaches the network until a human consents to exact terms, and a revoke stops the agents' +use of it. Setup goes through the Admin API (property, web stream, user-data acknowledgement, +Measurement Protocol secret, a key event per journey), a debug hit proves the wiring, fixed +Data API reports read settled data of the consented stream only, and the minimum-sample rule +decides how much of a goal usage may drive. Usage informs goals and is never a gate; the check +is structural (consent, every journey step's event is emitted, the app carries the stream, a +fresh debug hit). All network is a local fake (fixtures/fake-ga.py). python3 scripts/test/mission-telemetry.test.py """ import importlib.util import json import os +import shutil import stat import subprocess import sys @@ -21,6 +23,8 @@ BIN = os.path.join(ROOT, "bin") MI = os.path.join(BIN, "mission") T = tempfile.mkdtemp(prefix="aloop-telemetry-") +WT = T + "-wt" +T2 = tempfile.mkdtemp(prefix="aloop-telemetry-probe-") PASS = [0] sys.path.insert(0, BIN) import _mission as M # noqa: E402 @@ -32,9 +36,11 @@ FAKE = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), "fixtures", "fake-ga.py")], stdout=subprocess.PIPE, text=True) BASE = "http://127.0.0.1:%s" % FAKE.stdout.readline().strip() -ENV = dict(os.environ, HOME=os.path.join(T, "home"), ALADDIN_GA_BASE=BASE, ALADDIN_GA_ACCESS_TOKEN="fake-token") +ENV = dict(os.environ, HOME=os.path.join(T, "home"), ALADDIN_GA_BASE=BASE, ALADDIN_GA_ACCESS_TOKEN="fake-token", + TZ="Europe/Stockholm") for k in ("AGENTIC_HOST", "AGENTIC_GA_ACCESS_TOKEN", "AGENTIC_GA_TOKEN_COMMAND", "AGENTIC_GA_BASE"): ENV.pop(k, None) +GIT = ["git", "-c", "user.name=t", "-c", "user.email=t@example.com"] def ok(name): @@ -42,8 +48,8 @@ def ok(name): print("ok -", name) -def run(*args, env=None): - return subprocess.run([MI, "telemetry"] + list(args), cwd=T, env=env or ENV, capture_output=True, text=True) +def run(*args, env=None, cwd=T): + return subprocess.run([MI, "telemetry"] + list(args), cwd=cwd, env=env or ENV, capture_output=True, text=True) def fake(path, body=None): @@ -57,14 +63,18 @@ def calls(): return fake("/_log") -def events(kind=None): - p = os.path.join(T, ".aladdin", "logs", "events.jsonl") +def mp_calls(log): + return [c for c in log if c["path"] == "/mp/debug/mp/collect"] + + +def events(kind=None, root=T): + p = os.path.join(root, ".aladdin", "logs", "events.jsonl") evs = [json.loads(l) for l in open(p)] if os.path.exists(p) else [] return [e for e in evs if kind is None or e["event"] == kind] -def record(): - recs = M.artifacts(T, "telemetry") +def record(root=T): + recs = M.artifacts(root, "telemetry") assert len(recs) == 1, recs return recs[0] @@ -76,11 +86,15 @@ def journey(name, steps, status="active"): return jid -def write(rel, text): - p = os.path.join(T, rel) +def write(rel, text, root=T): + p = os.path.join(root, rel) os.makedirs(os.path.dirname(p), exist_ok=True) open(p, "w").write(text) - subprocess.run(["git", "add", rel], cwd=T, check=True) + subprocess.run(["git", "add", rel], cwd=root, check=True) + + +def unwrite(rel, root=T): + subprocess.run(["git", "rm", "-q", "-f", rel], cwd=root, check=True) try: @@ -92,33 +106,44 @@ def write(rel, text): for cmd in (["setup"], ["debug-hit"], ["report"], ["consent", "--by", "Adeel"]): r = run(*cmd) assert r.returncode == 3 and "ASK[telemetry]" in r.stderr and "telemetry init" in r.stderr, (cmd, r.stderr) + r = run("setup", "--account", "accounts/100", "--help") + assert r.returncode == 0 and "never a gate" in r.stdout, r.stderr # --help anywhere is help + assert run("setup", "--propery", "properties/1").returncode == 64 # a typo is not ignored + assert run("report", "extra").returncode == 64 and run("consent", "--by").returncode == 64 assert calls() == [], calls() - ok("telemetry is off until a human opts in: check passes, every network command asks, zero requests") + ok("telemetry is off until a human opts in: check passes, network commands ask, --help/typos run nothing") - # 2) init drafts exact terms (sealed, harness id, GA's verbatim acknowledgement); consent covers them + # 2) init drafts exact terms; only a human's consent counts, and it covers exactly those terms assert run("init").returncode == 64 # what is measured is part of the terms assert run("init", "--uri", "ftp://todo").returncode == 64 - r = run("init", "--uri", "https://todo.example", "--time-zone", "Europe/Stockholm") + r = run("init", "--uri", "https://todo.example") assert r.returncode == 0, r.stderr path, doc = record() assert M.problems(doc, "telemetry", path) == [], M.problems(doc, "telemetry", path) assert doc["status"] == "off" and doc["telemetry_id"].startswith("tlm_") and doc["sha256"] == M.digest(doc) assert doc["terms"]["acknowledgement"] == TM.ACKNOWLEDGEMENT and "never a gate" in doc["terms"]["purpose"] assert doc["terms"]["uri"] == "https://todo.example" and set(TM.SCOPES) == set(doc["terms"]["scopes"]) - assert run("init", "--uri", "https://other.example").returncode == 2 # one record; terms change by re-init --force + assert "service account" in doc["terms"]["credential"] and "cloud-platform" in doc["terms"]["credential"] + assert doc["settings"]["time_zone"] == "Europe/Stockholm", doc["settings"] # the machine's zone + assert run("init", "--uri", "https://other.example").returncode == 2 # one record; --force re-drafts r = run("setup") assert r.returncode == 3 and "consent" in r.stderr, r.stderr + forged = subprocess.run([os.path.join(BIN, "events"), "emit", "telemetry.consent", "--role", "supervisor", + "--data", "subject=" + M.rel(T, path), "--data", "sha256=" + TM.terms_sha(doc)], + cwd=T, env=ENV, capture_output=True, text=True) + assert forged.returncode == 0, forged.stderr + assert run("setup").returncode == 3 and run("check").returncode == 0 and calls() == [] # not a consent assert run("consent").returncode == 64 # who consents is required r = run("consent", "--by", "Adeel", "--reason", "my own app, day one") assert r.returncode == 0, r.stderr path, doc = record() ev = events("telemetry.consent") - assert len(ev) == 1 and ev[0]["role"] == "human" and ev[0]["data"]["by"] == "Adeel", ev - assert ev[0]["data"]["sha256"] == TM.terms_sha(doc) and ev[0]["data"]["subject"] == M.rel(T, path) + assert [e["role"] for e in ev] == ["supervisor", "human"] and ev[1]["data"]["by"] == "Adeel", ev + assert ev[1]["data"]["sha256"] == TM.terms_sha(doc) and ev[1]["data"]["subject"] == M.rel(T, path) assert doc["status"] == "consented" and run("check").returncode == 0 and calls() == [] - ok("init seals the terms (uri, scopes, GA's acknowledgement); consent is a human event on exactly those terms") + ok("init seals the terms (uri, credential, GA's acknowledgement); a forged non-human consent is ignored") - # 3) consent covers exact terms: a changed term asks again and setup refuses; reverting restores it + # 3) consent covers exact terms; a hand-edited record blocks every command, consent cannot launder it doc["terms"]["uri"] = "https://elsewhere.example" M.seal(doc) M.save(path, doc) @@ -132,78 +157,99 @@ def write(rel, text): raw = json.load(open(path)) raw["property"] = "properties/999" # a hand edit after sealing json.dump(raw, open(path, "w")) - r = run("check") - assert r.returncode == 2 and "sha256 does not match" in r.stderr, r.stderr + for cmd in (["check"], ["setup"], ["report"], ["debug-hit"], ["consent", "--by", "Adeel"]): + r = run(*cmd) + assert r.returncode == 2 and "sha256 does not match" in r.stderr, (cmd, r.stderr) + assert calls() == [] and json.load(open(path)) == raw # not re-sealed by consent M.seal(doc) M.save(path, doc) - ok("consent covers exact terms (changed terms ask again, hand edits block); no request was made") + ok("consent covers exact terms; a hand-edited record blocks check, setup, report, debug-hit and consent") - # 4) no Google credentials: setup asks the human once, with the pastable sign-in, and sends nothing + # 4) credentials: none asks once with the pastable sign-in; a token command works; endpoints are loopback only env = dict(ENV, ALADDIN_GA_TOKEN_COMMAND="false") env.pop("ALADDIN_GA_ACCESS_TOKEN") r = run("setup", env=env) assert r.returncode == 3 and "gcloud auth application-default login" in r.stderr, r.stderr - assert "analytics.edit" in r.stderr and calls() == [] + assert "analytics.edit" in r.stderr and "--impersonate-service-account" in r.stderr and calls() == [] r = run("auth") - assert r.returncode == 0 and "--client-id-file" in r.stdout and "analyticsadmin.googleapis.com" in r.stdout - r = run("setup", env=dict(ENV, ALADDIN_GA_BASE="https://evil.example")) - assert r.returncode == 64 and "loopback" in r.stderr and calls() == [], r.stderr - ok("no token: setup asks with the one-time sign-in; a non-loopback endpoint override is refused") - - # 5) setup through the Admin API: account choice is the human's when it is ambiguous or missing + assert r.returncode == 0 and "iamcredentials.googleapis.com" in r.stdout and "Account access management" in r.stdout + for bad in ("https://evil.example", "https://127.0.0.1.evil.example", "https://evil.example/localhost", + "http://localhost.evil.example", "http://user@127.0.0.1:9", "file:///127.0.0.1"): + r = run("setup", env=dict(ENV, ALADDIN_GA_BASE=bad)) + assert r.returncode == 64 and "loopback" in r.stderr, (bad, r.stderr) + assert calls() == [] + ok("no token: setup asks with the one-time sign-in; every non-loopback endpoint override is refused") + + # 5) setup through the Admin API: the account is the human's call when it is missing or ambiguous fake("/_control", {"accounts": 0}) r = run("setup") assert r.returncode == 3 and "analytics.google.com" in r.stderr, r.stderr fake("/_control", {"accounts": 2}) r = run("setup") assert r.returncode == 3 and "--account accounts/100" in r.stderr and "accounts/101" in r.stderr, r.stderr + assert run("setup", "--account", "accounts/7").returncode == 2 assert all(c["method"] == "GET" for c in calls()) # asking never creates anything - fake("/_control", {"accounts": 1}) - signup = journey("sign up", ["signup_started", "signup_done"]) + journey("sign up", ["signup_started", "signup_done"]) journey("add a todo", ["todo_add_opened", "todo_added"]) journey("old idea", ["old_step"], status="retired") - r = run("setup") + n = len(calls()) + r = run("setup", "--account", "accounts/101") assert r.returncode == 0, r.stderr path, doc = record() assert doc["status"] == "configured" and doc["property"].startswith("properties/") and M.problems(doc, "telemetry", path) == [] - assert doc["stream"]["measurement_id"].startswith("G-") and doc["stream"]["default_uri"] == "https://todo.example" - assert sorted(doc["key_events"]) == ["signup_done", "todo_added"], doc["key_events"] - log = calls() + assert doc["account"] == "accounts/101" and doc["stream"]["default_uri"] == "https://todo.example" + assert doc["stream"]["measurement_id"].startswith("G-") and doc["key_events"] == ["signup_done", "todo_added"] + log = calls()[n:] posts = [c["path"].split("/v1beta/")[1] for c in log if c["method"] == "POST"] assert posts[0] == "properties" and posts[1].endswith(":acknowledgeUserDataCollection"), posts assert [p.rsplit("/", 1)[-1] for p in posts[2:]] == ["dataStreams", "measurementProtocolSecrets", "keyEvents", "keyEvents"], posts prop_body = [c["body"] for c in log if c["path"].endswith("/v1beta/properties") and c["method"] == "POST"][0] - assert prop_body["parent"] == "accounts/100" and prop_body["timeZone"] == "Europe/Stockholm", prop_body - ack = [c["body"] for c in log if c["path"].endswith(":acknowledgeUserDataCollection")][0] - assert ack == {"acknowledgement": TM.ACKNOWLEDGEMENT} + assert prop_body["parent"] == "accounts/101" and prop_body["timeZone"] == "Europe/Stockholm", prop_body + assert [c["body"] for c in log if c["path"].endswith(":acknowledgeUserDataCollection")] == [{"acknowledgement": TM.ACKNOWLEDGEMENT}] assert all(c["auth"] == "Bearer fake-token" for c in log) - secret = [c for c in log if c["path"].endswith("measurementProtocolSecrets") and c["method"] == "POST"] - assert len(secret) == 1 sfile = TM.secret_file(T, doc["telemetry_id"]) assert os.path.isfile(sfile) and stat.S_IMODE(os.stat(sfile).st_mode) == 0o600, sfile assert M.rel(T, sfile).startswith(".aladdin" + os.sep), sfile - value = json.load(open(sfile))["api_secret"] + secret = json.load(open(sfile)) for d, _, fs in os.walk(os.path.join(T, "docs")): for f in fs: - assert value not in open(os.path.join(d, f)).read(), "the MP secret leaked into %s" % f + assert secret["api_secret"] not in open(os.path.join(d, f)).read(), "the MP secret leaked into %s" % f assert [e["data"].get("action") for e in events("telemetry.setup")] == ["setup"] - ok("setup: property, acknowledgement, web stream, MP secret, a key event per active journey; secret 0600 outside docs/") + ok("setup: --account picks; property, acknowledgement, web stream, MP secret (0600, outside docs/), key events") - # 6) setup is idempotent: a rerun finds everything and creates nothing + # 6) setup is idempotent and recovers: reruns create nothing; expired tokens, redirects and lost properties n = len(calls()) r = run("setup") assert r.returncode == 0, r.stderr - again = calls()[n:] - assert again and not [c for c in again if c["method"] == "POST" and not c["path"].endswith(":acknowledgeUserDataCollection")], again - assert record()[1]["property"] == doc["property"] - ok("a second setup reuses the property, stream, secret and key events (no duplicate creates)") + assert not [c for c in calls()[n:] if c["method"] != "GET"] and record()[1]["property"] == doc["property"] + env = dict(ENV, ALADDIN_GA_TOKEN_COMMAND="echo fake-token") + env.pop("ALADDIN_GA_ACCESS_TOKEN") + assert run("setup", env=env).returncode == 0 # the token command path + r = run("setup", env=dict(ENV, ALADDIN_GA_ACCESS_TOKEN="expired")) + assert r.returncode == 3 and "gcloud auth application-default login" in r.stderr, r.stderr # 401 → sign in + fake("/_control", {"redirect": "/stolen"}) + r = run("setup") + assert r.returncode == 2 and "302" in r.stderr, r.stderr + assert not [c for c in calls() if c["path"] == "/stolen"] # the token never follows + fake("/_control", {"redirect": ""}) + path, doc = record() + lost = dict(doc, property="properties/999") + store = lambda d: (M.seal(d), M.save(path, d)) # noqa: E731 + store(lost) + r = run("setup") + assert r.returncode == 3 and "not reachable" in r.stderr and "--account" in r.stderr, r.stderr + store(doc) + ok("a rerun creates nothing; a token command works; 401 asks; redirects are refused; a lost property asks") - # 7) the static emit check: every active journey step's event must be emitted by source code + # 7) the static emit check: every active journey step's event in non-test code; the app carries the stream r = run("check") assert r.returncode == 2 and "signup_started" in r.stderr and "not emitted" in r.stderr, r.stderr write("src/app.js", "gtag('event', 'signup_started');\ngtag('event', \"signup_done\");\ntrack(`todo_add_opened`);\n") - write("src/app.test.js", "expect(sent).toContain('todo_added')\n") # a test mentioning it does not count - write("docs/notes.md", "todo_added is emitted when … \n") # nor do docs + for rel in ("src/app.test.js", "docs/site/demo.js", "tests/helpers.js", "test_flow.py", ".storybook/preview.js", + "cypress/e2e/signup.cy.ts", "todo/tests.py", "src/conftest.py", "e2e/flow.ts", "src/TodoTest.java"): + write(rel, "expect(sent).toContain('todo_added')\n") # tests, docs, dot-dirs never count + write("src/later.js", "// TODO: send todo_added\n") # nor does a bare word + write("README.md", "todo_added is emitted when … 'todo_added'\n") # nor docs at the root r = run("check", "--json") assert r.returncode == 2, r.stderr rep = json.loads(r.stdout) @@ -211,71 +257,105 @@ def write(rel, text): assert "old_step" not in r.stderr # retired journeys are not checked write("src/todo.js", "export const add = () => gtag('event', 'todo_added');\n") r = run("check", "--json") - assert r.returncode == 2 and "debug hit" in r.stderr, r.stderr # instrumented, not yet proven + assert r.returncode == 2 and "measurement id" in r.stderr, r.stderr # instrumented, but to no stream assert json.loads(r.stdout)["emitted"]["todo_added"] == "src/todo.js:1", r.stdout - jid = journey("bad names", ["session_start", "2fast", "x" * 41]) + mid = doc["stream"]["measurement_id"] + write("src/ga.js", "gtag('config', '%s');\n" % mid) + r = run("check") + assert r.returncode == 2 and "measurement id" not in r.stderr and "no debug hit" in r.stderr, r.stderr + jid = journey("bad names", ["session_start", "2fast", "x" * 41, "google_x"]) r = run("check") - assert r.returncode == 2 and "session_start" in r.stderr and "reserved" in r.stderr and "2fast" in r.stderr, r.stderr + assert r.returncode == 2 and "'session_start' is reserved" in r.stderr and "2fast" in r.stderr, r.stderr + assert "'google_x' is reserved" in r.stderr os.remove(M.path_of(T, "journey", jid)) - ok("emit check: each active journey step's event appears in non-test source; GA name rules apply") + ok("emit check: quoted literals in non-test code (tests, docs, dot-dirs, comments ignored); the stream's id") - # 8) the debug hit: journey events go to GA's validation server (no OAuth token sent), and stay fresh + # 8) the debug hit: the secret is proven live on the Admin API, then GA's validation server checks the events n = len(calls()) r = run("debug-hit") assert r.returncode == 0, r.stderr - hits = calls()[n:] - assert len(hits) == 1 and hits[0]["path"] == "/mp/debug/mp/collect" and hits[0]["auth"] == "", hits - assert "measurement_id=G-" in hits[0]["query"] and "api_secret=" in hits[0]["query"] - assert sorted(e["name"] for e in hits[0]["body"]["events"]) == sorted(TM.journey_events(T)), hits[0]["body"] - assert hits[0]["body"]["client_id"] == TM.DEBUG_CLIENT_ID - r = run("check") - assert r.returncode == 0, r.stderr - journey("share a list", ["list_shared"]) - write("src/share.js", "gtag('event', 'list_shared')\n") + log = calls()[n:] + hits = mp_calls(log) + assert len(hits) == 1 and hits[0]["auth"] == "" and "measurement_id=%s" % mid in hits[0]["query"], hits + assert "api_secret=" + secret["api_secret"] in hits[0]["query"] + assert [c["path"] for c in log if c["path"].endswith("measurementProtocolSecrets")] and \ + all(c["auth"] == "Bearer fake-token" for c in log if c["path"].startswith("/admin")) + body = hits[0]["body"] + assert sorted(e["name"] for e in body["events"]) == TM.journey_events(T), body + assert body["client_id"] == TM.DEBUG_CLIENT_ID and body["validation_behavior"] == "ENFORCE_RECOMMENDATIONS" + assert run("check").returncode == 0 + journey("share a list", ["list_opened", "list_shared"]) r = run("check") - assert r.returncode == 2 and "stale" in r.stderr, r.stderr # a new event needs a new hit + assert r.returncode == 2 and "stale" in r.stderr and "key events" in r.stderr, r.stderr + subprocess.run(GIT + ["commit", "-qm", "app"], cwd=T, check=True) + subprocess.run(["git", "worktree", "add", "-q", WT, "HEAD"], cwd=T, check=True) + write("src/share.js", "gtag('event', 'list_opened'); gtag('event', 'list_shared')\n", root=WT) + rw = json.loads(run("check", "--json", cwd=WT).stdout) # a worker's worktree: its own code + assert rw["emitted"]["list_shared"] == "src/share.js:1" and not rw["missing"], rw + rm = json.loads(run("check", "--json").stdout) + assert [m["event"] for m in rm["missing"]] == ["list_opened", "list_shared"], rm + write("src/share.js", "gtag('event', 'list_opened'); gtag('event', 'list_shared')\n") + n = len(calls()) + assert run("setup").returncode == 0 and record()[1]["key_events"] == ["list_shared", "signup_done", "todo_added"] + assert [c["body"]["eventName"] for c in calls()[n:] if c["method"] == "POST"] == ["list_shared"] assert run("debug-hit").returncode == 0 and run("check").returncode == 0 - sf = json.load(open(sfile)) - json.dump(dict(sf, api_secret="wrong"), open(sfile, "w")) + json.dump(dict(secret, api_secret="wrong"), open(sfile, "w")) + n = len(calls()) r = run("debug-hit") - assert r.returncode == 2 and "api_secret" in r.stderr, r.stderr # GA's validation messages surface + assert r.returncode == 2 and "not live" in r.stderr and not mp_calls(calls()[n:]), r.stderr assert run("check").returncode == 2 # a failed hit is not a pass - json.dump(sf, open(sfile, "w")) + json.dump(secret, open(sfile, "w")) + fake("/_control", {"mp_messages": [{"fieldPath": "events.params", "validationCode": "VALUE_INVALID", + "description": "Param [p] has invalid type."}]}) + r = run("debug-hit") + assert r.returncode == 2 and "VALUE_INVALID" in r.stderr, r.stderr # GA's own messages surface + fake("/_control", {"mp_messages": []}) + bulk = journey("bulk", ["bulk_%02d" % i for i in range(30)]) + write("src/bulk.js", "\n".join("track('bulk_%02d')" % i for i in range(30)) + "\n") + n = len(calls()) + assert run("debug-hit").returncode == 0 + sizes = [len(c["body"]["events"]) for c in mp_calls(calls()[n:])] + assert sizes == [25, 11], sizes # 36 events in batches of ≤25 + os.remove(M.path_of(T, "journey", bulk)) + unwrite("src/bulk.js") assert run("debug-hit").returncode == 0 and run("check").returncode == 0 - ok("debug hit: every journey event validated by GA's debug endpoint; a new event or a failed hit blocks") + ok("debug hit: secret live on the stream, events validated in ≤25 batches; stale, failed or not-live blocks") - # 9) fixed reports on settled data; the minimum-sample rule decides usage's share; never a gate + # 9) fixed reports scoped to the stream, on settled data; the minimum-sample rule decides usage's share fake("/_control", {"users": 12, "new_users": 9, "events": {"signup_done": [14, 9], "todo_added": [40, 7], "page_view": [99, 12]}}) + sid = record()[1]["stream"]["name"].rsplit("/", 1)[-1] n = len(calls()) r = run("report", "--json") assert r.returncode == 0, r.stderr rep = json.loads(r.stdout) sent = [c["body"] for c in calls()[n:]] + want = [TM.request("users", [], sid), TM.request("journeys", TM.journey_events(T), sid)] + assert sorted(json.dumps(b, sort_keys=True) for b in sent) == sorted(json.dumps(b, sort_keys=True) for b in want), sent assert all(b["dateRanges"] == [{"startDate": "30daysAgo", "endDate": "3daysAgo"}] for b in sent), sent - assert {json.dumps(TM.request("users", []), sort_keys=True), json.dumps(TM.request("journeys", TM.journey_events(T)), sort_keys=True)} \ - == {json.dumps(b, sort_keys=True) for b in sent} + assert want[0]["dimensionFilter"]["filter"] == {"fieldName": "streamId", "stringFilter": {"matchType": "EXACT", "value": sid}} assert rep["kind"] == "settled" and rep["sample"] == {"users": 12, "min_users": 50, "days": 28, "thresholded": False, "met": False} assert rep["share"]["usage"] == 0 and rep["share"]["research"] == 1, rep["share"] - assert "page_view" not in json.dumps(rep["results"]["journeys"]), rep # only journey events are read - assert rep["definitions_sha256"] == M.digest(TM.REPORTS, drop=()) + assert "page_view" not in json.dumps(rep["results"]["journeys"]), rep # only journey events + assert rep["definitions_sha256"] == M.digest(TM.REPORTS, drop=()) and rep["stream"] == record()[1]["stream"]["name"] assert os.path.dirname(rep["path"]) == os.path.join("docs", "agents", "mission", "telemetry", "reports") - assert run("check").returncode == 0 # below the sample: still passes + assert run("check").returncode == 0 # below the sample: passes fake("/_control", {"users": 80, "new_users": 50}) rep = json.loads(run("report", "--json").stdout) assert rep["sample"]["met"] and rep["share"] == {"usage": 0.7, "research": 0.3, "reason": rep["share"]["reason"]}, rep fake("/_control", {"thresholded": True}) rep = json.loads(run("report", "--json").stdout) assert rep["sample"]["thresholded"] and not rep["sample"]["met"] and rep["share"]["usage"] == 0, rep - fake("/_control", {"thresholded": False, "realtime": {"todo_added": [3, 2]}}) + fake("/_control", {"thresholded": False, "realtime": {"todo_added": [3, 2], "page_view": [9, 4]}}) + n = len(calls()) r = run("report", "--realtime", "--json") rt = json.loads(r.stdout) - assert r.returncode == 0 and rt["kind"] == "realtime" and rt["share"] is None, rt # realtime never drives a goal - assert [c["path"] for c in calls()][-1].endswith(":runRealtimeReport") - r = run("share", "--json") - sh = json.loads(r.stdout) - assert r.returncode == 0 and sh["usage"] == 0 and sh["report"] != rt["report_id"], sh # latest SETTLED report + assert r.returncode == 0 and rt["kind"] == "realtime" and rt["share"] is None, rt # never drives a goal + assert rt["results"]["realtime"]["rows"] == [{"eventName": "todo_added", "eventCount": 3, "activeUsers": 2}], rt + assert [c["path"] for c in calls()[n:]][-1].endswith(":runRealtimeReport") + sh = json.loads(run("share", "--json").stdout) + assert sh["usage"] == 0 and sh["report"] != rt["report_id"], sh # latest SETTLED report assert len(events("telemetry.report")) == 4 and run("check").returncode == 0 - ok("reports: fixed definitions on settled data (≥48h), usage leads only past 50 users/28d, thresholding = below") + ok("reports: fixed definitions scoped to the stream, settled data (≥48h); usage leads only past 50 users/28d") # 10) the minimum-sample rule is code with fixed boundaries; report definitions stay aggregate-only assert TM.share(49, False, True)["usage"] == 0 and TM.share(50, False, True)["usage"] == 0.7 @@ -284,42 +364,75 @@ def write(rel, text): for name, d in TM.REPORTS.items(): dims = [x["name"] for x in d["body"].get("dimensions", [])] assert set(dims) <= TM.AGGREGATE_DIMENSIONS, (name, dims) + assert "streamId" in json.dumps(TM.request(name, ["x"], "1")), name assert not TM.AGGREGATE_DIMENSIONS & {"customUser:id", "userId", "clientId", "city"} - ok("share(): 49 users → 0, 50 → 0.7, thresholded or unsettled → 0, research ≥ 30%; reports read aggregates only") + ok("share(): 49 users → 0, 50 → 0.7, thresholded or unsettled → 0, research ≥ 30%; aggregate, stream-scoped") - # 11) a tampered report is ignored by share(); revoke stops all network and turns the check off + # 11) share skips tampered reports; revoke stops the agents (share included) and deletes the local secret reports = sorted(os.listdir(M.mdir(T, "telemetry", "reports"))) assert len(reports) == 4 - sh_before = json.loads(run("share", "--json").stdout)["report"] - p = M.mdir(T, "telemetry", "reports", sh_before + ".json") + latest = json.loads(run("share", "--json").stdout)["report"] + p = M.mdir(T, "telemetry", "reports", latest + ".json") raw = json.load(open(p)) - raw["sample"]["users"] = 10 ** 6 + raw["sample"]["thresholded"] = False json.dump(raw, open(p, "w")) - assert json.loads(run("share", "--json").stdout)["report"] != sh_before + sh = json.loads(run("share", "--json").stdout) + assert sh["report"] != latest and sh["usage"] == 0.7 and sh["users"] == 80, sh # the one before it assert run("revoke").returncode == 64 r = run("revoke", "--by", "Adeel", "--reason", "pausing") - assert r.returncode == 0, r.stderr - assert record()[1]["status"] == "revoked" and len(events("telemetry.revoked")) == 1 + assert r.returncode == 0 and "keeps sending" in r.stderr, r.stderr + assert record()[1]["status"] == "revoked" and len(events("telemetry.revoked")) == 1 and not os.path.exists(sfile) + sh = json.loads(run("share", "--json").stdout) + assert sh["usage"] == 0 and sh["report"] is None, sh n = len(calls()) for cmd in (["setup"], ["debug-hit"], ["report"]): r = run(*cmd) assert r.returncode == 3 and "revoked" in r.stderr, (cmd, r.stderr) assert calls()[n:] == [] r = run("check") - assert r.returncode == 0 and "revoked" in r.stderr, r.stderr + assert r.returncode == 0 and "revoked" in r.stderr and "src/ga.js" in r.stderr, r.stderr # names the live tag assert run("consent", "--by", "Adeel").returncode == 0 and record()[1]["status"] == "configured" - assert run("check").returncode == 0 - ok("revoke: setup, debug-hit and report refuse with no request; a fresh consent resumes; tampered reports skipped") + assert run("debug-hit").returncode == 2 # the secret is gone: setup again + n = len(calls()) + assert run("setup").returncode == 0 and not [c for c in calls()[n:] if c["method"] == "POST"] + assert run("debug-hit").returncode == 0 and run("check").returncode == 0 + r = run("init", "--uri", "https://next.example", "--force") + path, doc = record() + assert r.returncode == 0 and doc["stream"] is None and doc["status"] == "off" and doc["property"], doc + assert run("check").returncode == 3 and run("setup").returncode == 3 # new terms, new consent + assert run("consent", "--by", "Adeel").returncode == 0 and record()[1]["status"] == "consented" + assert run("setup").returncode == 0 and record()[1]["stream"]["default_uri"] == "https://next.example" + r = run("check") + assert r.returncode == 2 and "measurement id" in r.stderr, r.stderr # the new stream is not in the app + ok("revoke: agents stop (share too), secret deleted, live tag named; a new uri drops the old stream") # 12) discovery and help: the area is found by name, its docstring is its help r = subprocess.run([MI, "--list"], cwd=T, env=ENV, capture_output=True, text=True) assert "telemetry" in r.stdout.split(), r.stdout r = run("--help") assert r.returncode == 0 and "never a gate" in r.stdout and "mission telemetry setup" in r.stdout, r.stdout - assert run("nope").returncode == 64 + n = len(calls()) + assert run("nope").returncode == 64 and run("debug-hit", "--help").returncode == 0 and calls()[n:] == [] ok("`mission telemetry` is discovered; --help is the module docstring; unknown subcommands are usage errors") + + # 13) day one, no journeys yet: the debug hit sends a probe event so the wiring is still proven + subprocess.run(["git", "init", "-q"], cwd=T2, check=True) + env = dict(ENV, HOME=os.path.join(T2, "home")) + assert run("init", "--uri", "https://probe.example", env=env, cwd=T2).returncode == 0 + assert run("consent", "--by", "Adeel", env=env, cwd=T2).returncode == 0 + assert run("setup", "--account", "accounts/100", env=env, cwd=T2).returncode == 0 + n = len(calls()) + assert run("debug-hit", env=env, cwd=T2).returncode == 0 + assert [e["name"] for e in mp_calls(calls()[n:])[0]["body"]["events"]] == [TM.PROBE_EVENT] + assert run("check", env=env, cwd=T2).returncode == 2 + write("index.html", "\n" % record(T2)[1]["stream"]["measurement_id"], root=T2) + r = run("check", env=env, cwd=T2) + assert r.returncode == 0, r.stderr + ok("no journeys yet: the probe event proves the wiring; check passes once the page carries the stream") finally: FAKE.terminate() FAKE.wait(timeout=10) + for d in (T, WT, T2): + shutil.rmtree(d, ignore_errors=True) print("mission-telemetry: %d checks passed" % PASS[0])