From d8fbd75fba50be67d9defd4d681ba98d9db4a7fc Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 09:06:54 +0000 Subject: [PATCH] ci(threat-model): stop every pair of threat-model PRs conflicting on prose_version threat-model.yaml kept a sha256 of the whole prose, so any two pull requests that changed threat-model.md both rewrote that one line and the second to merge conflicted (#400, #508). The YAML's prose_version now names the prose in the same commit, and the validator's new --base check (run by CI against the merge commit's parent) fails a change to the prose that changes neither companion, which is what the hash enforced, without a line every change edits. Adds tests, including two branches that change the prose in different places merging cleanly and still validating. Fixes #507 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Bwc5go44W4hjHKu329MepB --- .github/workflows/threat-model.yml | 15 +- DEVLOG.md | 6 +- ci/test_validate_threat_model.py | 215 +++++++++++++++++++++++++++++ ci/validate_threat_model.py | 64 +++++++-- docs/requirements.md | 2 +- threat-model.md | 5 +- threat-model.yaml | 7 +- 7 files changed, 291 insertions(+), 23 deletions(-) create mode 100644 ci/test_validate_threat_model.py diff --git a/.github/workflows/threat-model.yml b/.github/workflows/threat-model.yml index fd279bd67..ccc7ba4f1 100644 --- a/.github/workflows/threat-model.yml +++ b/.github/workflows/threat-model.yml @@ -1,13 +1,13 @@ name: threat model -# Checks that threat-model.md (canonical), threat-model.yaml and threat-model.json agree: the YAML's prose hash, the -# property ids in all three, and the open-question numbers. See ci/validate_threat_model.py. +# Checks that threat-model.md (canonical), threat-model.yaml and threat-model.json agree: a change to the prose changes +# a companion too, the property ids in all three, and the open-question numbers. See ci/validate_threat_model.py. on: push: branches: [main] - paths: [threat-model.md, threat-model.yaml, threat-model.json, ci/validate_threat_model.py, .github/workflows/threat-model.yml] + paths: [threat-model.md, threat-model.yaml, threat-model.json, ci/validate_threat_model.py, ci/test_validate_threat_model.py, .github/workflows/threat-model.yml] pull_request: - paths: [threat-model.md, threat-model.yaml, threat-model.json, ci/validate_threat_model.py, .github/workflows/threat-model.yml] + paths: [threat-model.md, threat-model.yaml, threat-model.json, ci/validate_threat_model.py, ci/test_validate_threat_model.py, .github/workflows/threat-model.yml] workflow_dispatch: concurrency: @@ -23,6 +23,11 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v7 + with: + # The parent too: a pull request's merge commit or the push, compared with what it's based on + fetch-depth: 2 - uses: astral-sh/setup-uv@v7 - name: Validate the threat model - run: uv run --no-project --python 3.12 --with pyyaml python ci/validate_threat_model.py + run: uv run --no-project --python 3.12 --with pyyaml python ci/validate_threat_model.py ${{ github.event_name != 'workflow_dispatch' && '--base HEAD^1' || '' }} + - name: Test the validator + run: uv run --no-project --python 3.12 --with pyyaml python ci/test_validate_threat_model.py diff --git a/DEVLOG.md b/DEVLOG.md index c8cab4ee5..a92cdce0a 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -231,8 +231,10 @@ Todo: - [x] Align SECURITY.md's SSRF example ("a model server") with the admin-URL ruling (YAML hash refreshed; the plugin's validator was not available in the session that made this edit, so re-run it) -Watch out: the prose is canonical. After any edit to `threat-model.md`, regenerate the YAML and JSON (its hash is in -the YAML) and run `ci/validate_threat_model.py` (CI runs it on every change to these files). +Watch out: the prose is canonical. After any edit to `threat-model.md`, regenerate the YAML and JSON and run +`ci/validate_threat_model.py --base origin/main` (CI runs it on every change to these files and fails a prose change +that changes neither companion). The YAML no longer holds a hash of the prose, which made every pair of threat-model +pull requests conflict (#507). ## 2026-10-04 · Graph history: versions, replay, rollback diff --git a/ci/test_validate_threat_model.py b/ci/test_validate_threat_model.py new file mode 100644 index 000000000..e1255ca0d --- /dev/null +++ b/ci/test_validate_threat_model.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Tests for validate_threat_model.py: python3 ci/test_validate_threat_model.py (needs pyyaml and git)""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +import yaml + +from validate_threat_model import PROSE_VERSION, changed_since, validate + +PROSE = """# Model + +## 1.11 Provided + +| ID | Property | +| --- | --- | +| P-ONE | One. | + +## 1.12 Not provided + +| ID | Not provided | +| --- | --- | +| N-ONE | One. | + +## 1.15 Known non-findings + +| ID | What | +| --- | --- | +| KNF-ONE | One. | + +## 1.18 Open questions + +- **Q1**: why? +""" + +SIDECAR = { + "schema": "threat-model-sidecar/v2", + "prose_version": PROSE_VERSION, + "x-modeled-commit": "abc", + "x-model-date": "2026-10-17", + "x-repository": "https://example.com/lens", + "properties_claimed": [{"id": "P-ONE"}], + "components": [], + "entry_points": [], + "adversaries": [], + "downstream_responsibilities": [], + "known_misuses": [], + "dispositions": [], + "known_non_findings": [{"id": "KNF-ONE"}], + "properties_disclaimed": [{"id": "N-ONE"}], +} + +EXPORT = { + "spec_version": 1, + "repository": "https://example.com/lens", + "commit": "abc", + "date": "2026-10-17", + "properties_provided": [{"property": "P-ONE"}], + "components": [], + "entry_points": [], + "adversaries": [], + "downstream_responsibilities": [], + "known_misuse": [], + "dispositions": [], + "known_non_findings": [{"claim": "one"}], + "open_questions": [{"claim": "Q1: why?"}], + "properties_not_provided": [{"property": "N-ONE"}], +} + +GIT_ENV = { + **os.environ, + "GIT_AUTHOR_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@localhost", + "GIT_COMMITTER_NAME": "test", + "GIT_COMMITTER_EMAIL": "test@localhost", +} + + +class Model: + """A threat model in a throwaway git repository.""" + + def __init__(self, root: Path): + self.root = root + self.write(PROSE, SIDECAR, EXPORT) + self.git("init", "-q", "-b", "main") + self.commit("model") + + def write(self, prose=None, sidecar=None, export=None): + if prose is not None: + (self.root / "threat-model.md").write_text(prose) + if sidecar is not None: + (self.root / "threat-model.yaml").write_text( + yaml.safe_dump(sidecar, sort_keys=False) + ) + if export is not None: + (self.root / "threat-model.json").write_text( + json.dumps(export, indent=2) + "\n" + ) + + def read(self): + return ( + (self.root / "threat-model.md").read_text(), + yaml.safe_load((self.root / "threat-model.yaml").read_text()), + json.loads((self.root / "threat-model.json").read_text()), + ) + + def git(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-c", "commit.gpgsign=false", *args], + cwd=self.root, + env=GIT_ENV, + capture_output=True, + text=True, + ) + + def commit(self, message: str): + self.git("add", "-A") + done = self.git("commit", "-q", "-m", message) + assert done.returncode == 0, done.stderr + + +def add_claimed(model: Model, pid: str): + prose, sidecar, export = model.read() + prose = prose.replace("| P-ONE | One. |\n", f"| P-ONE | One. |\n| {pid} | Two. |\n") + sidecar["properties_claimed"].append({"id": pid}) + export["properties_provided"].append({"property": pid}) + model.write(prose, sidecar, export) + + +def add_disclaimed(model: Model, nid: str): + prose, sidecar, export = model.read() + prose = prose.replace("| N-ONE | One. |\n", f"| N-ONE | One. |\n| {nid} | Two. |\n") + sidecar["properties_disclaimed"].append({"id": nid}) + export["properties_not_provided"].append({"property": nid}) + model.write(prose, sidecar, export) + + +class ValidateTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.model = Model(Path(self.tmp.name)) + + def tearDown(self): + self.tmp.cleanup() + + def test_agreeing_model_passes(self): + self.assertEqual(validate(self.model.root), []) + self.assertEqual(changed_since(self.model.root, "HEAD"), []) + + def test_prose_hash_is_refused(self): + self.model.write( + sidecar={**SIDECAR, "prose_version": "threat-model.md@sha256:00"} + ) + [error] = validate(self.model.root) + self.assertIn(PROSE_VERSION, error) + + def test_id_only_in_prose_is_caught(self): + prose, _, _ = self.model.read() + self.model.write( + prose.replace("| P-ONE | One. |\n", "| P-ONE | One. |\n| P-TWO | Two. |\n") + ) + self.assertIn( + "P-TWO is defined in threat-model.md but not in threat-model.yaml", + validate(self.model.root), + ) + + def test_prose_change_without_companions_fails(self): + self.model.write(PROSE.replace("why?", "why not?")) + [error] = changed_since(self.model.root, "HEAD") + self.assertIn("threat-model.md changed since HEAD", error) + self.model.commit("prose only") + self.assertEqual(len(changed_since(self.model.root, "HEAD~1")), 1) + + def test_prose_change_with_a_companion_passes(self): + self.model.write( + PROSE.replace("why?", "why not?"), + export={**EXPORT, "open_questions": [{"claim": "Q1: why not?"}]}, + ) + self.assertEqual(changed_since(self.model.root, "HEAD"), []) + self.model.commit("prose and export") + self.assertEqual(changed_since(self.model.root, "HEAD~1"), []) + + def test_companions_alone_may_change(self): + self.model.write(sidecar={**SIDECAR, "x-note": "tidied"}) + self.assertEqual(changed_since(self.model.root, "HEAD"), []) + + def test_unknown_base_is_an_error(self): + [error] = changed_since(self.model.root, "no-such-ref") + self.assertIn("can't compare with 'no-such-ref'", error) + + def test_two_changes_to_the_prose_merge_cleanly(self): + # Issue #507: each branch changes the prose and the companions in its own place, so they merge without a + # conflict and the merge still validates. A hash of the whole prose in the YAML made every such pair conflict. + m = self.model + m.git("checkout", "-q", "-b", "claimed") + add_claimed(m, "P-TWO") + m.commit("claim P-TWO") + m.git("checkout", "-q", "main") + add_disclaimed(m, "N-TWO") + m.commit("disclaim N-TWO") + merged = m.git("merge", "-q", "--no-edit", "claimed") + self.assertEqual(merged.returncode, 0, merged.stdout + merged.stderr) + self.assertEqual(validate(m.root), []) + self.assertEqual(changed_since(m.root, "HEAD^1"), []) + self.assertEqual(changed_since(m.root, "HEAD^2"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/validate_threat_model.py b/ci/validate_threat_model.py index 9977ae68c..24034d1e2 100644 --- a/ci/validate_threat_model.py +++ b/ci/validate_threat_model.py @@ -4,7 +4,10 @@ The prose (threat-model.md) is canonical; the YAML and JSON are derived from it. This script is the repo's own stand-in for the threat-model plugin's validator, which lives outside the repo and so can't run in CI. It checks: -- the YAML's prose_version is the sha256 of threat-model.md; +- with --base REF, a change to threat-model.md since REF also changes threat-model.yaml or threat-model.json, so the + companions are regenerated with the prose (this replaced a sha256 of the prose kept in the YAML, which every pair of + pull requests touching the prose conflicted on, #507); +- the YAML's prose_version names the prose in the same commit; - both companions parse and carry their required top-level keys; - the YAML and JSON name the same repository, commit and date; - the claimed (P-), disclaimed (N-) and known-non-finding (KNF-) ids are the same set in all three files, and the @@ -12,19 +15,26 @@ - every P-, N-, KNF- or MX- id mentioned anywhere is one of those defined ids; - the open questions are numbered Q1..Qn in the JSON, and every Qn cited anywhere is one of them. -Usage: python ci/validate_threat_model.py [repo_root]. Exits 1 and lists every problem when anything disagrees. +Usage: python ci/validate_threat_model.py [--base REF] [repo_root]. CI passes --base HEAD^1; locally, use +--base origin/main. Exits 1 and lists every problem when anything disagrees. """ from __future__ import annotations -import hashlib +import argparse import json import re +import subprocess import sys from pathlib import Path import yaml +PROSE = "threat-model.md" +COMPANIONS = ("threat-model.yaml", "threat-model.json") +# The prose in the same commit. Not a hash: a hash of the whole prose is one line every prose change edits (#507). +PROSE_VERSION = "threat-model.md@same-commit" + ID_RE = re.compile( r"(? dict[str, set[str]]: def validate(root: Path) -> list[str]: errors: list[str] = [] - prose_raw = (root / "threat-model.md").read_bytes() - prose = prose_raw.decode() + prose = (root / "threat-model.md").read_text() yaml_text = (root / "threat-model.yaml").read_text() json_text = (root / "threat-model.json").read_text() try: @@ -63,11 +72,10 @@ def validate(root: Path) -> list[str]: except json.JSONDecodeError as e: return [f"threat-model.json does not parse: {e}"] - # Prose hash. - want = "threat-model.md@sha256:" + hashlib.sha256(prose_raw).hexdigest() - if y.get("prose_version") != want: + if y.get("prose_version") != PROSE_VERSION: errors.append( - f"threat-model.yaml prose_version is {y.get('prose_version')!r}; the prose hashes to {want!r}" + f"threat-model.yaml prose_version is {y.get('prose_version')!r}, expected {PROSE_VERSION!r} " + "(the prose in the same commit; ci/validate_threat_model.py --base checks the companions changed with it)" ) # Shape. @@ -170,11 +178,43 @@ def validate(root: Path) -> list[str]: return errors +def changed_since(root: Path, base: str) -> list[str]: + """Problems with a change since base: the prose changed but neither companion did.""" + git = ["git", "-C", str(root)] + since = subprocess.run( + [*git, "merge-base", base, "HEAD"], capture_output=True, text=True + ) + if since.returncode: + return [f"can't compare with {base!r}: {since.stderr.strip()}"] + # Against the working tree, so uncommitted edits count too. + changed = subprocess.run( + [*git, "diff", "--name-only", since.stdout.strip(), "--", PROSE, *COMPANIONS], + capture_output=True, + text=True, + check=True, + ).stdout.split() + if PROSE in changed and not any(c in changed for c in COMPANIONS): + return [ + f"{PROSE} changed since {base} but neither {' nor '.join(COMPANIONS)} did: regenerate them from the prose" + ] + return [] + + def main() -> int: - root = Path( - sys.argv[1] if len(sys.argv) > 1 else Path(__file__).resolve().parent.parent + parser = argparse.ArgumentParser( + description="Check that threat-model.md, .yaml and .json agree." + ) + parser.add_argument( + "--base", + help="also check that a change to the prose since this ref changes a companion", + ) + parser.add_argument( + "root", nargs="?", default=Path(__file__).resolve().parent.parent, type=Path ) - errors = validate(root) + args = parser.parse_args() + errors = validate(args.root) + if args.base: + errors += changed_since(args.root, args.base) for e in errors: print(f"error: {e}") if errors: diff --git a/docs/requirements.md b/docs/requirements.md index b8552cd7e..02a380d46 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -231,7 +231,7 @@ request, to another check that proves the same thing; loosening a requirement or | Chat-room answers have the links they built broken up; agents over MCP get the archive as data | [test_prompt_injection.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_prompt_injection.py) `test_a_room_gets_the_links_the_answer_built_broken_up`; [test_mcp.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_mcp.py) `test_what_the_archive_says_reaches_agents_as_data` | | HTTPS set up in Settings, with certificates that renew, only for its own domains and optionally only for client certificates | [test_https.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_https.py) `test_https_is_set_up_in_the_app`, `test_caddys_configuration_follows_the_settings`, `test_strict_sni_client_certificates_and_tls_13` | | Remote access through a Cloudflare tunnel | [test_tunnel.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_tunnel.py) `test_remote_access_is_set_up_in_the_app`, `test_a_quick_tunnel_runs_in_one_process_and_stops_when_turned_off`, `test_a_managed_tunnel_is_made_routed_and_given_a_dns_record` | -| The threat model's three files agree | [threat-model.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/threat-model.yml) `validate`, `Validate the threat model`; [validate_threat_model.py](https://github.com/adeelahmad/lense/blob/main/ci/validate_threat_model.py) | +| The threat model's three files agree, and a change to the prose comes with its companions, without a line every change edits | [threat-model.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/threat-model.yml) `validate`, `Validate the threat model`, `Test the validator`; [validate_threat_model.py](https://github.com/adeelahmad/lense/blob/main/ci/validate_threat_model.py); [test_validate_threat_model.py](https://github.com/adeelahmad/lense/blob/main/ci/test_validate_threat_model.py) `test_prose_change_without_companions_fails`, `test_two_changes_to_the_prose_merge_cleanly` | ## Use open standards diff --git a/threat-model.md b/threat-model.md index e938f05aa..dde8c2ba8 100644 --- a/threat-model.md +++ b/threat-model.md @@ -920,7 +920,10 @@ The sections each answer "lands in" are not yet rewritten; until they are, this ## 1.19 Machine-readable companions `threat-model.yaml` (sidecar, schema `threat-model-sidecar/v2`) and `threat-model.json` (flat export) sit beside -this file. Authority order: prose > YAML > JSON. Regenerate both whenever this prose changes. +this file. Authority order: prose > YAML > JSON. Regenerate both whenever this prose changes: CI +(`ci/validate_threat_model.py`) fails a change to this file that changes neither of them, and checks that the three +define the same ids. The YAML's `prose_version` names this file in the same commit rather than a hash of it, so +changes to different parts of the prose don't conflict. ## Appendix: prior security-policy back-map diff --git a/threat-model.yaml b/threat-model.yaml index b31e307a0..567079594 100644 --- a/threat-model.yaml +++ b/threat-model.yaml @@ -1,8 +1,11 @@ # threat-model.yaml — derived index; the prose threat model (threat-model.md) is canonical. -# Authority order: prose > yaml > json. Regenerate whenever the prose changes. +# Authority order: prose > yaml > json. Regenerate whenever the prose changes: CI fails a change to the prose that +# changes neither this file nor threat-model.json (ci/validate_threat_model.py --base). schema: threat-model-sidecar/v2 project: lens -prose_version: "threat-model.md@sha256:7914ab4b9c598440e8026ac0db1b968f42da60369831cac0884d9afc7ef42d60" +# The prose in the same commit, not a hash of it: a hash is one line every change to the prose edits, so any two pull +# requests touching the prose conflicted on it (#507). `git hash-object threat-model.md` names the exact prose. +prose_version: "threat-model.md@same-commit" model_status: under-review triage_policy: strict confidence: {documented: 205, maintainer: 46, inferred: 91, assumption: 8}