diff --git a/.github/workflows/bump-changelog-entry.cjs b/.github/workflows/bump-changelog-entry.cjs index 3258c5e..d1024e8 100644 --- a/.github/workflows/bump-changelog-entry.cjs +++ b/.github/workflows/bump-changelog-entry.cjs @@ -1,8 +1,14 @@ 'use strict'; // Installed by the engineering agent loop. Edit it in dlt-hub/engineering-agent-loop, not -// here. Writes a dependency bump's changelog entry from the two `uv.lock` files before and -// after the recipe ran, as a draft the reviewing pass then holds to the diff. +// here. Writes a dependency bump's changelog entry from the two `uv.lock` files, as a draft +// the reviewing pass then holds to the diff. +// +// The `--*-before` state the caller passes is the last release, not the tip the recipe just +// overwrote, so the entry is the whole account of what is unreleased. That is what lets a +// second bump before a release replace the bullets the bump before it wrote: both describe +// the same span, one of them a bump out of date. Appending instead grew `[Unreleased]` a +// second `Refreshed ...` line per run. const fs = require('node:fs'); @@ -86,13 +92,32 @@ function directSpecs(text) { const versions = (list) => list.map((p) => `\`${p.name}\` ${p.from} → ${p.to}`); +// How each of the four bullets opens. A second bump finds the first one's lines by these, so +// rewording one here is what rewords it everywhere, and a line a person wrote is left alone +// because it opens with something else +const LEADS = { + scaffold: 'Refreshed the bundled minimal workspace `uv.lock` to the latest' + + ' within-constraint versions', + root: 'Refreshed the root `uv.lock` to the latest within-constraint versions', + deps: "Synced the bundled minimal workspace dependencies with dlt's `workspace_deps()`", + skills: 'Refreshed the bundled skills to workbench', +}; + +/** Whether a changelog line is a bullet an earlier run of this script wrote. */ +const isOwnBullet = (line) => Object.values(LEADS).some((lead) => line.startsWith(`- ${lead}`)); + +/** The `LEADS` keys a run was given both sides of, which are the bullets it may replace. */ +function askedAbout({ scaffold, root, deps, skills }) { + return Object.keys(LEADS).filter((kind) => ({ scaffold, root, deps, skills })[kind]); +} + // direct dependencies carry both versions and transitives are counted, which is the shape // of every released entry -function lockBullet({ what, diff, direct }) { +function lockBullet({ lead, diff, direct }) { const isDirect = (p) => direct.has(p.name); const named = diff.moved.filter(isDirect); const rest = diff.moved.length - named.length; - const parts = [`Refreshed ${what} to the latest within-constraint versions`]; + const parts = [lead]; if (named.length) parts.push(` (${versions(named).join(', ')})`); const tail = []; if (rest) tail.push(`${rest} transitive update${rest === 1 ? '' : 's'}`); @@ -109,9 +134,7 @@ function lockBullet({ what, diff, direct }) { function renderEntry({ scaffold, root, direct, rootDirect = null, deps = null, skills = null }) { const bullets = []; if (scaffold && scaffold.moved.length + scaffold.added.length + scaffold.removed.length) { - bullets.push(lockBullet({ - what: 'the bundled minimal workspace `uv.lock`', diff: scaffold, direct, - })); + bullets.push(lockBullet({ lead: LEADS.scaffold, diff: scaffold, direct })); } if (deps && deps.added.length + deps.removed.length + deps.raised.length) { const said = []; @@ -120,21 +143,26 @@ function renderEntry({ scaffold, root, direct, rootDirect = null, deps = null, s said.push(`changed ${deps.raised.map((r) => `\`${r.name}\` to \`${r.to}\``).join(', ')}`); } if (deps.removed.length) said.push(`dropped ${deps.removed.map((n) => `\`${n}\``).join(', ')}`); - bullets.push("- Synced the bundled minimal workspace dependencies with dlt's " - + `\`workspace_deps()\`: ${said.join(', ')}.`); + bullets.push(`- ${LEADS.deps}: ${said.join(', ')}.`); } if (root && root.moved.length + root.added.length + root.removed.length) { - bullets.push(lockBullet({ what: 'the root `uv.lock`', diff: root, direct: rootDirect || direct })); + bullets.push(lockBullet({ lead: LEADS.root, diff: root, direct: rootDirect || direct })); } if (skills && skills.from !== skills.to) { - bullets.push(`- Refreshed the bundled skills to workbench \`${skills.to}\`.`); + bullets.push(`- ${LEADS.skills} \`${skills.to}\`.`); } return bullets; } -// appended to an existing `### Changed` block, so a second bump opens no second heading. -// Every released heading below is what the next release moves, and that is a person's act -function insertEntry(changelog, bullets) { +// Into the existing `### Changed` block under `[Unreleased]`, so a second bump opens no +// second heading, and in place of the bullets an earlier bump left there: both state what is +// unreleased and only the new ones are current. A line a person wrote stays where it is. +// Every released heading below is what the next release moves, and that is a person's act. +// +// `replaces` is which of the four a run computed, so a bullet that came out empty is dropped +// and one the caller was not asked about is left standing. The two look the same in +// `bullets` and mean opposite things +function insertEntry(changelog, bullets, replaces = Object.keys(LEADS)) { if (!bullets.length) return changelog; const lines = String(changelog).split('\n'); const at = lines.findIndex((l) => /^## \[Unreleased\]/.test(l)); @@ -142,17 +170,22 @@ function insertEntry(changelog, bullets) { const nextHeading = lines.findIndex((l, i) => i > at && /^## /.test(l)); const end = nextHeading === -1 ? lines.length : nextHeading; const changed = lines.findIndex((l, i) => i > at && i < end && /^### Changed\s*$/.test(l)); - if (changed !== -1) { - // the block ends at the next `###`, not at the next `##`: `[Unreleased]` carries - // `### Fixed` and `### Removed` beside it and the bullets belong above them - const after = lines.findIndex((l, i) => i > changed && i < end && /^### /.test(l)); - const blockEnd = after === -1 ? end : after; - let last = changed; - for (let i = changed + 1; i < blockEnd; i += 1) if (lines[i].trim()) last = i; - lines.splice(last + 1, 0, ...bullets); + if (changed === -1) { + lines.splice(at + 1, 0, '', '### Changed', ...bullets); return lines.join('\n'); } - lines.splice(at + 1, 0, '', '### Changed', ...bullets); + // the block ends at the next `###`, not at the next `##`: `[Unreleased]` carries + // `### Fixed` and `### Removed` beside it and the bullets belong above them + const after = lines.findIndex((l, i) => i > changed && i < end && /^### /.test(l)); + const blockEnd = after === -1 ? end : after; + const stale = replaces.map((kind) => `- ${LEADS[kind]}`); + const kept = lines.slice(changed + 1, blockEnd) + .filter((l) => !stale.some((lead) => l.startsWith(lead))); + while (kept.length && !kept[0].trim()) kept.shift(); + while (kept.length && !kept[kept.length - 1].trim()) kept.pop(); + // the trailing blank is the one separating this block from the `###` or `##` after it, and + // the file's own last newline where `[Unreleased]` is the only section + lines.splice(changed + 1, blockEnd - changed - 1, ...kept, ...bullets, ''); return lines.join('\n'); } @@ -184,14 +217,15 @@ function main(argv, io = console) { const skillsFrom = value('--skills-before'); const skillsTo = value('--skills-after'); - const bullets = renderEntry({ + const entry = { scaffold: lockDiff(value('--scaffold-before'), value('--scaffold-after')), root: lockDiff(value('--root-before'), value('--root-after')), direct, rootDirect: value('--root-direct') ? directNames(read(value('--root-direct'))) : null, deps, skills: skillsFrom && skillsTo ? { from: skillsFrom, to: skillsTo } : null, - }); + }; + const bullets = renderEntry(entry); if (!bullets.length) { io.error('bump-changelog-entry: nothing moved, so there is no entry to write'); @@ -200,7 +234,16 @@ function main(argv, io = console) { const changelog = value('--changelog'); if (changelog) { - fs.writeFileSync(changelog, insertEntry(fs.readFileSync(changelog, 'utf8'), bullets)); + const was = fs.readFileSync(changelog, 'utf8'); + const now = insertEntry(was, bullets, askedAbout(entry)); + // the bullets an earlier bump wrote, recomputed against the same release and come out + // the same: this run rewrote a lock without moving a version anybody reads + if (now === was) { + io.error('bump-changelog-entry: the entry is already under `## [Unreleased]`, so this' + + ' run has nothing new to say'); + return 1; + } + fs.writeFileSync(changelog, now); io.error(`bump-changelog-entry: wrote ${bullets.length} bullet(s) into ${changelog}`); } io.log(bullets.join('\n')); @@ -210,6 +253,6 @@ function main(argv, io = console) { if (require.main === module) process.exit(main(process.argv.slice(2))); module.exports = { - parseLock, normalise, diffLocks, directNames, directSpecs, lockBullet, renderEntry, - insertEntry, main, + LEADS, isOwnBullet, askedAbout, parseLock, normalise, diffLocks, directNames, directSpecs, + lockBullet, renderEntry, insertEntry, main, }; diff --git a/.github/workflows/bump-dlthub-init.yml b/.github/workflows/bump-dlthub-init.yml index 5aa786d..ac36b25 100644 --- a/.github/workflows/bump-dlthub-init.yml +++ b/.github/workflows/bump-dlthub-init.yml @@ -46,13 +46,16 @@ jobs: id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }} - private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }} + client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }} + private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.repository.default_branch }} token: ${{ steps.app-token.outputs.token }} + # the changelog entry is written against the last release tag, and a shallow clone + # carries no tag to describe + fetch-depth: 0 - name: Is the bundled lock behind a release id: decide @@ -107,17 +110,44 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "${HOME}/.local/bin" >> "$GITHUB_PATH" - # `git stash` would do the same and would also hide a failure - - name: Keep the locks and the scaffold deps as they are now + # The entry is written against the last release, not against the tip this run is about + # to overwrite, so it states the whole unreleased delta and replaces what the bump + # before it wrote. Written against the tip, a second bump between two releases appends + # a second `Refreshed ...` line describing the same packages a version later. + # + # A branch with no release tag behind it, and a tag predating one of these four files, + # both fall back to the tip. All four or none: two baselines in one entry read as + # packages arriving and going. + - name: Keep the state the entry is written against + id: base if: steps.decide.outputs.bump == 'true' run: | set -euo pipefail - mkdir -p /tmp/bump-before - cp uv.lock /tmp/bump-before/root.uv.lock - cp "${SCAFFOLD}/uv.lock" /tmp/bump-before/scaffold.uv.lock - cp "${SCAFFOLD}/pyproject.toml" /tmp/bump-before/scaffold.pyproject.toml - sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' scripts/generate_skills.py \ - > /tmp/bump-before/workbench-ref + mkdir -p /tmp/bump-base /tmp/bump-tag + + # `git stash` would do the same for the tip and would also hide a failure + cp uv.lock /tmp/bump-base/root.uv.lock + cp "${SCAFFOLD}/uv.lock" /tmp/bump-base/scaffold.uv.lock + cp "${SCAFFOLD}/pyproject.toml" /tmp/bump-base/scaffold.pyproject.toml + cp scripts/generate_skills.py /tmp/bump-base/generate_skills.py + against="this branch's tip" + + tag=$(git describe --tags --abbrev=0 --match 'v[0-9]*' 2>/dev/null || true) + if [ -z "$tag" ]; then + echo "::warning::no release tag is reachable from this branch." + elif git show "${tag}:uv.lock" > /tmp/bump-tag/root.uv.lock \ + && git show "${tag}:${SCAFFOLD}/uv.lock" > /tmp/bump-tag/scaffold.uv.lock \ + && git show "${tag}:${SCAFFOLD}/pyproject.toml" > /tmp/bump-tag/scaffold.pyproject.toml \ + && git show "${tag}:scripts/generate_skills.py" > /tmp/bump-tag/generate_skills.py + then + cp -r /tmp/bump-tag/. /tmp/bump-base/ + against="$tag" + else + echo "::warning::${tag} does not carry every file the entry is written from." + fi + + echo "::notice::the changelog entry is written against ${against}." + echo "against=${against}" >> "$GITHUB_OUTPUT" # one step, because a step's own name is not readable from a later one and the failure # report has to name the command that failed @@ -178,24 +208,26 @@ jobs: printf '%s\n' "writing the changelog entry" > /tmp/bump-failed-step set +e node "${GITHUB_WORKSPACE}/.github/workflows/bump-changelog-entry.cjs" \ - --scaffold-before /tmp/bump-before/scaffold.uv.lock \ + --scaffold-before /tmp/bump-base/scaffold.uv.lock \ --scaffold-after "${SCAFFOLD}/uv.lock" \ - --root-before /tmp/bump-before/root.uv.lock \ + --root-before /tmp/bump-base/root.uv.lock \ --root-after uv.lock \ --direct "${SCAFFOLD}/pyproject.toml" \ --root-direct pyproject.toml \ - --deps-before /tmp/bump-before/scaffold.pyproject.toml \ + --deps-before /tmp/bump-base/scaffold.pyproject.toml \ --deps-after "${SCAFFOLD}/pyproject.toml" \ - --skills-before "$(cat /tmp/bump-before/workbench-ref)" \ + --skills-before "$(sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' /tmp/bump-base/generate_skills.py)" \ --skills-after "$(sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' scripts/generate_skills.py)" \ --changelog CHANGELOG.md > /tmp/bump-entry.md code=$? set -e - # exit 1 is the script saying no package version moved, which a lock rewritten for - # its metadata alone can produce. A bump with nothing to say opens nothing + # exit 1 is the script saying this run adds nothing to `## [Unreleased]`: either no + # package version moved at all, or the entry it renders is the one the bump before + # it left there. A lock rewritten for its metadata alone produces both. A bump with + # nothing to say opens nothing if [ "$code" != "0" ]; then - echo "::warning::the recipe rewrote a lock but moved no package version, so there is no entry to write and no pull request to open." + echo "::warning::the recipe rewrote a lock and nothing it moved is new to \`## [Unreleased]\`, so there is no pull request to open." git checkout -- . && git clean -fdq echo "wrote=false" >> "$GITHUB_OUTPUT" exit 0 @@ -245,6 +277,7 @@ jobs: GH_REPO: ${{ github.repository }} BEHIND: ${{ steps.decide.outputs.behind }} BASE: ${{ github.event.repository.default_branch }} + AGAINST: ${{ steps.base.outputs.against }} run: | set -euo pipefail run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" @@ -261,7 +294,7 @@ jobs: echo '- [ ] Both `uv.lock` files resolve the latest versions their `pyproject.toml` constraints allow: `make lock-check` and `make scaffold-lock-check` pass.' echo '- [ ] The bundled workspace dependencies match dlt'"'"'s `workspace_deps()`: `make scaffold-deps-check` passes.' echo '- [ ] `skills/` matches what `scripts/generate_skills.py` produces at its pinned ref: `make check-skills` passes.' - echo '- [ ] `CHANGELOG.md` carries a `### Changed` block under `## [Unreleased]` naming every direct dependency of the bundled workspace whose version moved, with its old and new version.' + echo '- [ ] `CHANGELOG.md` carries a `### Changed` block under `## [Unreleased]` naming every direct dependency of the bundled workspace whose version moved since the last release, once each, with its old and new version.' echo '- [ ] `make ci` passes.' echo echo '## Scope' @@ -290,6 +323,11 @@ jobs: echo cat /tmp/bump-entry.md echo + echo "Everything unreleased, rather than only what this run moved. The entry is written" + echo "against ${AGAINST} and replaces the bullets an earlier bump left under" + echo '`## [Unreleased]`, so a version pair here may have moved in a bump that is already' + echo 'merged.' + echo echo '## How to check it' echo echo "\`make ci\` ran on [run ${GITHUB_RUN_ID}](${run_url}) before this branch was pushed," @@ -320,11 +358,12 @@ jobs: echo "::notice::reviewers requested on #${pr}: ${requested}" fi - # no reviewing pass is dispatched here, and that is #480. This repository is public, - # so it receives the parts of the loop that need no agent and none of the workflow - # files a pass runs from: a private repository's actions resolve for private - # consumers only, so such a run dies before its first step (#461). The reviewer - # requested above is who reads this. Restore the dispatch when a stage can run here. + # no reviewing pass is dispatched here. This repository is public, so it receives + # the parts of the loop that need no agent and none of the workflow files a pass + # runs from: the loop's own repository is private, GitHub shares a private + # repository's actions with private consumers only, and such a run dies before its + # first step. The reviewer requested above is who reads this. Restore the dispatch + # when a stage can run here. # stops the next hourly run too: the decide step reads an open issue as a bump in # flight diff --git a/.github/workflows/release-changelog.cjs b/.github/workflows/release-changelog.cjs index 1d8e25d..2eef521 100644 --- a/.github/workflows/release-changelog.cjs +++ b/.github/workflows/release-changelog.cjs @@ -5,9 +5,10 @@ // version heading, and `--notes` reads that heading's section back out as the body of the // GitHub release. // -// `--cut` is the inverse of `insertEntry` in `bump-changelog-entry.cjs`. That one appends a -// bullet to `[Unreleased]`; this one draws a line under everything there and leaves the -// heading empty for the next bump. +// `--cut` is the inverse of `insertEntry` in `bump-changelog-entry.cjs`. That one keeps +// `[Unreleased]` holding one bullet per thing a bump changes; this one draws a line under +// everything there and leaves the heading empty for the next bump. Emptying it is also what +// makes the next bump's entry start from this release rather than restate it. const fs = require('node:fs'); diff --git a/.github/workflows/release-dlthub-init.yml b/.github/workflows/release-dlthub-init.yml index 4d5be09..aaaf94a 100644 --- a/.github/workflows/release-dlthub-init.yml +++ b/.github/workflows/release-dlthub-init.yml @@ -46,8 +46,8 @@ jobs: id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }} - private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }} + client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }} + private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -223,8 +223,8 @@ jobs: id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }} - private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }} + client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }} + private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1