Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 71 additions & 28 deletions .github/workflows/bump-changelog-entry.cjs
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
'use strict';

// Installed by the engineering agent loop. Edit it in dlt-hub/engineering-agent-loop, not
// here. Writes a dependency bump's changelog entry from the two `uv.lock` files before and
// after the recipe ran, as a draft the reviewing pass then holds to the diff.
// here. Writes a dependency bump's changelog entry from the two `uv.lock` files, as a draft
// the reviewing pass then holds to the diff.
//
// The `--*-before` state the caller passes is the last release, not the tip the recipe just
// overwrote, so the entry is the whole account of what is unreleased. That is what lets a
// second bump before a release replace the bullets the bump before it wrote: both describe
// the same span, one of them a bump out of date. Appending instead grew `[Unreleased]` a
// second `Refreshed ...` line per run.

const fs = require('node:fs');

Expand Down Expand Up @@ -86,13 +92,32 @@ function directSpecs(text) {

const versions = (list) => list.map((p) => `\`${p.name}\` ${p.from} → ${p.to}`);

// How each of the four bullets opens. A second bump finds the first one's lines by these, so
// rewording one here is what rewords it everywhere, and a line a person wrote is left alone
// because it opens with something else
const LEADS = {
scaffold: 'Refreshed the bundled minimal workspace `uv.lock` to the latest'
+ ' within-constraint versions',
root: 'Refreshed the root `uv.lock` to the latest within-constraint versions',
deps: "Synced the bundled minimal workspace dependencies with dlt's `workspace_deps()`",
skills: 'Refreshed the bundled skills to workbench',
};

/** Whether a changelog line is a bullet an earlier run of this script wrote. */
const isOwnBullet = (line) => Object.values(LEADS).some((lead) => line.startsWith(`- ${lead}`));

/** The `LEADS` keys a run was given both sides of, which are the bullets it may replace. */
function askedAbout({ scaffold, root, deps, skills }) {
return Object.keys(LEADS).filter((kind) => ({ scaffold, root, deps, skills })[kind]);
}

// direct dependencies carry both versions and transitives are counted, which is the shape
// of every released entry
function lockBullet({ what, diff, direct }) {
function lockBullet({ lead, diff, direct }) {
const isDirect = (p) => direct.has(p.name);
const named = diff.moved.filter(isDirect);
const rest = diff.moved.length - named.length;
const parts = [`Refreshed ${what} to the latest within-constraint versions`];
const parts = [lead];
if (named.length) parts.push(` (${versions(named).join(', ')})`);
const tail = [];
if (rest) tail.push(`${rest} transitive update${rest === 1 ? '' : 's'}`);
Expand All @@ -109,9 +134,7 @@ function lockBullet({ what, diff, direct }) {
function renderEntry({ scaffold, root, direct, rootDirect = null, deps = null, skills = null }) {
const bullets = [];
if (scaffold && scaffold.moved.length + scaffold.added.length + scaffold.removed.length) {
bullets.push(lockBullet({
what: 'the bundled minimal workspace `uv.lock`', diff: scaffold, direct,
}));
bullets.push(lockBullet({ lead: LEADS.scaffold, diff: scaffold, direct }));
}
if (deps && deps.added.length + deps.removed.length + deps.raised.length) {
const said = [];
Expand All @@ -120,39 +143,49 @@ function renderEntry({ scaffold, root, direct, rootDirect = null, deps = null, s
said.push(`changed ${deps.raised.map((r) => `\`${r.name}\` to \`${r.to}\``).join(', ')}`);
}
if (deps.removed.length) said.push(`dropped ${deps.removed.map((n) => `\`${n}\``).join(', ')}`);
bullets.push("- Synced the bundled minimal workspace dependencies with dlt's "
+ `\`workspace_deps()\`: ${said.join(', ')}.`);
bullets.push(`- ${LEADS.deps}: ${said.join(', ')}.`);
}
if (root && root.moved.length + root.added.length + root.removed.length) {
bullets.push(lockBullet({ what: 'the root `uv.lock`', diff: root, direct: rootDirect || direct }));
bullets.push(lockBullet({ lead: LEADS.root, diff: root, direct: rootDirect || direct }));
}
if (skills && skills.from !== skills.to) {
bullets.push(`- Refreshed the bundled skills to workbench \`${skills.to}\`.`);
bullets.push(`- ${LEADS.skills} \`${skills.to}\`.`);
}
return bullets;
}

// appended to an existing `### Changed` block, so a second bump opens no second heading.
// Every released heading below is what the next release moves, and that is a person's act
function insertEntry(changelog, bullets) {
// Into the existing `### Changed` block under `[Unreleased]`, so a second bump opens no
// second heading, and in place of the bullets an earlier bump left there: both state what is
// unreleased and only the new ones are current. A line a person wrote stays where it is.
// Every released heading below is what the next release moves, and that is a person's act.
//
// `replaces` is which of the four a run computed, so a bullet that came out empty is dropped
// and one the caller was not asked about is left standing. The two look the same in
// `bullets` and mean opposite things
function insertEntry(changelog, bullets, replaces = Object.keys(LEADS)) {
if (!bullets.length) return changelog;
const lines = String(changelog).split('\n');
const at = lines.findIndex((l) => /^## \[Unreleased\]/.test(l));
if (at === -1) throw new Error('the changelog has no `## [Unreleased]` heading');
const nextHeading = lines.findIndex((l, i) => i > at && /^## /.test(l));
const end = nextHeading === -1 ? lines.length : nextHeading;
const changed = lines.findIndex((l, i) => i > at && i < end && /^### Changed\s*$/.test(l));
if (changed !== -1) {
// the block ends at the next `###`, not at the next `##`: `[Unreleased]` carries
// `### Fixed` and `### Removed` beside it and the bullets belong above them
const after = lines.findIndex((l, i) => i > changed && i < end && /^### /.test(l));
const blockEnd = after === -1 ? end : after;
let last = changed;
for (let i = changed + 1; i < blockEnd; i += 1) if (lines[i].trim()) last = i;
lines.splice(last + 1, 0, ...bullets);
if (changed === -1) {
lines.splice(at + 1, 0, '', '### Changed', ...bullets);
return lines.join('\n');
}
lines.splice(at + 1, 0, '', '### Changed', ...bullets);
// the block ends at the next `###`, not at the next `##`: `[Unreleased]` carries
// `### Fixed` and `### Removed` beside it and the bullets belong above them
const after = lines.findIndex((l, i) => i > changed && i < end && /^### /.test(l));
const blockEnd = after === -1 ? end : after;
const stale = replaces.map((kind) => `- ${LEADS[kind]}`);
const kept = lines.slice(changed + 1, blockEnd)
.filter((l) => !stale.some((lead) => l.startsWith(lead)));
while (kept.length && !kept[0].trim()) kept.shift();
while (kept.length && !kept[kept.length - 1].trim()) kept.pop();
// the trailing blank is the one separating this block from the `###` or `##` after it, and
// the file's own last newline where `[Unreleased]` is the only section
lines.splice(changed + 1, blockEnd - changed - 1, ...kept, ...bullets, '');
return lines.join('\n');
}

Expand Down Expand Up @@ -184,14 +217,15 @@ function main(argv, io = console) {
const skillsFrom = value('--skills-before');
const skillsTo = value('--skills-after');

const bullets = renderEntry({
const entry = {
scaffold: lockDiff(value('--scaffold-before'), value('--scaffold-after')),
root: lockDiff(value('--root-before'), value('--root-after')),
direct,
rootDirect: value('--root-direct') ? directNames(read(value('--root-direct'))) : null,
deps,
skills: skillsFrom && skillsTo ? { from: skillsFrom, to: skillsTo } : null,
});
};
const bullets = renderEntry(entry);

if (!bullets.length) {
io.error('bump-changelog-entry: nothing moved, so there is no entry to write');
Expand All @@ -200,7 +234,16 @@ function main(argv, io = console) {

const changelog = value('--changelog');
if (changelog) {
fs.writeFileSync(changelog, insertEntry(fs.readFileSync(changelog, 'utf8'), bullets));
const was = fs.readFileSync(changelog, 'utf8');
const now = insertEntry(was, bullets, askedAbout(entry));
// the bullets an earlier bump wrote, recomputed against the same release and come out
// the same: this run rewrote a lock without moving a version anybody reads
if (now === was) {
io.error('bump-changelog-entry: the entry is already under `## [Unreleased]`, so this'
+ ' run has nothing new to say');
return 1;
}
fs.writeFileSync(changelog, now);
io.error(`bump-changelog-entry: wrote ${bullets.length} bullet(s) into ${changelog}`);
}
io.log(bullets.join('\n'));
Expand All @@ -210,6 +253,6 @@ function main(argv, io = console) {
if (require.main === module) process.exit(main(process.argv.slice(2)));

module.exports = {
parseLock, normalise, diffLocks, directNames, directSpecs, lockBullet, renderEntry,
insertEntry, main,
LEADS, isOwnBullet, askedAbout, parseLock, normalise, diffLocks, directNames, directSpecs,
lockBullet, renderEntry, insertEntry, main,
};
85 changes: 62 additions & 23 deletions .github/workflows/bump-dlthub-init.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,16 @@ jobs:
id: app-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }}
client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }}

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
token: ${{ steps.app-token.outputs.token }}
# the changelog entry is written against the last release tag, and a shallow clone
# carries no tag to describe
fetch-depth: 0

- name: Is the bundled lock behind a release
id: decide
Expand Down Expand Up @@ -107,17 +110,44 @@ jobs:
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "${HOME}/.local/bin" >> "$GITHUB_PATH"

# `git stash` would do the same and would also hide a failure
- name: Keep the locks and the scaffold deps as they are now
# The entry is written against the last release, not against the tip this run is about
# to overwrite, so it states the whole unreleased delta and replaces what the bump
# before it wrote. Written against the tip, a second bump between two releases appends
# a second `Refreshed ...` line describing the same packages a version later.
#
# A branch with no release tag behind it, and a tag predating one of these four files,
# both fall back to the tip. All four or none: two baselines in one entry read as
# packages arriving and going.
- name: Keep the state the entry is written against
id: base
if: steps.decide.outputs.bump == 'true'
run: |
set -euo pipefail
mkdir -p /tmp/bump-before
cp uv.lock /tmp/bump-before/root.uv.lock
cp "${SCAFFOLD}/uv.lock" /tmp/bump-before/scaffold.uv.lock
cp "${SCAFFOLD}/pyproject.toml" /tmp/bump-before/scaffold.pyproject.toml
sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' scripts/generate_skills.py \
> /tmp/bump-before/workbench-ref
mkdir -p /tmp/bump-base /tmp/bump-tag

# `git stash` would do the same for the tip and would also hide a failure
cp uv.lock /tmp/bump-base/root.uv.lock
cp "${SCAFFOLD}/uv.lock" /tmp/bump-base/scaffold.uv.lock
cp "${SCAFFOLD}/pyproject.toml" /tmp/bump-base/scaffold.pyproject.toml
cp scripts/generate_skills.py /tmp/bump-base/generate_skills.py
against="this branch's tip"

tag=$(git describe --tags --abbrev=0 --match 'v[0-9]*' 2>/dev/null || true)
if [ -z "$tag" ]; then
echo "::warning::no release tag is reachable from this branch."
elif git show "${tag}:uv.lock" > /tmp/bump-tag/root.uv.lock \
&& git show "${tag}:${SCAFFOLD}/uv.lock" > /tmp/bump-tag/scaffold.uv.lock \
&& git show "${tag}:${SCAFFOLD}/pyproject.toml" > /tmp/bump-tag/scaffold.pyproject.toml \
&& git show "${tag}:scripts/generate_skills.py" > /tmp/bump-tag/generate_skills.py
then
cp -r /tmp/bump-tag/. /tmp/bump-base/
against="$tag"
else
echo "::warning::${tag} does not carry every file the entry is written from."
fi

echo "::notice::the changelog entry is written against ${against}."
echo "against=${against}" >> "$GITHUB_OUTPUT"

# one step, because a step's own name is not readable from a later one and the failure
# report has to name the command that failed
Expand Down Expand Up @@ -178,24 +208,26 @@ jobs:
printf '%s\n' "writing the changelog entry" > /tmp/bump-failed-step
set +e
node "${GITHUB_WORKSPACE}/.github/workflows/bump-changelog-entry.cjs" \
--scaffold-before /tmp/bump-before/scaffold.uv.lock \
--scaffold-before /tmp/bump-base/scaffold.uv.lock \
--scaffold-after "${SCAFFOLD}/uv.lock" \
--root-before /tmp/bump-before/root.uv.lock \
--root-before /tmp/bump-base/root.uv.lock \
--root-after uv.lock \
--direct "${SCAFFOLD}/pyproject.toml" \
--root-direct pyproject.toml \
--deps-before /tmp/bump-before/scaffold.pyproject.toml \
--deps-before /tmp/bump-base/scaffold.pyproject.toml \
--deps-after "${SCAFFOLD}/pyproject.toml" \
--skills-before "$(cat /tmp/bump-before/workbench-ref)" \
--skills-before "$(sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' /tmp/bump-base/generate_skills.py)" \
--skills-after "$(sed -n 's/^WORKBENCH_REF_SHORT = "\(.*\)"$/\1/p' scripts/generate_skills.py)" \
--changelog CHANGELOG.md > /tmp/bump-entry.md
code=$?
set -e

# exit 1 is the script saying no package version moved, which a lock rewritten for
# its metadata alone can produce. A bump with nothing to say opens nothing
# exit 1 is the script saying this run adds nothing to `## [Unreleased]`: either no
# package version moved at all, or the entry it renders is the one the bump before
# it left there. A lock rewritten for its metadata alone produces both. A bump with
# nothing to say opens nothing
if [ "$code" != "0" ]; then
echo "::warning::the recipe rewrote a lock but moved no package version, so there is no entry to write and no pull request to open."
echo "::warning::the recipe rewrote a lock and nothing it moved is new to \`## [Unreleased]\`, so there is no pull request to open."
git checkout -- . && git clean -fdq
echo "wrote=false" >> "$GITHUB_OUTPUT"
exit 0
Expand Down Expand Up @@ -245,6 +277,7 @@ jobs:
GH_REPO: ${{ github.repository }}
BEHIND: ${{ steps.decide.outputs.behind }}
BASE: ${{ github.event.repository.default_branch }}
AGAINST: ${{ steps.base.outputs.against }}
run: |
set -euo pipefail
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
Expand All @@ -261,7 +294,7 @@ jobs:
echo '- [ ] Both `uv.lock` files resolve the latest versions their `pyproject.toml` constraints allow: `make lock-check` and `make scaffold-lock-check` pass.'
echo '- [ ] The bundled workspace dependencies match dlt'"'"'s `workspace_deps()`: `make scaffold-deps-check` passes.'
echo '- [ ] `skills/` matches what `scripts/generate_skills.py` produces at its pinned ref: `make check-skills` passes.'
echo '- [ ] `CHANGELOG.md` carries a `### Changed` block under `## [Unreleased]` naming every direct dependency of the bundled workspace whose version moved, with its old and new version.'
echo '- [ ] `CHANGELOG.md` carries a `### Changed` block under `## [Unreleased]` naming every direct dependency of the bundled workspace whose version moved since the last release, once each, with its old and new version.'
echo '- [ ] `make ci` passes.'
echo
echo '## Scope'
Expand Down Expand Up @@ -290,6 +323,11 @@ jobs:
echo
cat /tmp/bump-entry.md
echo
echo "Everything unreleased, rather than only what this run moved. The entry is written"
echo "against ${AGAINST} and replaces the bullets an earlier bump left under"
echo '`## [Unreleased]`, so a version pair here may have moved in a bump that is already'
echo 'merged.'
echo
echo '## How to check it'
echo
echo "\`make ci\` ran on [run ${GITHUB_RUN_ID}](${run_url}) before this branch was pushed,"
Expand Down Expand Up @@ -320,11 +358,12 @@ jobs:
echo "::notice::reviewers requested on #${pr}: ${requested}"
fi

# no reviewing pass is dispatched here, and that is #480. This repository is public,
# so it receives the parts of the loop that need no agent and none of the workflow
# files a pass runs from: a private repository's actions resolve for private
# consumers only, so such a run dies before its first step (#461). The reviewer
# requested above is who reads this. Restore the dispatch when a stage can run here.
# no reviewing pass is dispatched here. This repository is public, so it receives
# the parts of the loop that need no agent and none of the workflow files a pass
# runs from: the loop's own repository is private, GitHub shares a private
# repository's actions with private consumers only, and such a run dies before its
# first step. The reviewer requested above is who reads this. Restore the dispatch
# when a stage can run here.

# stops the next hourly run too: the decide step reads an open issue as a bump in
# flight
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/release-changelog.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
// version heading, and `--notes` reads that heading's section back out as the body of the
// GitHub release.
//
// `--cut` is the inverse of `insertEntry` in `bump-changelog-entry.cjs`. That one appends a
// bullet to `[Unreleased]`; this one draws a line under everything there and leaves the
// heading empty for the next bump.
// `--cut` is the inverse of `insertEntry` in `bump-changelog-entry.cjs`. That one keeps
// `[Unreleased]` holding one bullet per thing a bump changes; this one draws a line under
// everything there and leaves the heading empty for the next bump. Emptying it is also what
// makes the next bump's entry start from this release rather than restate it.

const fs = require('node:fs');

Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/release-dlthub-init.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ jobs:
id: app-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }}
client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }}

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand Down Expand Up @@ -223,8 +223,8 @@ jobs:
id: app-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
client-id: ${{ vars.AGENT_LOOP_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_APP_PRIVATE_KEY }}
client-id: ${{ vars.AGENT_LOOP_PUBLIC_APP_CLIENT_ID }}
private-key: ${{ secrets.AGENT_LOOP_PUBLIC_APP_PRIVATE_KEY }}

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

Expand Down
Loading