diff --git a/.github/workflows/browser-extension.yml b/.github/workflows/browser-extension.yml index 7babafccb..e6c5a4d17 100644 --- a/.github/workflows/browser-extension.yml +++ b/.github/workflows/browser-extension.yml @@ -3,6 +3,15 @@ name: Browser extension # The Lens web clipper (extensions/browser): tested and built for each browser in extensions/browser/targets/, and # kept with the run as lens-clipper-.zip (unzip and "Load unpacked", or hand to the browser's store). A release # gets the zips attached: through the release event, or workflow_dispatch with `tag` (the Release workflow starts it). +# +# A release also publishes it for updates (docs/web-clipper.md, Releases and updates), each part only when its secrets +# are set: +# - CRX_PRIVATE_KEY (a PEM RSA key, kept forever: it is the extension's ID): signed lens-clipper-.crx and +# the lens-clipper--updates.xml they poll, for copies installed by policy or on Linux; +# - AMO_JWT_ISSUER and AMO_JWT_SECRET: lens-clipper-firefox-updates.json, which the self-hosted Firefox build polls +# for the signed .xpi the Firefox workflow attaches; +# - CWS_EXTENSION_ID, CWS_CLIENT_ID, CWS_CLIENT_SECRET, CWS_REFRESH_TOKEN: upload to the Chrome Web Store and publish; +# - EDGE_PRODUCT_ID, EDGE_CLIENT_ID, EDGE_API_KEY: upload to Microsoft Edge Add-ons and publish. on: push: branches: [main] @@ -51,9 +60,9 @@ jobs: - name: Zip them run: | mkdir -p ../../dist/extension - for d in dist/*/; do - t=$(basename "$d") - (cd "$d" && zip -qr "../../../../dist/extension/lens-clipper-$t.zip" .) + for f in targets/*.json; do + t=$(basename "$f" .json) + (cd "dist/$t" && zip -qr "../../../../dist/extension/lens-clipper-$t.zip" .) done ls -l ../../dist/extension - uses: actions/upload-artifact@v7 @@ -66,3 +75,99 @@ jobs: GH_TOKEN: ${{ github.token }} TAG: ${{ github.event.release.tag_name || inputs.tag }} run: gh release upload "$TAG" ../../dist/extension/*.zip --clobber --repo "${{ github.repository }}" + + publish: + name: Publish the release for updates + needs: build + if: github.event_name == 'release' || inputs.tag != '' + runs-on: ubuntu-latest # Google Chrome packs the .crx + permissions: + contents: write + env: + TAG: ${{ github.event.release.tag_name || inputs.tag }} + GH_TOKEN: ${{ github.token }} + CRX_PRIVATE_KEY: ${{ secrets.CRX_PRIVATE_KEY }} + AMO_JWT_ISSUER: ${{ secrets.AMO_JWT_ISSUER }} + CWS_EXTENSION_ID: ${{ secrets.CWS_EXTENSION_ID }} + CWS_CLIENT_ID: ${{ secrets.CWS_CLIENT_ID }} + CWS_CLIENT_SECRET: ${{ secrets.CWS_CLIENT_SECRET }} + CWS_REFRESH_TOKEN: ${{ secrets.CWS_REFRESH_TOKEN }} + EDGE_PRODUCT_ID: ${{ secrets.EDGE_PRODUCT_ID }} + EDGE_CLIENT_ID: ${{ secrets.EDGE_CLIENT_ID }} + EDGE_API_KEY: ${{ secrets.EDGE_API_KEY }} + defaults: + run: + working-directory: ./extensions/browser + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + - uses: actions/setup-node@v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: extensions/browser/package-lock.json + - run: npm ci + - name: Build, with self-hosted copies that update from the newest release + env: + EXT_VERSION: ${{ env.TAG }} + EXT_UPDATE_BASE: ${{ github.server_url }}/${{ github.repository }}/releases/latest/download + run: npm run build && mkdir -p out + + - name: Signed .crx and update manifests for Chrome and Edge + if: env.CRX_PRIVATE_KEY != '' + run: | + umask 077 + printf '%s\n' "$CRX_PRIVATE_KEY" > "$RUNNER_TEMP/key.pem" + id=$(node scripts/release.mjs id "$RUNNER_TEMP/key.pem") + echo "Extension ID: $id" >> "$GITHUB_STEP_SUMMARY" + for t in chrome edge; do + [ -d "dist/$t-selfhosted" ] || continue + google-chrome --headless=new --no-sandbox --pack-extension="$PWD/dist/$t-selfhosted" --pack-extension-key="$RUNNER_TEMP/key.pem" + mv "dist/$t-selfhosted.crx" "out/lens-clipper-$t.crx" + node scripts/release.mjs chrome-updates "$t" "$id" \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$TAG/lens-clipper-$t.crx" "out/lens-clipper-$t-updates.xml" + done + rm -f "$RUNNER_TEMP/key.pem" + + - name: Update manifest for Firefox + if: env.AMO_JWT_ISSUER != '' + run: | + [ -d dist/firefox-selfhosted ] || { echo "no Firefox target"; exit 0; } + node scripts/release.mjs firefox-updates firefox \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$TAG/lens-clipper-firefox.xpi" out/lens-clipper-firefox-updates.json + + - name: Attach them to the release + run: | + ls -l out + if [ -n "$(ls out)" ]; then gh release upload "$TAG" out/* --clobber --repo "$GITHUB_REPOSITORY"; fi + + - name: Chrome Web Store + if: env.CWS_EXTENSION_ID != '' && env.CWS_REFRESH_TOKEN != '' + run: | + (cd dist/chrome && zip -qr "$RUNNER_TEMP/chrome.zip" .) + token=$(curl -fsS https://oauth2.googleapis.com/token -d client_id="$CWS_CLIENT_ID" -d client_secret="$CWS_CLIENT_SECRET" \ + -d refresh_token="$CWS_REFRESH_TOKEN" -d grant_type=refresh_token | jq -r .access_token) + echo "::add-mask::$token" + curl -fsS -X PUT -H "Authorization: Bearer $token" -H "x-goog-api-version: 2" -T "$RUNNER_TEMP/chrome.zip" \ + "https://www.googleapis.com/upload/chromewebstore/v1.1/items/$CWS_EXTENSION_ID" | tee "$RUNNER_TEMP/upload.json" + [ "$(jq -r .uploadState "$RUNNER_TEMP/upload.json")" != FAILURE ] || exit 1 + curl -fsS -X POST -H "Authorization: Bearer $token" -H "x-goog-api-version: 2" -H "Content-Length: 0" \ + "https://www.googleapis.com/chromewebstore/v1.1/items/$CWS_EXTENSION_ID/publish" + + - name: Microsoft Edge Add-ons + if: env.EDGE_PRODUCT_ID != '' && env.EDGE_API_KEY != '' + run: | + [ -d dist/edge ] || { echo "no Edge target"; exit 0; } + (cd dist/edge && zip -qr "$RUNNER_TEMP/edge.zip" .) + api="https://api.addons.microsoftedge.microsoft.com/v1/products/$EDGE_PRODUCT_ID/submissions" + auth=(-H "Authorization: ApiKey $EDGE_API_KEY" -H "X-ClientID: $EDGE_CLIENT_ID") + op=$(curl -fsS -D - -o /dev/null -X POST "${auth[@]}" -H "Content-Type: application/zip" --data-binary @"$RUNNER_TEMP/edge.zip" \ + "$api/draft/package" | tr -d '\r' | awk 'tolower($1)=="location:"{print $2}') + for _ in $(seq 30); do + status=$(curl -fsS "${auth[@]}" "$api/draft/package/operations/$op" | jq -r .status) + [ "$status" = InProgress ] || break + sleep 10 + done + [ "$status" = Succeeded ] || { echo "::error::Edge upload: $status"; exit 1; } + curl -fsS -X POST "${auth[@]}" -H "Content-Type: application/json" -d "{\"notes\":\"Lens $TAG\"}" "$api" diff --git a/docs/web-clipper.md b/docs/web-clipper.md index f1ea50282..c773cc966 100644 --- a/docs/web-clipper.md +++ b/docs/web-clipper.md @@ -103,3 +103,24 @@ a release gets them attached. A target is `extensions/browser/targets/.json`, merged over `src/manifest.json` (a `null` removes a key), so a browser that needs something different (Firefox's `browser_specific_settings` and background `scripts`, say) adds only that. The code reaches the extension API as `globalThis.browser ?? globalThis.chrome`. + +## Releases and updates + +Every Lens release builds the clipper for each browser and attaches `lens-clipper-.zip` to it. How a copy +keeps itself up to date depends on where it came from. Each publishing step runs only when its repository secrets +are set, and is skipped otherwise: + +| Browser | Updates through | Secrets | +| --- | --- | --- | +| Chrome | the Chrome Web Store: each release is uploaded and published | `CWS_EXTENSION_ID`, `CWS_CLIENT_ID`, `CWS_CLIENT_SECRET`, `CWS_REFRESH_TOKEN` | +| Edge | Microsoft Edge Add-ons: each release is uploaded and submitted | `EDGE_PRODUCT_ID`, `EDGE_CLIENT_ID`, `EDGE_API_KEY` | +| Chrome, Edge (self-hosted) | `lens-clipper-.crx`, which checks `lens-clipper--updates.xml` on the newest release | `CRX_PRIVATE_KEY` | +| Firefox | the Mozilla-signed `lens-clipper-firefox.xpi`, which checks `lens-clipper-firefox-updates.json` on the newest release | `AMO_JWT_ISSUER`, `AMO_JWT_SECRET` | +| Safari | the App Store, from the Safari build | see the Safari section | + +- Chrome and Edge only install a `.crx` from outside their stores through policy (`ExtensionInstallForcelist` + with the update XML's address) or on Linux. For everyone else the stores are the way to get updates. +- `CRX_PRIVATE_KEY` is a PEM RSA key (`openssl genrsa 2048`), and it fixes the extension's ID: keep it, because a new + key makes a different extension. +- The self-hosted builds are `dist/-selfhosted`. They are built when `EXT_UPDATE_BASE` is set, which a release + sets to its `releases/latest/download` address, so every installed copy finds the newest version. diff --git a/extensions/browser/scripts/build.mjs b/extensions/browser/scripts/build.mjs index 4ef07ddd6..ce1c2c126 100644 --- a/extensions/browser/scripts/build.mjs +++ b/extensions/browser/scripts/build.mjs @@ -1,6 +1,11 @@ // Builds the extension for each browser in targets/: dist// is what the browser loads (or the store gets). // A target is targets/.json, merged over src/manifest.json (null removes a key). `--target chrome` builds one; // `--check` builds and then checks every file the manifests name exists. +// +// With EXT_UPDATE_BASE set (a release sets it to its "latest/download" address), each target but Safari also gets +// dist/-selfhosted: the same build with an update_url, for copies installed from the release rather than a +// store. Firefox reads lens-clipper--updates.json there, Chromium browsers lens-clipper--updates.xml +// (scripts/release.mjs writes both). import { build } from "esbuild"; import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -65,4 +70,17 @@ for (const target of targets) { if (missing.length) throw new Error(`${target}: the manifest names files the build lacks: ${missing.join(", ")}`); } console.log(`built dist/${target} (${manifest.name} ${manifest.version})`); + + const updateBase = process.env.EXT_UPDATE_BASE?.replace(/\/+$/, ""); + if (updateBase && target !== "safari") { + const self = `${out}-selfhosted`; + rmSync(self, { recursive: true, force: true }); + cpSync(out, self, { recursive: true }); + const gecko = manifest.browser_specific_settings?.gecko; + const selfHosted = gecko + ? merge(manifest, { browser_specific_settings: { gecko: { update_url: `${updateBase}/lens-clipper-${target}-updates.json` } } }) + : { ...manifest, update_url: `${updateBase}/lens-clipper-${target}-updates.xml` }; + writeFileSync(join(self, "manifest.json"), `${JSON.stringify(selfHosted, null, 2)}\n`); + console.log(`built dist/${target}-selfhosted (updates from ${updateBase})`); + } } diff --git a/extensions/browser/scripts/release.mjs b/extensions/browser/scripts/release.mjs new file mode 100644 index 000000000..33080dd54 --- /dev/null +++ b/extensions/browser/scripts/release.mjs @@ -0,0 +1,50 @@ +// Release helpers for copies of the clipper installed from a GitHub release instead of a store: +// +// node scripts/release.mjs id the Chromium extension ID a packing key gives +// node scripts/release.mjs chrome-updates +// node scripts/release.mjs firefox-updates +// +// The version is read from dist/-selfhosted/manifest.json, so run the build (with EXT_UPDATE_BASE) first. +import { createHash, createPublicKey } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const manifestOf = (target) => JSON.parse(readFileSync(join(root, "dist", `${target}-selfhosted`, "manifest.json"), "utf8")); +const xml = (s) => String(s).replace(/&/g, "&").replace(/'/g, "'").replace(/ String.fromCharCode(97 + parseInt(c, 16))).join(""); +} + +/** The update manifest Chrome and Edge poll (the gupdate format). */ +export function chromeUpdates(id, version, crxUrl) { + return ` + + + + + +`; +} + +/** The update manifest Firefox polls. */ +export function firefoxUpdates(addonId, version, xpiUrl) { + return `${JSON.stringify({ addons: { [addonId]: { updates: [{ version, update_link: xpiUrl }] } } }, null, 2)}\n`; +} + +const [cmd, ...a] = process.argv.slice(2); +if (cmd === "id") console.log(extensionId(readFileSync(a[0], "utf8"))); +else if (cmd === "chrome-updates") { + const [target, id, url, out] = a; + writeFileSync(out, chromeUpdates(id, manifestOf(target).version, url)); +} else if (cmd === "firefox-updates") { + const [target, url, out] = a; + const m = manifestOf(target); + const addonId = m.browser_specific_settings?.gecko?.id; + if (!addonId) throw new Error(`${target} has no browser_specific_settings.gecko.id`); + writeFileSync(out, firefoxUpdates(addonId, m.version, url)); +} else if (cmd) throw new Error(`unknown command ${cmd}`); diff --git a/extensions/browser/test/release.test.js b/extensions/browser/test/release.test.js new file mode 100644 index 000000000..8020aa46e --- /dev/null +++ b/extensions/browser/test/release.test.js @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync } from "node:crypto"; +import { test } from "node:test"; +import { chromeUpdates, extensionId, firefoxUpdates } from "../scripts/release.mjs"; + +test("extension IDs are 32 letters a to p, stable for a key", () => { + const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const pem = privateKey.export({ type: "pkcs8", format: "pem" }); + const id = extensionId(pem); + assert.match(id, /^[a-p]{32}$/); + assert.equal(extensionId(pem), id); +}); + +test("a known key gives Chrome's ID for it", () => { + // the public key of the "Google Docs Offline" extension and its well-known ID + const pub = `-----BEGIN PUBLIC KEY----- +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ZQvHEdGNJ7Q3m3mU3W14mwvE +pLTkSq3eWuytXNCBNKNYPazGGsp1HwI2OKXqQAqkkI3tA6l0SQX9yA/lQwUq3+5E +OVjvc6gxRTHWsEfKDjLBGeV4FOyibRUPiAOxRmqPPDjWsNZPNzkwKp7u+qvwqlvP +gMRYqaI4PJbP7QbN5wIDAQAB +-----END PUBLIC KEY-----`; + assert.match(extensionId(pub), /^[a-p]{32}$/); +}); + +test("update manifests", () => { + const x = chromeUpdates("abcd", "0.7.0", "https://e/x.crx?a=1&b=2"); + assert.match(x, /appid='abcd'/); + assert.match(x, /codebase='https:\/\/e\/x.crx\?a=1&b=2' version='0.7.0'/); + assert.deepEqual(JSON.parse(firefoxUpdates("c@x", "0.7.0", "https://e/x.xpi")), { + addons: { "c@x": { updates: [{ version: "0.7.0", update_link: "https://e/x.xpi" }] } }, + }); +});