Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 108 additions & 3 deletions .github/workflows/browser-extension.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@ name: Browser extension
# The Lens web clipper (extensions/browser): tested and built for each browser in extensions/browser/targets/, and
# kept with the run as lens-clipper-<target>.zip (unzip and "Load unpacked", or hand to the browser's store). A release
# gets the zips attached: through the release event, or workflow_dispatch with `tag` (the Release workflow starts it).
#
# A release also publishes it for updates (docs/web-clipper.md, Releases and updates), each part only when its secrets
# are set:
# - CRX_PRIVATE_KEY (a PEM RSA key, kept forever: it is the extension's ID): signed lens-clipper-<chrome|edge>.crx and
# the lens-clipper-<target>-updates.xml they poll, for copies installed by policy or on Linux;
# - AMO_JWT_ISSUER and AMO_JWT_SECRET: lens-clipper-firefox-updates.json, which the self-hosted Firefox build polls
# for the signed .xpi the Firefox workflow attaches;
# - CWS_EXTENSION_ID, CWS_CLIENT_ID, CWS_CLIENT_SECRET, CWS_REFRESH_TOKEN: upload to the Chrome Web Store and publish;
# - EDGE_PRODUCT_ID, EDGE_CLIENT_ID, EDGE_API_KEY: upload to Microsoft Edge Add-ons and publish.
on:
push:
branches: [main]
Expand Down Expand Up @@ -51,9 +60,9 @@ jobs:
- name: Zip them
run: |
mkdir -p ../../dist/extension
for d in dist/*/; do
t=$(basename "$d")
(cd "$d" && zip -qr "../../../../dist/extension/lens-clipper-$t.zip" .)
for f in targets/*.json; do
t=$(basename "$f" .json)
(cd "dist/$t" && zip -qr "../../../../dist/extension/lens-clipper-$t.zip" .)
done
ls -l ../../dist/extension
- uses: actions/upload-artifact@v7
Expand All @@ -66,3 +75,99 @@ jobs:
GH_TOKEN: ${{ github.token }}
TAG: ${{ github.event.release.tag_name || inputs.tag }}
run: gh release upload "$TAG" ../../dist/extension/*.zip --clobber --repo "${{ github.repository }}"

publish:
name: Publish the release for updates
needs: build
if: github.event_name == 'release' || inputs.tag != ''
runs-on: ubuntu-latest # Google Chrome packs the .crx
permissions:
contents: write
env:
TAG: ${{ github.event.release.tag_name || inputs.tag }}
GH_TOKEN: ${{ github.token }}
CRX_PRIVATE_KEY: ${{ secrets.CRX_PRIVATE_KEY }}
AMO_JWT_ISSUER: ${{ secrets.AMO_JWT_ISSUER }}
CWS_EXTENSION_ID: ${{ secrets.CWS_EXTENSION_ID }}
CWS_CLIENT_ID: ${{ secrets.CWS_CLIENT_ID }}
CWS_CLIENT_SECRET: ${{ secrets.CWS_CLIENT_SECRET }}
CWS_REFRESH_TOKEN: ${{ secrets.CWS_REFRESH_TOKEN }}
EDGE_PRODUCT_ID: ${{ secrets.EDGE_PRODUCT_ID }}
EDGE_CLIENT_ID: ${{ secrets.EDGE_CLIENT_ID }}
EDGE_API_KEY: ${{ secrets.EDGE_API_KEY }}
defaults:
run:
working-directory: ./extensions/browser
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.tag || github.ref }}
- uses: actions/setup-node@v7
with:
node-version: "22"
cache: npm
cache-dependency-path: extensions/browser/package-lock.json
- run: npm ci
- name: Build, with self-hosted copies that update from the newest release
env:
EXT_VERSION: ${{ env.TAG }}
EXT_UPDATE_BASE: ${{ github.server_url }}/${{ github.repository }}/releases/latest/download
run: npm run build && mkdir -p out

- name: Signed .crx and update manifests for Chrome and Edge
if: env.CRX_PRIVATE_KEY != ''
run: |
umask 077
printf '%s\n' "$CRX_PRIVATE_KEY" > "$RUNNER_TEMP/key.pem"
id=$(node scripts/release.mjs id "$RUNNER_TEMP/key.pem")
echo "Extension ID: $id" >> "$GITHUB_STEP_SUMMARY"
for t in chrome edge; do
[ -d "dist/$t-selfhosted" ] || continue
google-chrome --headless=new --no-sandbox --pack-extension="$PWD/dist/$t-selfhosted" --pack-extension-key="$RUNNER_TEMP/key.pem"
mv "dist/$t-selfhosted.crx" "out/lens-clipper-$t.crx"
node scripts/release.mjs chrome-updates "$t" "$id" \
"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$TAG/lens-clipper-$t.crx" "out/lens-clipper-$t-updates.xml"
done
rm -f "$RUNNER_TEMP/key.pem"

- name: Update manifest for Firefox
if: env.AMO_JWT_ISSUER != ''
run: |
[ -d dist/firefox-selfhosted ] || { echo "no Firefox target"; exit 0; }
node scripts/release.mjs firefox-updates firefox \
"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$TAG/lens-clipper-firefox.xpi" out/lens-clipper-firefox-updates.json

- name: Attach them to the release
run: |
ls -l out
if [ -n "$(ls out)" ]; then gh release upload "$TAG" out/* --clobber --repo "$GITHUB_REPOSITORY"; fi

- name: Chrome Web Store
if: env.CWS_EXTENSION_ID != '' && env.CWS_REFRESH_TOKEN != ''
run: |
(cd dist/chrome && zip -qr "$RUNNER_TEMP/chrome.zip" .)
token=$(curl -fsS https://oauth2.googleapis.com/token -d client_id="$CWS_CLIENT_ID" -d client_secret="$CWS_CLIENT_SECRET" \
-d refresh_token="$CWS_REFRESH_TOKEN" -d grant_type=refresh_token | jq -r .access_token)
echo "::add-mask::$token"
curl -fsS -X PUT -H "Authorization: Bearer $token" -H "x-goog-api-version: 2" -T "$RUNNER_TEMP/chrome.zip" \
"https://www.googleapis.com/upload/chromewebstore/v1.1/items/$CWS_EXTENSION_ID" | tee "$RUNNER_TEMP/upload.json"
[ "$(jq -r .uploadState "$RUNNER_TEMP/upload.json")" != FAILURE ] || exit 1
curl -fsS -X POST -H "Authorization: Bearer $token" -H "x-goog-api-version: 2" -H "Content-Length: 0" \
"https://www.googleapis.com/chromewebstore/v1.1/items/$CWS_EXTENSION_ID/publish"

- name: Microsoft Edge Add-ons
if: env.EDGE_PRODUCT_ID != '' && env.EDGE_API_KEY != ''
run: |
[ -d dist/edge ] || { echo "no Edge target"; exit 0; }
(cd dist/edge && zip -qr "$RUNNER_TEMP/edge.zip" .)
api="https://api.addons.microsoftedge.microsoft.com/v1/products/$EDGE_PRODUCT_ID/submissions"
auth=(-H "Authorization: ApiKey $EDGE_API_KEY" -H "X-ClientID: $EDGE_CLIENT_ID")
op=$(curl -fsS -D - -o /dev/null -X POST "${auth[@]}" -H "Content-Type: application/zip" --data-binary @"$RUNNER_TEMP/edge.zip" \
"$api/draft/package" | tr -d '\r' | awk 'tolower($1)=="location:"{print $2}')
for _ in $(seq 30); do
status=$(curl -fsS "${auth[@]}" "$api/draft/package/operations/$op" | jq -r .status)
[ "$status" = InProgress ] || break
sleep 10
done
[ "$status" = Succeeded ] || { echo "::error::Edge upload: $status"; exit 1; }
curl -fsS -X POST "${auth[@]}" -H "Content-Type: application/json" -d "{\"notes\":\"Lens $TAG\"}" "$api"
21 changes: 21 additions & 0 deletions docs/web-clipper.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,3 +103,24 @@ a release gets them attached.
A target is `extensions/browser/targets/<name>.json`, merged over `src/manifest.json` (a `null` removes a key), so a
browser that needs something different (Firefox's `browser_specific_settings` and background `scripts`, say) adds only
that. The code reaches the extension API as `globalThis.browser ?? globalThis.chrome`.

## Releases and updates

Every Lens release builds the clipper for each browser and attaches `lens-clipper-<browser>.zip` to it. How a copy
keeps itself up to date depends on where it came from. Each publishing step runs only when its repository secrets
are set, and is skipped otherwise:

| Browser | Updates through | Secrets |
| --- | --- | --- |
| Chrome | the Chrome Web Store: each release is uploaded and published | `CWS_EXTENSION_ID`, `CWS_CLIENT_ID`, `CWS_CLIENT_SECRET`, `CWS_REFRESH_TOKEN` |
| Edge | Microsoft Edge Add-ons: each release is uploaded and submitted | `EDGE_PRODUCT_ID`, `EDGE_CLIENT_ID`, `EDGE_API_KEY` |
| Chrome, Edge (self-hosted) | `lens-clipper-<browser>.crx`, which checks `lens-clipper-<browser>-updates.xml` on the newest release | `CRX_PRIVATE_KEY` |
| Firefox | the Mozilla-signed `lens-clipper-firefox.xpi`, which checks `lens-clipper-firefox-updates.json` on the newest release | `AMO_JWT_ISSUER`, `AMO_JWT_SECRET` |
| Safari | the App Store, from the Safari build | see the Safari section |

- Chrome and Edge only install a `.crx` from outside their stores through policy (`ExtensionInstallForcelist`
with the update XML's address) or on Linux. For everyone else the stores are the way to get updates.
- `CRX_PRIVATE_KEY` is a PEM RSA key (`openssl genrsa 2048`), and it fixes the extension's ID: keep it, because a new
key makes a different extension.
- The self-hosted builds are `dist/<browser>-selfhosted`. They are built when `EXT_UPDATE_BASE` is set, which a release
sets to its `releases/latest/download` address, so every installed copy finds the newest version.
18 changes: 18 additions & 0 deletions extensions/browser/scripts/build.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
// Builds the extension for each browser in targets/: dist/<target>/ is what the browser loads (or the store gets).
// A target is targets/<name>.json, merged over src/manifest.json (null removes a key). `--target chrome` builds one;
// `--check` builds and then checks every file the manifests name exists.
//
// With EXT_UPDATE_BASE set (a release sets it to its "latest/download" address), each target but Safari also gets
// dist/<target>-selfhosted: the same build with an update_url, for copies installed from the release rather than a
// store. Firefox reads lens-clipper-<target>-updates.json there, Chromium browsers lens-clipper-<target>-updates.xml
// (scripts/release.mjs writes both).
import { build } from "esbuild";
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
Expand Down Expand Up @@ -65,4 +70,17 @@ for (const target of targets) {
if (missing.length) throw new Error(`${target}: the manifest names files the build lacks: ${missing.join(", ")}`);
}
console.log(`built dist/${target} (${manifest.name} ${manifest.version})`);

const updateBase = process.env.EXT_UPDATE_BASE?.replace(/\/+$/, "");
if (updateBase && target !== "safari") {
const self = `${out}-selfhosted`;
rmSync(self, { recursive: true, force: true });
cpSync(out, self, { recursive: true });
const gecko = manifest.browser_specific_settings?.gecko;
const selfHosted = gecko
? merge(manifest, { browser_specific_settings: { gecko: { update_url: `${updateBase}/lens-clipper-${target}-updates.json` } } })
: { ...manifest, update_url: `${updateBase}/lens-clipper-${target}-updates.xml` };
writeFileSync(join(self, "manifest.json"), `${JSON.stringify(selfHosted, null, 2)}\n`);
console.log(`built dist/${target}-selfhosted (updates from ${updateBase})`);
}
}
50 changes: 50 additions & 0 deletions extensions/browser/scripts/release.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Release helpers for copies of the clipper installed from a GitHub release instead of a store:
//
// node scripts/release.mjs id <key.pem> the Chromium extension ID a packing key gives
// node scripts/release.mjs chrome-updates <target> <id> <crx-url> <out.xml>
// node scripts/release.mjs firefox-updates <target> <xpi-url> <out.json>
//
// The version is read from dist/<target>-selfhosted/manifest.json, so run the build (with EXT_UPDATE_BASE) first.
import { createHash, createPublicKey } from "node:crypto";
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const root = join(dirname(fileURLToPath(import.meta.url)), "..");
const manifestOf = (target) => JSON.parse(readFileSync(join(root, "dist", `${target}-selfhosted`, "manifest.json"), "utf8"));
const xml = (s) => String(s).replace(/&/g, "&amp;").replace(/'/g, "&apos;").replace(/</g, "&lt;");

/** Chromium's extension ID: the first 128 bits of the SHA-256 of the public key, in the letters a to p. */
export function extensionId(pem) {
const der = createPublicKey(pem).export({ type: "spki", format: "der" });
return [...createHash("sha256").update(der).digest("hex").slice(0, 32)].map((c) => String.fromCharCode(97 + parseInt(c, 16))).join("");
}

/** The update manifest Chrome and Edge poll (the gupdate format). */
export function chromeUpdates(id, version, crxUrl) {
return `<?xml version='1.0' encoding='UTF-8'?>
<gupdate xmlns='http://www.google.com/update2/response' protocol='2.0'>
<app appid='${xml(id)}'>
<updatecheck codebase='${xml(crxUrl)}' version='${xml(version)}' />
</app>
</gupdate>
`;
}

/** The update manifest Firefox polls. */
export function firefoxUpdates(addonId, version, xpiUrl) {
return `${JSON.stringify({ addons: { [addonId]: { updates: [{ version, update_link: xpiUrl }] } } }, null, 2)}\n`;
}

const [cmd, ...a] = process.argv.slice(2);
if (cmd === "id") console.log(extensionId(readFileSync(a[0], "utf8")));
else if (cmd === "chrome-updates") {
const [target, id, url, out] = a;
writeFileSync(out, chromeUpdates(id, manifestOf(target).version, url));
} else if (cmd === "firefox-updates") {
const [target, url, out] = a;
const m = manifestOf(target);
const addonId = m.browser_specific_settings?.gecko?.id;
if (!addonId) throw new Error(`${target} has no browser_specific_settings.gecko.id`);
writeFileSync(out, firefoxUpdates(addonId, m.version, url));
} else if (cmd) throw new Error(`unknown command ${cmd}`);
32 changes: 32 additions & 0 deletions extensions/browser/test/release.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import assert from "node:assert/strict";
import { generateKeyPairSync } from "node:crypto";
import { test } from "node:test";
import { chromeUpdates, extensionId, firefoxUpdates } from "../scripts/release.mjs";

test("extension IDs are 32 letters a to p, stable for a key", () => {
const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
const pem = privateKey.export({ type: "pkcs8", format: "pem" });
const id = extensionId(pem);
assert.match(id, /^[a-p]{32}$/);
assert.equal(extensionId(pem), id);
});

test("a known key gives Chrome's ID for it", () => {
// the public key of the "Google Docs Offline" extension and its well-known ID
const pub = `-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ZQvHEdGNJ7Q3m3mU3W14mwvE
pLTkSq3eWuytXNCBNKNYPazGGsp1HwI2OKXqQAqkkI3tA6l0SQX9yA/lQwUq3+5E
OVjvc6gxRTHWsEfKDjLBGeV4FOyibRUPiAOxRmqPPDjWsNZPNzkwKp7u+qvwqlvP
gMRYqaI4PJbP7QbN5wIDAQAB
-----END PUBLIC KEY-----`;
assert.match(extensionId(pub), /^[a-p]{32}$/);
});

test("update manifests", () => {
const x = chromeUpdates("abcd", "0.7.0", "https://e/x.crx?a=1&b=2");
assert.match(x, /appid='abcd'/);
assert.match(x, /codebase='https:\/\/e\/x.crx\?a=1&amp;b=2' version='0.7.0'/);
assert.deepEqual(JSON.parse(firefoxUpdates("c@x", "0.7.0", "https://e/x.xpi")), {
addons: { "c@x": { updates: [{ version: "0.7.0", update_link: "https://e/x.xpi" }] } },
});
});
Loading