Repository navigation
256 lines (245 loc) · 10.8 KB
/
Copy pathci.yml
File metadata and controls
256 lines (245 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
name: CI
on:
push:
branches: [main]
pull_request:
# closed: a pull request that's merged or closed cancels its runs that are still waiting or going (every job skips)
types: [opened, synchronize, reopened, closed]
# by hand: every project, not only the affected ones
workflow_dispatch:
# Pushes to a pull request cancel its older runs, and so does closing it
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
actions: read # to find main's last green run, the base a push to main is compared with
# Nx (nx.json, and each project's project.json) runs the checks, and only for the projects a change affects: a pull
# request is compared with the main it merges into, a push to main with main's last green CI run.
#
# Every job starts at once and asks Nx itself (.github/actions/affected); one that isn't affected stops after that
# and passes. No job waits for another, so none queues for a free runner twice. The backend's tests are split across
# runners by their recorded durations (fastapi_backend/tests/durations.json, pytest-split): add a shard to the matrix
# to split them further.
#
# Each task runs fresh here: Nx's cache is for a contributor's machine, and CI neither reads nor fills one.
env:
NX_SKIP_NX_CACHE: "true"
# Every workflow runs on GitHub's ubuntu-latest. To use a self-hosted runner instead, set the repository variable
# CI_RUNNER (Settings → Secrets and variables → Actions → Variables) to its labels as JSON: ["self-hosted","Linux","X64"]
jobs:
backend:
name: FastAPI (embedded SurrealDB) ${{ matrix.shard }}/${{ strategy.job-total }}
if: github.event.action != 'closed'
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
strategy:
fail-fast: false
matrix:
shard: [1, 2]
defaults:
run:
working-directory: ./fastapi_backend
env:
ACCESS_SECRET_KEY: ci-access-secret-key-0123456789abcdef
SPLITS: ${{ strategy.job-total }}
GROUP: ${{ matrix.shard }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # the history back to the base the change is compared with
filter: tree:0
- uses: ./.github/actions/affected
id: affected
with:
project: backend
- uses: ./.github/actions/backend-setup
id: setup
if: steps.affected.outputs.run == 'true'
# the commands are the targets in fastapi_backend/project.json; the first shard runs the checks that aren't tests
- name: Lint
if: steps.affected.outputs.run == 'true' && matrix.shard == 1
run: pnpm exec nx run backend:lint
- name: Type check
if: steps.affected.outputs.run == 'true' && matrix.shard == 1
run: pnpm exec nx run backend:typecheck
- name: OpenAPI schema is up to date
if: steps.affected.outputs.run == 'true' && matrix.shard == 1
run: pnpm exec nx run backend:openapi-check
- name: Tests
# a failed check above doesn't stop them
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
# a few minutes a shard on GitHub's runners; the cap keeps a hung test from holding a runner for long
timeout-minutes: 30
run: |
pnpm exec nx run backend:test:ci -- --splits="$SPLITS" --group="$GROUP"
mv .coverage ".coverage.$GROUP"
# this shard's share of the coverage; a push to main adds them up (Backend coverage, below)
- uses: actions/upload-artifact@v7
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
with:
name: backend-coverage-${{ matrix.shard }}
path: fastapi_backend/.coverage.*
include-hidden-files: true
retention-days: 7
backend-surrealdb-server:
name: FastAPI (SurrealDB server) ${{ matrix.shard }}/${{ strategy.job-total }}
if: github.event.action != 'closed'
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
strategy:
fail-fast: false
matrix:
shard: [1, 2]
defaults:
run:
working-directory: ./fastapi_backend
env:
ACCESS_SECRET_KEY: ci-access-secret-key-0123456789abcdef
SPLITS: ${{ strategy.job-total }}
GROUP: ${{ matrix.shard }}
# a self-hosted runner's Docker outlives the job: a name of its own and any free port, removed at the end
SURREAL_CONTAINER: lens-ci-surreal-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
filter: tree:0
- uses: ./.github/actions/affected
id: affected
with:
project: backend
- name: Start SurrealDB
if: steps.affected.outputs.run == 'true'
run: |
# surrealkv, not memory: the memory engine loses concurrent writes (SurrealDB 3.2.4)
# on a tmpfs: the data is thrown away, and syncing every write to the runner's disk made each test's schema take seconds
docker run -d --name "$SURREAL_CONTAINER" --tmpfs /data:mode=1777 -p 127.0.0.1::8000 surrealdb/surrealdb:v3.2.4 start --user root --pass root surrealkv:/data/lens.db
port=$(docker port "$SURREAL_CONTAINER" 8000/tcp | head -n1 | sed 's/.*://')
echo "LENS_TEST_SURREAL_URL=ws://127.0.0.1:$port" >> "$GITHUB_ENV"
for _ in $(seq 1 30); do curl -sf "http://127.0.0.1:$port/health" && break; sleep 1; done
- uses: ./.github/actions/backend-setup
if: steps.affected.outputs.run == 'true'
- name: Tests against the server
if: steps.affected.outputs.run == 'true'
# a few minutes a shard on GitHub's runners; the cap keeps a hung test from holding a runner for long
timeout-minutes: 30
run: pnpm exec nx run backend:test:server -- --splits="$SPLITS" --group="$GROUP"
- name: Remove SurrealDB
if: always()
run: docker rm -f "$SURREAL_CONTAINER" || true
# the shards' coverage added up, on main (on a pull request it would hold the checks until another runner is free)
coverage:
name: Backend coverage
needs: backend
if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.backend.result == 'success' }}
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
defaults:
run:
working-directory: ./fastapi_backend
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v7
with:
pattern: backend-coverage-*
path: fastapi_backend
merge-multiple: true
- uses: astral-sh/setup-uv@v7
with:
cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }}
python-version: "3.12"
- name: Add up the shards
run: |
# no shard data: the backend wasn't affected, so there's nothing to add up
if ! ls .coverage.* >/dev/null 2>&1; then echo "No coverage: the backend wasn't affected"; exit 0; fi
uv run coverage combine
uv run coverage xml -o coverage.xml
uv run coverage report | tee report.txt
{ echo "### Backend coverage"; echo '```'; grep -E '^(Name|TOTAL)' report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
with:
name: backend-coverage
path: fastapi_backend/coverage.xml
if-no-files-found: ignore
frontend:
name: Next.js
if: github.event.action != 'closed'
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
filter: tree:0
- uses: ./.github/actions/affected
id: affected
with:
project: frontend
cache-dependency-path: |
pnpm-lock.yaml
nextjs-frontend/pnpm-lock.yaml
- name: Install
if: steps.affected.outputs.run == 'true'
working-directory: ./nextjs-frontend
run: pnpm install --frozen-lockfile
# the commands are the targets in nextjs-frontend/project.json
- name: Lint
if: steps.affected.outputs.run == 'true'
run: pnpm exec nx run frontend:lint
- name: Type check
if: steps.affected.outputs.run == 'true'
run: pnpm exec nx run frontend:typecheck
- name: Tests
if: steps.affected.outputs.run == 'true'
run: |
set -o pipefail
pnpm exec nx run frontend:test:ci | tee coverage.txt
{ echo "### Frontend coverage"; echo '```'; sed -n '/Coverage summary/,/^=*$/p' coverage.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
- uses: actions/upload-artifact@v7
if: steps.affected.outputs.run == 'true'
with:
name: frontend-coverage
path: nextjs-frontend/coverage/lcov.info
- name: Build
if: steps.affected.outputs.run == 'true'
run: pnpm exec nx run frontend:build
env:
API_BASE_URL: http://localhost:8000
AUTH_SECRET: ci-auth-secret
projects:
name: Other projects
if: github.event.action != 'closed'
runs-on: ${{ fromJSON(vars.CI_RUNNER || '"ubuntu-latest"') }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
filter: tree:0
# every other project with lint, typecheck, test or build targets: a new project.json is all it takes to join
- uses: ./.github/actions/affected
id: affected
- name: Affected projects
env:
PROJECTS: ${{ steps.affected.outputs.projects }}
run: |
{
echo "### Affected projects"
if [ -n "${NX_BASE:-}" ]; then echo "Comparing \`${NX_BASE:0:12}\` with \`${NX_HEAD:0:12}\`."; else echo "Run by hand: every project."; fi
echo
if [ -z "$PROJECTS" ]; then echo "Nx couldn't say, so every job runs."; else
echo "$PROJECTS" | jq -r 'if length == 0 then "None: nothing to build or test." else map("- " + .) | join("\n") end'
fi
} >> "$GITHUB_STEP_SUMMARY"
- uses: astral-sh/setup-uv@v7
if: steps.affected.outputs.run == 'true'
with:
cache-local-path: ${{ runner.environment == 'self-hosted' && format('{0}/uv-cache', runner.tool_cache) || '' }}
python-version: "3.12"
- name: Lint, type check, test and build them
if: steps.affected.outputs.run == 'true'
env:
OTHERS: ${{ steps.affected.outputs.others }}
run: |
# all of them when Nx couldn't say which
if [ -n "$OTHERS" ]; then
pnpm exec nx run-many -t lint typecheck test build --outputStyle=static -p "$(echo "$OTHERS" | jq -r 'join(",")')"
else
pnpm exec nx run-many -t lint typecheck test build --outputStyle=static --exclude=backend,frontend
fi