Repository navigation
455 lines (445 loc) · 20.3 KB
/
Copy pathverify.yml
File metadata and controls
455 lines (445 loc) · 20.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
name: Verify
on:
pull_request:
# Runs on main too, so main holds the dependency cache (DuckDB's C++ build included) that
# every pull request restores; caches saved by a pull request are visible only to that PR.
push:
branches: [main]
workflow_dispatch:
permissions:
contents: read
# A new push to a pull request cancels its older run. Every push to main (and every manual run) gets
# a group of its own commit, so the next merge neither cancels it nor replaces it while it waits:
# GitHub keeps only one pending run per group, so cancel-in-progress: false alone would not do.
concurrency:
group: verify-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
changes:
name: Classify verification scope
runs-on: ubuntu-24.04
outputs:
code: ${{ steps.scope.outputs.code }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
# Pull requests touching only documentation skip the build and test jobs; everything
# else, and manual runs, verify in full.
- name: Select relevant jobs
id: scope
env:
EVENT_NAME: ${{ github.event_name }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if [[ "${EVENT_NAME}" != "pull_request" ]]; then
echo "code=true" >> "${GITHUB_OUTPUT}"
exit 0
fi
code=false
changed=false
while IFS= read -r -d '' path; do
changed=true
case "${path}" in
# Its output blocks are checked by running its commands (tests/playground).
website/src/content/docs/docs/quickstart.mdx)
code=true
;;
docs/*|website/*|.github/workflows/website.yml|.github/demos/*|.github/*.png|*.md|LICENSE|.gitignore)
;;
*)
code=true
;;
esac
done < <(git diff --name-only -z "${BASE_SHA}" "${HEAD_SHA}")
if [[ "${changed}" != "true" ]]; then
code=true
fi
echo "code=${code}" >> "${GITHUB_OUTPUT}"
checks:
name: Checks and unit tests
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-24.04
# A cold cache compiles DuckDB from source (about 25 minutes); warm runs take a few.
timeout-minutes: 60
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
- run: make clean-partial-duckdb-sources
- uses: astral-sh/setup-uv@v6
with:
version: "0.12.9"
- name: Install unixODBC and the PostgreSQL ODBC driver
run: sudo apt-get update -qq && sudo apt-get install -y -qq unixodbc-dev odbc-postgresql
# Before any cargo command can rewrite it: the release publishes with --locked, which
# refuses a Cargo.lock that no longer matches the manifests.
- name: Check Cargo.lock matches the manifests
run: cargo metadata --locked --format-version 1 > /dev/null
- run: make check-ci
- run: make test-unit
# Builds the sqlload extension (crates/sqlload-python) in the same target directory, so the
# rust-cache above covers it, and runs the Python package's tests.
- run: make test-python
- run: make test-bench-harness
# Checks what crates.io would receive (files, metadata, lockfile) without compiling it again:
# the steps above already build it, and recompiling the packaged copy re-unpacks DuckDB's
# bundled sources into a cached build folder, which fails when that folder is not empty.
- name: Package the crate as crates.io would
run: cargo publish --dry-run --no-verify --locked -p sqlload
containers:
name: Container tests
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-24.04
# A cold cache compiles DuckDB from source (about 25 minutes); warm runs take a few.
timeout-minutes: 60
# packages: read lets the job sign in to GHCR when it has to pull the bastion image.
permissions:
contents: read
packages: read
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- uses: Swatinem/rust-cache@v2
with:
shared-key: containers
- run: make clean-partial-duckdb-sources
# GHCR rate-limits pulls by runner address even when signed in, so the SSH bastion's image is kept in the
# Actions cache, keyed by its digest. The tests name it by digest, which Docker's classic image store drops
# on `docker load`; the containerd image store keeps it, so the job switches Docker to that store first.
- name: Use the containerd image store
run: |
config=/etc/docker/daemon.json
current=$(sudo cat "$config" 2>/dev/null || echo '{}')
updated=$(jq '.features["containerd-snapshotter"] = true' <<<"$current")
sudo tee "$config" >/dev/null <<<"$updated"
sudo systemctl restart docker
timeout 60 sh -c 'until docker info >/dev/null 2>&1; do sleep 1; done'
if ! docker info -f '{{.DriverStatus}}' | grep -q io.containerd.snapshotter.v1; then
echo "::error::Docker is not using the containerd image store: $(docker info -f '{{.DriverStatus}}')"
exit 1
fi
# The image as the testkit pins it, and its cache key.
- name: Read the SSH bastion image
id: bastion
run: |
pins=$(tr -d '\n' < crates/sqlload-testkit/src/ssh/constants.rs)
name=$(grep -o 'const IMAGE: &str = *"[^"]*"' <<<"$pins" | cut -d'"' -f2 || true)
tag=$(grep -o 'const TAG: &str = *"[^"]*"' <<<"$pins" | cut -d'"' -f2 || true)
if [ -z "$name" ] || [ -z "$tag" ] || [ "${tag#*@}" = "$tag" ]; then
echo "::error::cannot read the SSH bastion image's IMAGE and TAG (with its digest) from crates/sqlload-testkit/src/ssh/constants.rs"
exit 1
fi
{
echo "image=$name:$tag"
echo "tagged=$name:${tag%%@*}"
echo "key=ssh-bastion-${tag#*@}"
} >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@v4
id: bastion-cache
with:
path: ${{ runner.temp }}/ssh-bastion.tar
key: ${{ steps.bastion.outputs.key }}
# Loaded from the cache when it resolves by digest there; otherwise pulled once, signed in, so the tests find
# it here, GHCR's rate limit (toomanyrequests) getting one retry a minute later and any other failure none,
# then saved for the cache.
- name: Load or pull the SSH bastion image
id: bastion-image
env:
IMAGE: ${{ steps.bastion.outputs.image }}
TAGGED: ${{ steps.bastion.outputs.tagged }}
ARCHIVE: ${{ runner.temp }}/ssh-bastion.tar
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GHCR_USER: ${{ github.actor }}
run: |
if [ -f "$ARCHIVE" ]; then
docker load -i "$ARCHIVE"
if docker image inspect "$IMAGE" >/dev/null 2>&1; then
echo "loaded $IMAGE from the cache"
exit 0
fi
echo "::warning::the cached SSH bastion image does not resolve as $IMAGE; pulling it"
fi
echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin
for attempt in 1 2; do
if out=$(timeout -k 10 240 docker pull "$IMAGE" 2>&1); then
echo "$out"
docker tag "$IMAGE" "$TAGGED"
docker save -o "$ARCHIVE" "$TAGGED"
echo "pulled=true" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "$out"
if [ "$attempt" = 1 ] && grep -qiE "toomanyrequests|429 Too Many Requests" <<<"$out"; then
echo "GHCR rate-limited the pull of $IMAGE; retrying once in 60 s"
sleep 60
continue
fi
break
done
echo "::error::cannot pull the SSH bastion image $IMAGE: $(tail -n 1 <<<"$out")"
exit 1
- uses: actions/cache/save@v4
if: steps.bastion-image.outputs.pulled == 'true' && steps.bastion-cache.outputs.cache-hit != 'true'
with:
path: ${{ runner.temp }}/ssh-bastion.tar
key: ${{ steps.bastion.outputs.key }}
- run: |
docker pull redpandadata/redpanda:v25.1.4
docker pull postgres:17-alpine
docker pull chrislusf/seaweedfs:3.97
docker pull clickhouse/clickhouse-server:25.8
docker pull clickhouse/clickhouse-server:26.10
docker pull clickhouse/clickhouse-server:25.3
docker pull mysql:8.4
docker pull mcr.microsoft.com/mssql/server:2022-latest
# unixODBC with psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC,
# which the ODBC source's driver tests read through.
- name: Install unixODBC and the ODBC drivers
run: .github/scripts/install_odbc_drivers.sh --with-dev
# Compiles first, so the test step's timeout measures the tests alone.
- name: Build container tests
run: cargo test -p sqlload --features container-tests,odbc --test exporter --test loader --no-run --quiet
- name: Install gdb for stall dumps
continue-on-error: true
run: command -v gdb || (sudo apt-get update -qq && sudo apt-get install -y -qq gdb)
# A run of the tests takes 16 to 20 minutes. When the tests print nothing for seven minutes
# (libtest marks each finished test, and notes one running past 60 seconds), the wrapper
# prints every thread's stack in the test process, so a hang names the stuck call.
- name: Run container tests
timeout-minutes: 40
run: python3 .github/scripts/run_with_stall_dump.py 420 make test-containers
- name: Run ClickHouse container tests on ClickHouse 26.10
timeout-minutes: 10
run: python3 .github/scripts/run_with_stall_dump.py 420 make test-clickhouse-latest
- name: Run ClickHouse container tests on ClickHouse 25.3
timeout-minutes: 10
run: python3 .github/scripts/run_with_stall_dump.py 420 make test-clickhouse-older
- name: Dump containers and disk after a failure
if: failure()
run: |
docker ps -a
for container in $(docker ps -aq); do
echo "::group::Logs of $(docker inspect --format '{{.Name}} {{.Config.Image}}' "${container}")"
docker logs --tail 300 --timestamps "${container}" 2>&1 || true
echo "::endgroup::"
done
df -h
docker system df
# The unit tests on Windows, then the executable end to end: drive paths, a UNC share, the
# psqlODBC driver against the runner's own Postgres, a postgres_cdc load streaming through
# the loopback relay Windows uses instead of a socket file, and a plan and load inside a job
# object's memory limit.
windows:
name: Windows
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: windows-2025
# A cold cache compiles DuckDB from source with MSVC (about 40 minutes); warm runs take a few.
timeout-minutes: 90
env:
E2E_DIR: C:\sqlload-e2e
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- uses: Swatinem/rust-cache@v2
with:
shared-key: windows
- name: Remove incomplete DuckDB sources
shell: bash
run: |
for dir in target/duckdb-sources/*/; do
[ -d "$dir" ] && [ ! -e "$dir.complete" ] && echo "removing incomplete $dir" && rm -rf "$dir"
done; true
- name: Unit tests
run: cargo test -p sqlload --lib --features odbc --quiet
- name: Build the executable
run: cargo build -p sqlload --features odbc --bin sqlload --quiet
- name: Start Postgres with logical replication
shell: pwsh
run: |
Add-Content -Path "$env:PGDATA\postgresql.conf" -Value "wal_level = logical"
Set-Service -Name postgresql-x64-17 -StartupType Manual
Start-Service -Name postgresql-x64-17
# psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC.
- name: Install the ODBC drivers
shell: pwsh
run: .github/scripts/install_odbc_drivers.ps1
# The image's SQL Server LocalDB and MySQL 8.4, for the ODBC drivers' cursor checks. MySQL gets a
# data folder and port of its own, its root user without a password, and runs as a service: the
# runner ends the processes a step starts when the step ends.
- name: Start SQL Server LocalDB and MySQL
shell: pwsh
run: |
$localdb = (Get-Command SqlLocalDB.exe -ErrorAction SilentlyContinue).Source
if (-not $localdb) { $localdb = (Get-ChildItem "C:\Program Files\Microsoft SQL Server" -Recurse -Filter SqlLocalDB.exe | Select-Object -First 1).FullName }
& $localdb create sqlload -s
if ($LASTEXITCODE -ne 0) { throw "SqlLocalDB exited $LASTEXITCODE" }
$mysqld = (Get-ChildItem "C:\Program Files\MySQL" -Recurse -Filter mysqld.exe | Select-Object -First 1).FullName
$ini = "C:\mysql-sqlload.ini"
Set-Content -Path $ini -Value "[mysqld]`ndatadir=C:/mysql-data`nport=3307`nmysqlx=OFF`n"
& $mysqld --defaults-file=$ini --initialize-insecure --console
if ($LASTEXITCODE -ne 0) { throw "mysqld --initialize-insecure exited $LASTEXITCODE" }
& $mysqld --install sqlload-mysql --defaults-file=$ini
if ($LASTEXITCODE -ne 0) { throw "mysqld --install exited $LASTEXITCODE" }
Start-Service -Name sqlload-mysql
foreach ($attempt in 1..60) {
if ((Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { break }
Start-Sleep -Seconds 2
}
if (-not (Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) {
Get-ChildItem C:\mysql-data -Filter *.err | Get-Content
throw "MySQL never listened on 3307"
}
- name: Share a folder as \\localhost\sqlload-e2e
shell: pwsh
run: |
New-Item -ItemType Directory -Force -Path "$env:E2E_DIR\share" | Out-Null
New-SmbShare -Name sqlload-e2e -Path "$env:E2E_DIR\share" -FullAccess Everyone | Out-Null
- name: End-to-end checks
shell: pwsh
run: |
python -m pip install --quiet duckdb pyodbc
python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cursor-drivers --cdc --job-object
# Postgres to DuckDB, 1M narrow rows, sqlload alone with a release build: a wall-time figure
# in the job summary, not a gate (shared runners are too noisy to gate on time), so Verify
# does not wait on it.
e2e:
name: End-to-end timing (report only)
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-24.04
# A cold cache compiles DuckDB from source in release (about 25 minutes); warm runs take a few.
timeout-minutes: 60
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- uses: Swatinem/rust-cache@v2
with:
shared-key: e2e
- run: make clean-partial-duckdb-sources
- run: docker pull postgres:17
- uses: astral-sh/setup-uv@v6
with:
version: "0.12.9"
- name: End-to-end timing (report only)
run: make bench-e2e BENCH_OUT=benchmarks/results/e2e
- name: Report the timing
if: always()
run: |
{
echo "### End-to-end: Postgres to DuckDB, sqlload, 1M narrow rows (not gating)"
echo
cat benchmarks/results/e2e/results.md 2>/dev/null || echo "No results."
} >> "${GITHUB_STEP_SUMMARY}"
# Instruction counts of the hot paths (crates/sqlload-benches) under Callgrind, for the head and
# the merge base in one job; any benchmark more than 5% slower fails unless the pull request has
# the `perf-regression-accepted` label (add it, then re-run this job). Runs on main too, which
# keeps the dependency cache warm and records each commit's counts as an artifact.
perf:
name: Performance gate
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 60
permissions:
contents: read
pull-requests: read
env:
# Instruction counts only: no cache simulation, which is slower and not gated on.
BENCH_ARGS: --output-format=json --callgrind-args=--cache-sim=no
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.95.0
- uses: Swatinem/rust-cache@v2
with:
shared-key: perf
- run: make clean-partial-duckdb-sources
# A runner an earlier run downloaded comes back from the cache without a cargo install record,
# so setup-gungraun's `cargo install` fallback (used when the download fails) stops on
# "already exists"; removing it lets every install strategy put the right version back.
- name: Remove the cached gungraun-runner
run: rm -f "${CARGO_HOME:-${HOME}/.cargo}/bin/gungraun-runner"
- uses: gungraun/setup-gungraun@v1
- name: Benchmark the merge base
if: github.event_name == 'pull_request'
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
# Both switches discard tracked changes, such as a build's rewrite of a stale Cargo.lock,
# so neither side's leftovers block or leak into the other. perf-base.jsonl is untracked.
head=$(git rev-parse HEAD)
git checkout --quiet --force --detach "$(git merge-base "${BASE_SHA}" "${head}")"
if [[ -d crates/sqlload-benches ]]; then
make bench-perf > perf-base.jsonl
else
echo "The merge base has no benchmarks." >&2
: > perf-base.jsonl
fi
git checkout --quiet --force --detach "${head}"
- name: Benchmark the head
run: make bench-perf > perf-head.jsonl
- name: Compare
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
NUMBER: ${{ github.event.pull_request.number }}
run: |
# Labels are read now, not from the event, so re-running this job after adding the label passes.
accept=()
if gh api "repos/${REPOSITORY}/pulls/${NUMBER}" --jq '.labels[].name' | grep -qx 'perf-regression-accepted'; then
accept=(--accept)
fi
python3 benchmarks/perf/compare.py --base perf-base.jsonl --head perf-head.jsonl \
--threshold 5 --summary "${GITHUB_STEP_SUMMARY}" "${accept[@]}"
# Decision 55: counts kept per main commit, so improvement stays visible over time.
- uses: actions/upload-artifact@v4
if: github.event_name != 'pull_request'
with:
name: perf-counts-${{ github.sha }}
path: perf-head.jsonl
verify:
name: Verify
if: always()
needs: [changes, checks, containers, perf, windows]
runs-on: ubuntu-24.04
steps:
- name: Confirm every required job passed or was skipped for a docs-only change
env:
CHANGES_RESULT: ${{ needs.changes.result }}
CHECKS_RESULT: ${{ needs.checks.result }}
CONTAINERS_RESULT: ${{ needs.containers.result }}
PERF_RESULT: ${{ needs.perf.result }}
WINDOWS_RESULT: ${{ needs.windows.result }}
run: |
if [[ "${CHANGES_RESULT}" != "success" ]]; then
echo "Scope classification: ${CHANGES_RESULT}" >&2
exit 1
fi
for result in "${CHECKS_RESULT}" "${CONTAINERS_RESULT}" "${PERF_RESULT}" "${WINDOWS_RESULT}"; do
case "${result}" in
success|skipped) ;;
*)
echo "A verification job ended with ${result}" >&2
exit 1
;;
esac
done