Repository navigation
250 lines (245 loc) · 11 KB
/
Copy pathrelease-performance.yml
File metadata and controls
250 lines (245 loc) · 11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
name: Release Performance
run-name: >-
${{ github.event.pull_request.title || github.event.head_commit.message || github.workflow }}
# Every push to main compares main with the latest release on the same runner. A regression fails
# the run and opens or updates one issue labelled performance-regression; it does not gate the
# release pull request. A main run that does not finish, such as one that errors, loses its runner
# or times out, opens or updates one issue labelled release-performance-unfinished. Pull requests
# that change the comparison itself run it too.
on:
push:
branches:
- main
workflow_dispatch:
inputs:
baseline:
description: "Published sqlbuild version to compare against (default: the release before the candidate)"
required: false
type: string
pull_request:
paths:
- .github/workflows/release-performance.yml
- .github/actions/dist-build-memory/**
- scripts/compare_release_performance.py
- scripts/release_performance/**
- scripts/compile_performance_ratio/_helpers/edit.py
- scripts/compile_performance_ratio/_helpers/measure.py
- scripts/compile_performance_ratio/_helpers/report.py
- scripts/compile_performance_ratio/constants.py
- scripts/compile_performance_ratio/exceptions.py
- scripts/compile_performance_ratio/models.py
- scripts/cold_compile_performance/_helpers/dense_project.py
- scripts/cold_compile_performance/constants.py
permissions:
contents: read
# Every main commit gets its own comparison, so a later push never cancels the run that would
# report an earlier regression. Pull request runs supersede older runs of the same pull request.
concurrency:
group: >-
release-performance-${{ github.event_name }}-${{
github.event_name == 'push' && github.sha || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
compare:
name: Compare with the previous release
runs-on: ubuntu-24.04
# A backstop above the slowest normal install and build (about 51 minutes together) plus the
# comparison's own limit, so a hung comparison stops at its step and still uploads evidence.
timeout-minutes: 130
outputs:
regressed: ${{ steps.compare.outputs.regressed }}
steps:
- uses: actions/checkout@v4
with:
# Release tags identify the previous release before its wheels reach PyPI.
fetch-depth: 0
- uses: ./.github/actions/setup-uv
with:
version: "0.12.3"
enable-cache: true
cache-dependency-glob: |
uv.lock
Cargo.lock
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- name: Install Python
run: uv python install 3.12
- name: Install dependencies
run: uv sync --locked --all-groups --all-extras
# The candidate uses the released wheels' cargo profile, so both sides of the comparison match.
# That build can need more memory than the hosted runner has; see the action.
- name: Add swap and limit cargo jobs for the candidate build
uses: ./.github/actions/dist-build-memory
with:
step: start
- name: Build candidate wheel
env:
MATURIN_PEP517_ARGS: --profile dist
run: |
uv build --wheel --out-dir "${RUNNER_TEMP}/candidate"
uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate"
- name: Report memory use of the candidate build
if: always()
uses: ./.github/actions/dist-build-memory
with:
step: report
# A release tag created while the candidate built names the baseline's benchmark generator.
- name: Fetch release tags
run: git fetch --tags --force --quiet origin
# Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited
# model recompiles uncached; tighten it back to 1.15x once native rendering is the default.
# Comparisons take 10 to 21 minutes, plus up to 30 while a new baseline reaches PyPI.
- name: Compare candidate and previous release on this runner
id: compare
timeout-minutes: 65
env:
BASELINE: ${{ inputs.baseline }}
EVENT_NAME: ${{ github.event_name }}
run: |
wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl)
evidence="${RUNNER_TEMP}/release-performance"
mkdir -p "${evidence}"
args=(
--candidate "${wheels[0]}"
--work-dir "${RUNNER_TEMP}/release-performance-work"
--output "${evidence}/release-performance.json"
)
if [[ -n "${BASELINE}" ]]; then
args+=(--baseline "${BASELINE}")
fi
# A main commit that is not itself a release still carries the latest release's version,
# so it is compared with that release once PyPI serves it, and with the newest published
# release before it until then. A release commit bumps the version and is compared with
# the newest published release before it.
if [[ "${EVENT_NAME}" == "push" ]]; then
project_version() {
python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
}
version="$(project_version < pyproject.toml)"
previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
if [[ "${version}" == "${previous}" ]]; then
args+=(--same-version-baseline)
fi
fi
# The comparison's own exit status, not tee's, decides the outcome.
set +e
uv run python -m scripts.compare_release_performance "${args[@]}" \
| tee "${evidence}/release-performance.md"
status="${PIPESTATUS[0]}"
set -e
regressed=false
if [[ "${status}" != "0" && -f "${evidence}/release-performance.json" ]] && \
jq -e 'any(.verdicts[]; .regressed)' "${evidence}/release-performance.json" > /dev/null; then
regressed=true
fi
echo "status=${status}" >> "${GITHUB_OUTPUT}"
echo "regressed=${regressed}" >> "${GITHUB_OUTPUT}"
- name: Upload comparison evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/release-performance
if-no-files-found: warn
retention-days: 90
- name: Fail on a regression or comparison error
if: steps.compare.outputs.status != '0'
env:
STATUS: ${{ steps.compare.outputs.status }}
run: |
echo "The release performance comparison failed with exit status ${STATUS:-unknown}" >&2
exit 1
report:
name: Report regression or unfinished comparison
needs: compare
if: ${{ always() && github.event_name == 'push' && needs.compare.result != 'success' }}
runs-on: ubuntu-24.04
# Overlapping main runs report one at a time so they update a single issue.
concurrency:
group: release-performance-report
cancel-in-progress: false
permissions:
actions: read
checks: read
contents: read
issues: write
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- uses: actions/download-artifact@v4
if: needs.compare.outputs.regressed == 'true'
with:
name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
path: evidence
- name: Open or update the performance regression issue
if: needs.compare.outputs.regressed == 'true'
env:
LABEL: performance-regression
run: |
{
echo "Release performance regressed at ${GITHUB_SHA} on main."
echo
echo "Comparison run: ${RUN_URL}"
echo
cat evidence/release-performance.md
} > report.md
gh label create "${LABEL}" --color B60205 \
--description "Main is slower or larger than the latest release" 2>/dev/null || true
issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \
--json number --jq '.[0].number // empty')"
if [[ -n "${issue}" ]]; then
gh issue comment "${issue}" --body-file report.md
echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}"
else
gh issue create --title "Release performance regression on main" \
--label "${LABEL}" --body-file report.md
fi
- uses: actions/checkout@v4
if: needs.compare.outputs.regressed != 'true'
with:
sparse-checkout: scripts
# A failed compare job, or one stopped by its own timeout, did not finish. A run cancelled by
# a newer run or by hand is also `cancelled`, but without the timeout annotation, so the
# script prints nothing for it.
- name: Open or update the unfinished comparison issue
if: needs.compare.outputs.regressed != 'true'
env:
LABEL: release-performance-unfinished
RESULT: ${{ needs.compare.result }}
REGRESSED: ${{ needs.compare.outputs.regressed }}
run: |
job="$(gh api "repos/${GH_REPO}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
--jq '.jobs[] | select(.name == "Compare with the previous release") | .id')"
if [[ -z "${job}" ]]; then
echo "::error::The compare job is not listed in run attempt ${GITHUB_RUN_ATTEMPT}"
exit 1
fi
gh api "repos/${GH_REPO}/check-runs/${job}/annotations" > annotations.json
reason="$(python3 -m scripts.report_unfinished_release_performance \
--result "${RESULT}" --regressed "${REGRESSED}" --annotations annotations.json)"
if [[ -z "${reason}" ]]; then
echo "The comparison was cancelled by a newer run or by hand; nothing to report." \
>> "${GITHUB_STEP_SUMMARY}"
exit 0
fi
{
echo "The release performance comparison did not finish at ${GITHUB_SHA} on main."
echo
echo "Comparison run: ${RUN_URL} (its job summary lists the phases that started)"
echo
echo "${reason}"
} > report.md
gh label create "${LABEL}" --color FBCA04 \
--description "A main release performance comparison did not finish" 2>/dev/null || true
issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \
--json number --jq '.[0].number // empty')"
if [[ -n "${issue}" ]]; then
gh issue comment "${issue}" --body-file report.md
echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}"
else
gh issue create --title "Release performance comparison did not finish on main" \
--label "${LABEL}" --body-file report.md
fi