From 7491ea8a16ee576c9e1728bfc7e0b189c906a113 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 17:15:53 +0200 Subject: [PATCH 1/6] Post the changes in the PR comment, and say why one didn't post The free review comment was a bare link: a reviewer had to open it to find out what changed, having already read the same changes in the inline annotations. It now lists the changes, severity, check id and endpoint, up to five rows and a count of the rest, so the comment answers the question on the pull request. The data comes from a json render that replaces the exit-code probe the has-changes decision used, so the comment and the count read the same comparison and the number of oasdiff runs is unchanged. A comment that could not be posted reported the same way whether the job was missing pull-requests: write or the pull request came from a fork, where GitHub makes the token read-only and no permission helps. The two are now separate: the first is a warning naming the permission to add, the second a plain line, and they are distinguished in the delivery mode the action reports. Co-Authored-By: Claude Opus 5 --- .github/workflows/test-breaking.yaml | 199 ++++++++++++++++++++++++++ .github/workflows/test-changelog.yaml | 96 +++++++++++++ README.md | 16 +-- breaking/entrypoint.sh | 79 ++++++++-- changelog/entrypoint.sh | 81 ++++++++--- 5 files changed, 433 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test-breaking.yaml b/.github/workflows/test-breaking.yaml index b1171cd..99b1b76 100644 --- a/.github/workflows/test-breaking.yaml +++ b/.github/workflows/test-breaking.yaml @@ -348,3 +348,202 @@ jobs: exit 1 ;; esac + + oasdiff_breaking_comment_lists_the_changes: + runs-on: ubuntu-latest + name: Test the PR comment lists the changes + # The comment is what a reviewer reads on the pull request, so it carries + # the changes themselves, capped so a large changelog cannot exceed + # GitHub's 65,536-character comment limit. Stubs oasdiff (a 7-change json + # render) and curl (captures the comment body), so the assertions read the + # exact body the action would post. + steps: + - uses: actions/checkout@v7 + - name: Stub oasdiff + curl and run the entrypoint + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$a" in + json) fmt=json ;; + githubactions) fmt=gha ;; + --open) fmt=open ;; + esac + done + case "${fmt:-text}" in + json) + printf '[' + i=1 + while [ "$i" -le 7 ]; do + [ "$i" -gt 1 ] && printf ',' + printf '{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets%s","section":"paths"}' "$i" + i=$((i+1)) + done + printf ']' + ;; + gha) echo "::error file=specs/base.yaml,line=11::API removed" ;; + open) echo "https://www.oasdiff.com/review/e/stub-id#k=stub-key" >&2 ;; + *) echo "7 changes: 7 error, 0 warning, 0 info" ;; + esac + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$prev" in -X) method=$a ;; esac + prev=$a + done + case " $* " in + *api/telemetry*) cat > /tmp/run/telemetry.json; exit 0 ;; + esac + if [ "${method:-GET}" = GET ]; then echo '[]'; exit 0; fi + cat > /tmp/run/comment.json + printf '201' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REF=refs/pull/7/merge + export GITHUB_REPOSITORY=acme/api + export GITHUB_OUTPUT=/tmp/run/output + export GITHUB_STEP_SUMMARY=/tmp/run/summary + export GITHUB_ACTION_REF=v0 + printf '{"pull_request":{"head":{"repo":{"full_name":"acme/api"}}}}' > /tmp/run/event.json + export GITHUB_EVENT_PATH=/tmp/run/event.json + # jq stays the system jq; only oasdiff and curl are stubbed. + export PATH=/tmp/stub:$PATH + + ./breaking/entrypoint.sh 'specs/base.yaml' 'specs/revision-breaking.yaml' \ + '' '' 'false' '' '' '' '' 'false' 'false' '' '' '' 'false' 'true' 'fake-token' + + if [ ! -f /tmp/run/comment.json ]; then + echo "FAIL: no comment was posted" >&2 + exit 1 + fi + body=$(jq -r '.body' < /tmp/run/comment.json) + echo "--- comment body ---" + echo "$body" + for expected in \ + '7 breaking changes in this pull request' \ + '| severity | change | endpoint |' \ + 'api-removed-without-deprecation' \ + 'GET /pets5' \ + '2 more in the review.' \ + 'oasdiff Pro' + do + case "$body" in + *"$expected"*) ;; + *) echo "FAIL: the comment is missing: $expected" >&2; exit 1 ;; + esac + done + # Capped at 5 rows: the 6th change must not be listed. + case "$body" in + *'GET /pets6'*) echo "FAIL: the comment lists more than the cap" >&2; exit 1 ;; + esac + mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) + if [ "$mode" != "comment" ]; then + echo "FAIL: expected delivery mode 'comment', got '$mode'" >&2 + exit 1 + fi + echo "PASS" + + oasdiff_breaking_comment_failure_names_the_fix: + runs-on: ubuntu-latest + name: Test a blocked PR comment says what to change + # A comment that cannot post is reported two different ways, because only + # one of them is actionable: a job without pull-requests:write gets a + # warning naming the permission, while a pull request from a fork gets a + # plain line, since GitHub makes the token read-only there whatever the + # job does. Both cases are also reported separately in the delivery mode. + steps: + - uses: actions/checkout@v7 + - name: Stub oasdiff + a rejecting curl, run with and without a fork event + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$a" in + json) fmt=json ;; + githubactions) fmt=gha ;; + --open) fmt=open ;; + esac + done + case "${fmt:-text}" in + json) printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}]' ;; + gha) echo "::error file=specs/base.yaml,line=11::API removed" ;; + open) echo "https://www.oasdiff.com/review/e/stub-id#k=stub-key" >&2 ;; + *) echo "1 changes: 1 error, 0 warning, 0 info" ;; + esac + STUB + chmod +x /tmp/stub/oasdiff + + # Rejects the comment with 403, the shape GitHub returns both for a + # missing permission and for a fork's read-only token. + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$prev" in -X) method=$a ;; esac + prev=$a + done + case " $* " in + *api/telemetry*) cat > /tmp/run/telemetry.json; exit 0 ;; + esac + if [ "${method:-GET}" = GET ]; then echo '[]'; exit 0; fi + cat > /dev/null + printf '403' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REF=refs/pull/7/merge + export GITHUB_REPOSITORY=acme/api + export GITHUB_OUTPUT=/tmp/run/output + export GITHUB_STEP_SUMMARY=/tmp/run/summary + export GITHUB_ACTION_REF=v0 + export GITHUB_EVENT_PATH=/tmp/run/event.json + export PATH=/tmp/stub:$PATH + + run_entrypoint () { + ./breaking/entrypoint.sh 'specs/base.yaml' 'specs/revision-breaking.yaml' \ + '' '' 'false' '' '' '' '' 'false' 'false' '' '' '' 'false' 'true' 'fake-token' + } + + echo "=== same-repo pull request: expect a warning naming the permission" + printf '{"pull_request":{"head":{"repo":{"full_name":"acme/api"}}}}' > /tmp/run/event.json + out=$(run_entrypoint 2>&1) + echo "$out" | grep -v '^flags:\|^running oasdiff' + case "$out" in + *"::warning::oasdiff: couldn't post the review link"*"pull-requests: write"*) ;; + *) echo "FAIL: no warning naming the permission" >&2; exit 1 ;; + esac + mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) + if [ "$mode" != "comment_failed" ]; then + echo "FAIL: expected delivery mode 'comment_failed', got '$mode'" >&2 + exit 1 + fi + + echo "=== fork pull request: expect a plain line and no warning" + printf '{"pull_request":{"head":{"repo":{"full_name":"contrib/api-fork"}}}}' > /tmp/run/event.json + out=$(run_entrypoint 2>&1) + echo "$out" | grep -v '^flags:\|^running oasdiff' + case "$out" in + *"comes from a fork"*) ;; + *) echo "FAIL: a fork pull request was not reported as one" >&2; exit 1 ;; + esac + case "$out" in + *"::warning::oasdiff: couldn't post the review link"*) + echo "FAIL: a fork pull request must not warn about a permission it cannot have" >&2 + exit 1 + ;; + esac + mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) + if [ "$mode" != "comment_fork" ]; then + echo "FAIL: expected delivery mode 'comment_fork', got '$mode'" >&2 + exit 1 + fi + echo "PASS" diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index 27c0664..31cb9fd 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -229,3 +229,99 @@ jobs: echo "FAIL: review URL not in step summary" >&2; exit 1 fi echo "PASS" + + oasdiff_changelog_comment_lists_the_changes: + runs-on: ubuntu-latest + name: Test the PR comment lists the changes + # Sibling of the breaking action's job of the same name: the two + # entrypoints build the same comment, so a change to one that misses the + # other fails here. Stubs oasdiff (a 6-change json render) and curl + # (captures the comment body). + steps: + - uses: actions/checkout@v7 + - name: Stub oasdiff + curl and run the entrypoint + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$a" in + json) fmt=json ;; + githubactions) fmt=gha ;; + --open) fmt=open ;; + esac + done + case "${fmt:-text}" in + json) + printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}' + printf ',{"id":"response-property-became-optional","text":"became optional","level":1,"operation":"POST","path":"/pets","section":"paths"}' + i=3 + while [ "$i" -le 6 ]; do + printf ',{"id":"request-property-added","text":"added","level":1,"operation":"PUT","path":"/pets%s","section":"paths"}' "$i" + i=$((i+1)) + done + printf ']' + ;; + gha) echo "::error file=specs/base.yaml,line=11::API removed" ;; + open) echo "https://www.oasdiff.com/review/e/stub-id#k=stub-key" >&2 ;; + *) echo "6 changes: 1 error, 0 warning, 5 info" ;; + esac + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$prev" in -X) method=$a ;; esac + prev=$a + done + case " $* " in + *api/telemetry*) cat > /tmp/run/telemetry.json; exit 0 ;; + esac + if [ "${method:-GET}" = GET ]; then echo '[]'; exit 0; fi + cat > /tmp/run/comment.json + printf '201' + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REF=refs/pull/7/merge + export GITHUB_REPOSITORY=acme/api + export GITHUB_OUTPUT=/tmp/run/output + export GITHUB_STEP_SUMMARY=/tmp/run/summary + export GITHUB_ACTION_REF=v0 + printf '{"pull_request":{"head":{"repo":{"full_name":"acme/api"}}}}' > /tmp/run/event.json + export GITHUB_EVENT_PATH=/tmp/run/event.json + export PATH=/tmp/stub:$PATH + + ./changelog/entrypoint.sh 'specs/base.yaml' 'specs/revision.yaml' \ + 'false' '' '' 'false' 'false' '' '' '' 'false' '' '' '' 'false' 'true' 'fake-token' '' + + if [ ! -f /tmp/run/comment.json ]; then + echo "FAIL: no comment was posted" >&2 + exit 1 + fi + body=$(jq -r '.body' < /tmp/run/comment.json) + echo "--- comment body ---" + echo "$body" + for expected in \ + '6 API changes in this pull request' \ + 'api-removed-without-deprecation' \ + 'response-property-became-optional' \ + '| error | ' \ + '| info | ' \ + '1 more in the review.' \ + 'oasdiff Pro' + do + case "$body" in + *"$expected"*) ;; + *) echo "FAIL: the comment is missing: $expected" >&2; exit 1 ;; + esac + done + mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) + if [ "$mode" != "comment" ]; then + echo "FAIL: expected delivery mode 'comment', got '$mode'" >&2 + exit 1 + fi + echo "PASS" diff --git a/README.md b/README.md index f07b2b6..214c867 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ on: branches: [ "main" ] permissions: contents: read - pull-requests: write # lets the action post the review link as a PR comment + pull-requests: write # lets the action post the changes as a PR comment jobs: breaking-changes: runs-on: ubuntu-latest @@ -48,7 +48,7 @@ jobs: github-token: ${{ github.token }} ``` -This compares your spec on the PR branch against the base branch and fails the workflow if any breaking changes are found. When changes are found it posts a side-by-side review link as a PR comment; drop `github-token` and the `pull-requests: write` permission to keep that link in the job summary instead. +This compares your spec on the PR branch against the base branch and fails the workflow if any breaking changes are found. When changes are found it posts a PR comment listing them, linking to a side-by-side review; drop `github-token` and the `pull-requests: write` permission to keep that link in the job summary instead. --- @@ -70,7 +70,7 @@ The following actions run the oasdiff CLI directly in your GitHub runner — no ### Check for breaking changes -Detects changes that break existing API clients and writes inline `::error::` annotations on the pull request's Files changed tab. Fails the workflow when changes at or above the `fail-on` severity are found. When changes are found it also uploads the comparison and links to a full side-by-side review (the `review` input, on by default); the two specs are encrypted in CI before upload, so the server cannot read them. The link is posted as a pull-request comment when you pass `github-token` (and grant `pull-requests: write`); otherwise, and on fork PRs where the token is read-only, it falls back to the job summary. +Detects changes that break existing API clients and writes inline `::error::` annotations on the pull request's Files changed tab. Fails the workflow when changes at or above the `fail-on` severity are found. When changes are found it also uploads the comparison and links to a full side-by-side review (the `review` input, on by default); the two specs are encrypted in CI before upload, so the server cannot read them. The changes and the link are posted as a pull-request comment when you pass `github-token` (and grant `pull-requests: write`); otherwise, and on fork PRs where the token is read-only, the link falls back to the job summary. ```yaml name: oasdiff @@ -79,7 +79,7 @@ on: branches: [ "main" ] permissions: contents: read - pull-requests: write # lets the action post the review link as a PR comment + pull-requests: write # lets the action post the changes as a PR comment jobs: breaking-changes: runs-on: ubuntu-latest @@ -112,11 +112,11 @@ jobs: | `output-to-file` | `''` | Write output to this file path instead of stdout | file path | | `allow-external-refs` | `false` | Resolve external `$ref`s. Defaults to `false` to prevent SSRF on untrusted pull requests. Set `true` if your spec references external URLs or loads split files by file path | `true`, `false` | | `review` | `true` | When changes are found, upload the comparison to oasdiff.com and link to a direct side-by-side review. The two specs are encrypted in CI before upload and the decryption key stays in the URL fragment, so the server cannot read them. Set `false` to skip the upload, so no spec leaves CI | `true`, `false` | -| `github-token` | `''` | Token used to post the review link as a pull-request comment, so reviewers see it on the PR instead of only in the job summary. Pass `${{ github.token }}` and grant `permissions: pull-requests: write`. Optional; omit it to keep the link in the job summary only. Fork PRs (read-only token) fall back to the summary | `${{ github.token }}` | +| `github-token` | `''` | Token used to post the changes as a pull-request comment, so reviewers see them on the PR instead of only in the job summary. Pass `${{ github.token }}` and grant `permissions: pull-requests: write`. Optional; omit it to keep the link in the job summary only. Fork PRs (read-only token) fall back to the summary | `${{ github.token }}` | ### Generate a changelog -Outputs the changes that can affect API consumers (breaking and non-breaking) between two specs. Documentation-only edits, such as descriptions and examples, are not included; use the [diff action](#generate-a-diff-report) to see every difference in the API definition. When changes are found it also uploads the comparison and links to a full side-by-side review (the `review` input, on by default); the two specs are encrypted in CI before upload, so the server cannot read them. The link is posted as a pull-request comment when you pass `github-token` (and grant `pull-requests: write`); otherwise, and on fork PRs where the token is read-only, it falls back to the job summary. +Outputs the changes that can affect API consumers (breaking and non-breaking) between two specs. Documentation-only edits, such as descriptions and examples, are not included; use the [diff action](#generate-a-diff-report) to see every difference in the API definition. When changes are found it also uploads the comparison and links to a full side-by-side review (the `review` input, on by default); the two specs are encrypted in CI before upload, so the server cannot read them. The changes and the link are posted as a pull-request comment when you pass `github-token` (and grant `pull-requests: write`); otherwise, and on fork PRs where the token is read-only, the link falls back to the job summary. ```yaml name: oasdiff @@ -125,7 +125,7 @@ on: branches: [ "main" ] permissions: contents: read - pull-requests: write # lets the action post the review link as a PR comment + pull-requests: write # lets the action post the changes as a PR comment jobs: changelog: runs-on: ubuntu-latest @@ -157,7 +157,7 @@ jobs: | `output-to-file` | `''` | Write output to this file path instead of stdout | file path | | `allow-external-refs` | `false` | Resolve external `$ref`s. Defaults to `false` to prevent SSRF on untrusted pull requests. Set `true` if your spec references external URLs or loads split files by file path | `true`, `false` | | `review` | `true` | When changes are found, upload the comparison to oasdiff.com and link to a direct side-by-side review. The two specs are encrypted in CI before upload and the decryption key stays in the URL fragment, so the server cannot read them. Set `false` to skip the upload, so no spec leaves CI | `true`, `false` | -| `github-token` | `''` | Token used to post the review link as a pull-request comment, so reviewers see it on the PR instead of only in the job summary. Pass `${{ github.token }}` and grant `permissions: pull-requests: write`. Optional; omit it to keep the link in the job summary only. Fork PRs (read-only token) fall back to the summary | `${{ github.token }}` | +| `github-token` | `''` | Token used to post the changes as a pull-request comment, so reviewers see them on the PR instead of only in the job summary. Pass `${{ github.token }}` and grant `permissions: pull-requests: write`. Optional; omit it to keep the link in the job summary only. Fork PRs (read-only token) fall back to the summary | `${{ github.token }}` | ### Generate a diff report diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index 43e1f7b..c305461 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -24,6 +24,38 @@ readonly allow_external_refs="${15}" readonly review="${16}" readonly github_token="${17}" +# from_fork reports whether this run is a pull request from a fork. GitHub makes +# GITHUB_TOKEN read-only there, so a comment cannot post however the job is +# configured, and the job summary is the only place the link can go. +from_fork () { + [ -n "${GITHUB_EVENT_PATH:-}" ] || return 1 + [ -f "$GITHUB_EVENT_PATH" ] || return 1 + head_repo=$(jq -r '.pull_request.head.repo.full_name // empty' < "$GITHUB_EVENT_PATH" 2>/dev/null) || return 1 + [ -n "$head_repo" ] || return 1 + [ "$head_repo" != "${GITHUB_REPOSITORY:-}" ] +} + +# review_comment_changes writes the changes themselves into the comment, so a +# reviewer reads what broke on the pull request instead of following a link to +# find out. Capped at $comment_change_limit rows: a changelog can run to +# thousands of changes and a GitHub comment is limited to 65,536 characters. +# $1: the json render of the comparison (Run 3). +comment_change_limit=5 +review_comment_changes () { + changes_file="$1" + [ -s "$changes_file" ] || return 0 + + jq -r --argjson max "$comment_change_limit" ' + def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; + def endpoint: + if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; + "| severity | change | endpoint |", + "| --- | --- | --- |", + (.[:$max][] | "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"), + (if length > $max then "", "\(length - $max) more in the review." else empty end) + ' < "$changes_file" 2>/dev/null || true +} + # post_review_comment posts (or updates) a single pull-request comment with the # free side-by-side review link, so reviewers see it on the PR rather than only # in the job summary (a low-traffic page most users never open). Best-effort and @@ -78,14 +110,21 @@ post_review_comment () { | jq -r --arg m "$marker" 'map(select(.body | contains($m))) | .[0].id // empty' 2>/dev/null) || existing_id="" if [ -n "$review_url" ]; then + if [ "${change_count:-0}" -eq 1 ]; then + heading="1 breaking change in this pull request" + else + heading="${change_count:-0} breaking changes in this pull request" + fi body="${marker} -### 📋 [View the side-by-side API change review](${review_url}) +### ⚠️ ${heading} -The link expires in 7 days. +$(review_comment_changes "${changes_json:-}") + +[See them side by side, with the old and new spec](${review_url}) (the link expires in 7 days) 🔒 Your API specs are encrypted in CI before they're uploaded. The decryption key stays in this link's URL fragment (after the #), which browsers never send to a server, so oasdiff cannot read your specs. [How it works →](https://www.oasdiff.com/docs/free-review#privacy) -Posted automatically by the [oasdiff GitHub Action](https://www.oasdiff.com/docs/free-review#github-action). To turn this off (no spec upload, no comment), set \`review: false\` on the action." +Posted automatically by the [oasdiff GitHub Action](https://www.oasdiff.com/docs/free-review#github-action). To approve or reject each change here, with a commit status that holds the merge until the team signs off, see [oasdiff Pro](https://www.oasdiff.com/pricing). To turn this comment off (no spec upload, no comment), set \`review: false\` on the action." elif [ -n "$existing_id" ]; then body="${marker} ### ✅ No breaking changes in the latest revision." @@ -110,9 +149,14 @@ The link expires in 7 days. if [ "$code" -ge 200 ] && [ "$code" -lt 300 ]; then [ -n "$review_url" ] && report_review_delivery "comment" echo "oasdiff: posted the side-by-side review link as a PR comment." + elif from_fork; then + # Expected, not a misconfiguration: report it apart from the failures + # the user can act on, and say it plainly rather than as a warning. + [ -n "$review_url" ] && report_review_delivery "comment_fork" + echo "oasdiff: this pull request comes from a fork, where GitHub makes the token read-only, so the review link is in the job summary instead of a comment." else [ -n "$review_url" ] && report_review_delivery "comment_failed" - echo "::notice::oasdiff: couldn't post the review link as a PR comment (HTTP ${code}). On fork pull requests the token is read-only; otherwise grant 'permissions: pull-requests: write'. The link is still in the job summary." + echo "::warning::oasdiff: couldn't post the review link as a PR comment (HTTP ${code}). Add 'permissions: pull-requests: write' to the job that runs this action. The link is in the job summary meanwhile." fi } @@ -229,17 +273,22 @@ oasdiff breaking "$base" "$revision" $flags --format githubactions --template= | delimiter=$(cat /proc/sys/kernel/random/uuid | tr -d '-') echo "breaking<<$delimiter" >>"$GITHUB_OUTPUT" -# Whether there are breaking changes is read off oasdiff's exit code with -# --fail-on=WARN (breaking renders WARN-and-above, and WARN is the lowest level -# it accepts for fail-on): exit 1 means at least one breaking change, 0 means -# none. This is format-proof; parsing $breaking_changes is not, because its shape -# follows any 'format' set in .oasdiff.yaml (e.g. json renders "[]", not "No ..."). -# The explicit --fail-on overrides a config fail-on for this probe only; Run 1's -# authoritative gate exit code is untouched. -changes_exit=0 +# Run 3: one json render answers two questions, how many breaking changes +# there are and what they are, so the count below and the PR comment read the +# same data. json is format-proof, where parsing $breaking_changes is not: its +# shape follows any 'format' set in .oasdiff.yaml (json renders "[]", not +# "No ..."). --format and --template= override that config for this render +# only; Run 1's authoritative gate exit code is untouched. The render goes to a +# file, never a variable, because a large changelog on a command line hits +# ARG_MAX. +changes_json=$(mktemp) # shellcheck disable=SC2086 # the flags variables hold multiple arguments; splitting is intended -oasdiff breaking "$base" "$revision" $flags --fail-on=WARN --template= >/dev/null 2>&1 || changes_exit=$? -if [ "$changes_exit" -eq 1 ]; then +oasdiff breaking "$base" "$revision" $flags --format json --template= > "$changes_json" 2>/dev/null || true +change_count=$(jq 'length' < "$changes_json" 2>/dev/null) || change_count=0 +case "$change_count" in + ''|*[!0-9]*) change_count=0 ;; +esac +if [ "$change_count" -gt 0 ]; then write_output "$(echo "$breaking_changes" | head -n 1)" "$breaking_changes" free_review_url="" @@ -294,6 +343,8 @@ else fi fi +rm -f "$changes_json" + echo "$delimiter" >>"$GITHUB_OUTPUT" # review_url is a single-line output, written after the multiline `breaking` # block is closed so it doesn't get folded into that value. Empty when there diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index c14ce97..222bb29 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -45,6 +45,38 @@ readonly review="${16}" readonly github_token="${17}" readonly oasdiff_token="${18}" +# from_fork reports whether this run is a pull request from a fork. GitHub makes +# GITHUB_TOKEN read-only there, so a comment cannot post however the job is +# configured, and the job summary is the only place the link can go. +from_fork () { + [ -n "${GITHUB_EVENT_PATH:-}" ] || return 1 + [ -f "$GITHUB_EVENT_PATH" ] || return 1 + head_repo=$(jq -r '.pull_request.head.repo.full_name // empty' < "$GITHUB_EVENT_PATH" 2>/dev/null) || return 1 + [ -n "$head_repo" ] || return 1 + [ "$head_repo" != "${GITHUB_REPOSITORY:-}" ] +} + +# review_comment_changes writes the changes themselves into the comment, so a +# reviewer reads what changed on the pull request instead of following a link to +# find out. Capped at $comment_change_limit rows: a changelog can run to +# thousands of changes and a GitHub comment is limited to 65,536 characters. +# $1: the json render of the comparison. +comment_change_limit=5 +review_comment_changes () { + changes_file="$1" + [ -s "$changes_file" ] || return 0 + + jq -r --argjson max "$comment_change_limit" ' + def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; + def endpoint: + if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; + "| severity | change | endpoint |", + "| --- | --- | --- |", + (.[:$max][] | "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"), + (if length > $max then "", "\(length - $max) more in the review." else empty end) + ' < "$changes_file" 2>/dev/null || true +} + # post_review_comment posts (or updates) a single pull-request comment with the # free side-by-side review link, so reviewers see it on the PR rather than only # in the job summary (a low-traffic page most users never open). Best-effort and @@ -99,14 +131,21 @@ post_review_comment () { | jq -r --arg m "$marker" 'map(select(.body | contains($m))) | .[0].id // empty' 2>/dev/null) || existing_id="" if [ -n "$review_url" ]; then + if [ "${change_count:-0}" -eq 1 ]; then + heading="1 API change in this pull request" + else + heading="${change_count:-0} API changes in this pull request" + fi body="${marker} -### 📋 [View the side-by-side API change review](${review_url}) +### 📋 ${heading} -The link expires in 7 days. +$(review_comment_changes "${changes_json:-}") + +[See them side by side, with the old and new spec](${review_url}) (the link expires in 7 days) 🔒 Your API specs are encrypted in CI before they're uploaded. The decryption key stays in this link's URL fragment (after the #), which browsers never send to a server, so oasdiff cannot read your specs. [How it works →](https://www.oasdiff.com/docs/free-review#privacy) -Posted automatically by the [oasdiff GitHub Action](https://www.oasdiff.com/docs/free-review#github-action). To turn this off (no spec upload, no comment), set \`review: false\` on the action." +Posted automatically by the [oasdiff GitHub Action](https://www.oasdiff.com/docs/free-review#github-action). To approve or reject each change here, with a commit status that holds the merge until the team signs off, see [oasdiff Pro](https://www.oasdiff.com/pricing). To turn this comment off (no spec upload, no comment), set \`review: false\` on the action." elif [ -n "$existing_id" ]; then body="${marker} ### ✅ No API changes in the latest revision." @@ -131,9 +170,14 @@ The link expires in 7 days. if [ "$code" -ge 200 ] && [ "$code" -lt 300 ]; then [ -n "$review_url" ] && report_review_delivery "comment" echo "oasdiff: posted the side-by-side review link as a PR comment." + elif from_fork; then + # Expected, not a misconfiguration: report it apart from the failures + # the user can act on, and say it plainly rather than as a warning. + [ -n "$review_url" ] && report_review_delivery "comment_fork" + echo "oasdiff: this pull request comes from a fork, where GitHub makes the token read-only, so the review link is in the job summary instead of a comment." else [ -n "$review_url" ] && report_review_delivery "comment_failed" - echo "::notice::oasdiff: couldn't post the review link as a PR comment (HTTP ${code}). On fork pull requests the token is read-only; otherwise grant 'permissions: pull-requests: write'. The link is still in the job summary." + echo "::warning::oasdiff: couldn't post the review link as a PR comment (HTTP ${code}). Add 'permissions: pull-requests: write' to the job that runs this action. The link is in the job summary meanwhile." fi } @@ -380,19 +424,22 @@ if [ "$exit_code" -ge 2 ]; then fi rm -f "$_err" -# Decide whether there are changes independently of --format. The user-facing -# $output varies by format (json/yaml render "[]", markup adds a header line), so -# parsing it is fragile. Instead read oasdiff's exit code with --fail-on=INFO: -# exit 1 means the changelog has at least one change at or above --level, exit 0 -# means it's empty. --level stays in $flags so detection matches what the user -# sees; the explicit --fail-on overrides any config fail-on for this probe only, -# and the rendered run above is untouched. --template= overrides a 'template' set -# in .oasdiff.yaml, which would otherwise error this probe (templates are rejected -# for the default text format) and make it a false negative. -changes_exit=0 +# One json render answers two questions, how many changes there are and what +# they are, so the count below and the PR comment read the same data. The +# user-facing $output cannot be parsed for either: it varies by format +# (json/yaml render "[]", markup adds a header line). --level stays in $flags so +# this matches what the user sees, while --format and --template= override that +# config for this render only and the rendered run above is untouched. The +# render goes to a file, never a variable, because a large changelog on a +# command line hits ARG_MAX. +changes_json=$(mktemp) # shellcheck disable=SC2086 # the flags variables hold multiple arguments; splitting is intended -oasdiff changelog "$base" "$revision" $flags --fail-on=INFO --template= >/dev/null 2>&1 || changes_exit=$? -if [ "$changes_exit" -eq 1 ]; then +oasdiff changelog "$base" "$revision" $flags --format json --template= > "$changes_json" 2>/dev/null || true +change_count=$(jq 'length' < "$changes_json" 2>/dev/null) || change_count=0 +case "$change_count" in + ''|*[!0-9]*) change_count=0 ;; +esac +if [ "$change_count" -gt 0 ]; then write_output "$output" free_review_url="" @@ -458,6 +505,8 @@ else fi fi +rm -f "$changes_json" + echo "$delimiter" >>"$GITHUB_OUTPUT" # review_url is a single-line output, written after the multiline `changelog` # block is closed so it doesn't get folded into that value. Empty when there From 50cd0a87882d03de005e03233e1c2d2ff1a0db27 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 17:17:48 +0200 Subject: [PATCH 2/6] Teach the Pro-path stubs the json render Both Pro jobs signalled "there are changes" by exiting 1 on the --fail-on probe, which the json render replaced, so the Pro branch never ran. Co-Authored-By: Claude Opus 5 --- .github/workflows/test-changelog.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index 31cb9fd..abee626 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -105,12 +105,13 @@ jobs: set -euo pipefail mkdir -p /tmp/stub /tmp/run - # The entrypoint invokes oasdiff three times: the changes probe - # (--fail-on=INFO -> exit 1 means changes exist), the user-facing - # render, and the authenticated --open upload. Branch on the args. + # The entrypoint invokes oasdiff three times: the json render (its + # length says whether there are changes, and feeds the PR comment), + # the user-facing render, and the authenticated --open upload. Branch + # on the args. cat > /tmp/stub/oasdiff <<'STUB' #!/bin/sh - for a in "$@"; do [ "$a" = "--fail-on=INFO" ] && exit 1; done + for a in "$@"; do [ "$a" = "json" ] && { printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}]'; exit 0; }; done for a in "$@"; do if [ "$a" = "--open" ]; then echo 'Error: upload failed (HTTP 402): {"code":"subscription_expired","message":"Your oasdiff trial or subscription has expired. Renew at https://www.oasdiff.com/pricing to continue using oasdiff Pro.","upgrade_url":"https://www.oasdiff.com/pricing"}' >&2 @@ -162,7 +163,7 @@ jobs: runs-on: ubuntu-latest name: Test changelog Pro path posts the review comment and sets the gate # Companion to changelog_pro_expired_trial_warns_without_failing: the success - # path. Stubs oasdiff (changes present; --open prints a /review/ep URL + + # path. Stubs oasdiff (one change in the json render; --open prints a /review/ep URL + # "review status: pending") and curl (GitHub API: empty comment list, then # 2xx for the comment + status POSTs). Asserts the Pro review comment is # posted, the oasdiff commit status is set, and the review URL reaches the @@ -177,7 +178,7 @@ jobs: cat > /tmp/stub/oasdiff <<'STUB' #!/bin/sh - for a in "$@"; do [ "$a" = "--fail-on=INFO" ] && exit 1; done + for a in "$@"; do [ "$a" = "json" ] && { printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}]'; exit 0; }; done for a in "$@"; do if [ "$a" = "--open" ]; then echo "oasdiff: opening review at https://www.oasdiff.com/review/ep/test-token-uuid#k=test-key" >&2 From a1c739c4351c906909375c35349b48a1fead60f8 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 18:01:30 +0200 Subject: [PATCH 3/6] Order the comment's rows by severity and collapse the rest Severest first, so the five rows that stay open are the ones that decide whether the pull request can merge rather than the first five the comparison happened to produce. The remainder collapses into a details block, up to fifty rows, so a reviewer gets every change without leaving the pull request and the comment stays short. Beyond fifty, the count goes in a line and the review carries the rest. Co-Authored-By: Claude Opus 5 --- .github/workflows/test-breaking.yaml | 14 +++++++++----- .github/workflows/test-changelog.yaml | 9 ++++++++- breaking/entrypoint.sh | 28 ++++++++++++++++++++------- changelog/entrypoint.sh | 28 ++++++++++++++++++++------- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-breaking.yaml b/.github/workflows/test-breaking.yaml index 99b1b76..ecdd7c4 100644 --- a/.github/workflows/test-breaking.yaml +++ b/.github/workflows/test-breaking.yaml @@ -431,7 +431,8 @@ jobs: '| severity | change | endpoint |' \ 'api-removed-without-deprecation' \ 'GET /pets5' \ - '2 more in the review.' \ + '
2 more' \ + 'GET /pets6' \ 'oasdiff Pro' do case "$body" in @@ -439,10 +440,13 @@ jobs: *) echo "FAIL: the comment is missing: $expected" >&2; exit 1 ;; esac done - # Capped at 5 rows: the 6th change must not be listed. - case "$body" in - *'GET /pets6'*) echo "FAIL: the comment lists more than the cap" >&2; exit 1 ;; - esac + # Five rows stay open and the rest collapse, so the 6th change is + # listed but only inside the details block. + open_rows=$(printf '%s' "$body" | sed -n '1,/
/p' | grep -c '^| error ') + if [ "$open_rows" -ne 5 ]; then + echo "FAIL: expected 5 rows above the collapsed block, got $open_rows" >&2 + exit 1 + fi mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) if [ "$mode" != "comment" ]; then echo "FAIL: expected delivery mode 'comment', got '$mode'" >&2 diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index abee626..c0ea994 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -312,7 +312,7 @@ jobs: 'response-property-became-optional' \ '| error | ' \ '| info | ' \ - '1 more in the review.' \ + '
1 more' \ 'oasdiff Pro' do case "$body" in @@ -320,6 +320,13 @@ jobs: *) echo "FAIL: the comment is missing: $expected" >&2; exit 1 ;; esac done + # Severest first, so the row that decides whether the pull request + # can merge is never the one that collapses. + first_row=$(printf '%s' "$body" | grep -m1 '^| \(error\|warning\|info\) ') + case "$first_row" in + '| error | '*) ;; + *) echo "FAIL: the first row is not the error: $first_row" >&2; exit 1 ;; + esac mode=$(jq -r '.props.mode' < /tmp/run/telemetry.json) if [ "$mode" != "comment" ]; then echo "FAIL: expected delivery mode 'comment', got '$mode'" >&2 diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index c305461..399265d 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -37,22 +37,36 @@ from_fork () { # review_comment_changes writes the changes themselves into the comment, so a # reviewer reads what broke on the pull request instead of following a link to -# find out. Capped at $comment_change_limit rows: a changelog can run to +# find out. Severest first, so the rows above the fold are the ones that decide +# whether the pull request can merge; $comment_rows_shown of them stay open and +# the rest collapse, up to $comment_rows_listed, since a changelog can run to # thousands of changes and a GitHub comment is limited to 65,536 characters. # $1: the json render of the comparison (Run 3). -comment_change_limit=5 +comment_rows_shown=5 +comment_rows_listed=50 review_comment_changes () { changes_file="$1" [ -s "$changes_file" ] || return 0 - jq -r --argjson max "$comment_change_limit" ' + jq -r --argjson shown "$comment_rows_shown" --argjson listed "$comment_rows_listed" ' def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - "| severity | change | endpoint |", - "| --- | --- | --- |", - (.[:$max][] | "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"), - (if length > $max then "", "\(length - $max) more in the review." else empty end) + def row: "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"; + def header: "| severity | change | endpoint |", "| --- | --- | --- |"; + sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all + | ($all | length) as $total + | header, + ($all[:$shown][] | row), + (if $total > $shown then + "", + "
\([$total, $listed] | min - $shown) more", + "", + header, + ($all[$shown:$listed][] | row), + "
" + else empty end), + (if $total > $listed then "", "\($total - $listed) more in the review." else empty end) ' < "$changes_file" 2>/dev/null || true } diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index 222bb29..3a9851a 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -58,22 +58,36 @@ from_fork () { # review_comment_changes writes the changes themselves into the comment, so a # reviewer reads what changed on the pull request instead of following a link to -# find out. Capped at $comment_change_limit rows: a changelog can run to +# find out. Severest first, so the rows above the fold are the ones that decide +# whether the pull request can merge; $comment_rows_shown of them stay open and +# the rest collapse, up to $comment_rows_listed, since a changelog can run to # thousands of changes and a GitHub comment is limited to 65,536 characters. # $1: the json render of the comparison. -comment_change_limit=5 +comment_rows_shown=5 +comment_rows_listed=50 review_comment_changes () { changes_file="$1" [ -s "$changes_file" ] || return 0 - jq -r --argjson max "$comment_change_limit" ' + jq -r --argjson shown "$comment_rows_shown" --argjson listed "$comment_rows_listed" ' def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - "| severity | change | endpoint |", - "| --- | --- | --- |", - (.[:$max][] | "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"), - (if length > $max then "", "\(length - $max) more in the review." else empty end) + def row: "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"; + def header: "| severity | change | endpoint |", "| --- | --- | --- |"; + sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all + | ($all | length) as $total + | header, + ($all[:$shown][] | row), + (if $total > $shown then + "", + "
\([$total, $listed] | min - $shown) more", + "", + header, + ($all[$shown:$listed][] | row), + "
" + else empty end), + (if $total > $listed then "", "\($total - $listed) more in the review." else empty end) ' < "$changes_file" 2>/dev/null || true } From 38ae0f555a50908ba4c7a64db67eaf0d8be48db6 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 18:10:07 +0200 Subject: [PATCH 4/6] Give the Pro comment the same change table The free comment lists the changes and the Pro comment did not, so the paid surface showed less than the free one. The Pro comment now carries the same count, table and link, with the gate state above them, which is the part only Pro has. Its test asserts the body rather than only the log line. Co-Authored-By: Claude Opus 5 --- .github/workflows/test-changelog.yaml | 37 ++++++++++++++++++++++++--- changelog/entrypoint.sh | 11 +++++++- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index c0ea994..676eca4 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -190,12 +190,21 @@ jobs: STUB chmod +x /tmp/stub/oasdiff - # The POST calls carry -w %{http_code}; return a 2xx. The comment-list - # GET has no such flag; return an empty JSON array so jq finds no - # existing comment and the entrypoint POSTs a new one. + # The POST calls carry -w %{http_code}; return a 2xx, keeping the + # comment body so the assertions can read it. The comment-list GET has + # no such flag; return an empty JSON array so jq finds no existing + # comment and the entrypoint POSTs a new one. cat > /tmp/stub/curl <<'STUB' #!/bin/sh - for a in "$@"; do case "$a" in *"%{http_code}"*) echo "201"; exit 0 ;; esac; done + for a in "$@"; do + case "$a" in *"%{http_code}"*) http_code=1 ;; esac + case "$a" in *"/issues/"*) comment=1 ;; esac + done + if [ -n "${http_code:-}" ]; then + if [ -n "${comment:-}" ]; then cat > /tmp/run/pro-comment.json; else cat > /dev/null; fi + echo "201" + exit 0 + fi echo "[]" STUB chmod +x /tmp/stub/curl @@ -220,6 +229,26 @@ jobs: echo "--- step summary ---"; cat "$GITHUB_STEP_SUMMARY" if [ "$rc" -ne 0 ]; then echo "FAIL: expected exit 0, got $rc" >&2; exit 1; fi + # The paid comment must carry at least what the free one does: the + # changes themselves, plus the gate state only Pro has. + if [ ! -f /tmp/run/pro-comment.json ]; then + echo "FAIL: the Pro comment body was never posted" >&2; exit 1 + fi + pro_body=$(jq -r '.body' < /tmp/run/pro-comment.json) + echo "--- Pro comment body ---" + echo "$pro_body" + for expected in \ + '1 API change in this pull request' \ + 'Review required' \ + '| severity | change | endpoint |' \ + 'api-removed-without-deprecation' \ + 'Review and approve the changes' + do + case "$pro_body" in + *"$expected"*) ;; + *) echo "FAIL: the Pro comment is missing: $expected" >&2; exit 1 ;; + esac + done if ! echo "$out" | grep -q "oasdiff: posted the Pro review comment."; then echo "FAIL: Pro review comment not posted" >&2; exit 1 fi diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index 3a9851a..2abc255 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -272,11 +272,20 @@ pro_comment () { "${api}/repos/${owner}/${repo}/issues/${pr_number}/comments?per_page=100" 2>/dev/null \ | jq -r --arg m "$marker" 'map(select(.body | contains($m))) | .[0].id // empty' 2>/dev/null) || existing_id="" + if [ "${change_count:-0}" -eq 1 ]; then + heading="1 API change in this pull request" + else + heading="${change_count:-0} API changes in this pull request" + fi body="${marker} -### 📋 [Review and approve the API changes](${review_url}) +### 📋 ${heading} ${status_line} +$(review_comment_changes "${changes_json:-}") + +[Review and approve the changes](${review_url}) + 🔒 The comparison is encrypted in CI before it's uploaded. The decryption key stays in this link's fragment (after the #), which browsers never send to a server, so oasdiff cannot read your specs. Posted automatically by the [oasdiff GitHub Action](https://www.oasdiff.com/docs/setup). The \`oasdiff\` status check passes once every change is approved; make it a required check to block merging." From 47792e086c646adf83869287401d4602f8de4d28 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 18:17:12 +0200 Subject: [PATCH 5/6] Name the change in each row and link its check id Five rows reading new-optional-request-property on the same endpoint told a reviewer nothing about which property was added. Each row now carries the change's own sentence, which names the property, followed by the check id linked to its catalog page, the shape the Pro review comment already uses. A pipe in the text is escaped so it cannot break the table. Co-Authored-By: Claude Opus 5 --- .github/workflows/test-breaking.yaml | 3 ++- breaking/entrypoint.sh | 4 +++- changelog/entrypoint.sh | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-breaking.yaml b/.github/workflows/test-breaking.yaml index ecdd7c4..67a4009 100644 --- a/.github/workflows/test-breaking.yaml +++ b/.github/workflows/test-breaking.yaml @@ -429,7 +429,8 @@ jobs: for expected in \ '7 breaking changes in this pull request' \ '| severity | change | endpoint |' \ - 'api-removed-without-deprecation' \ + 'API removed [' \ + 'https://www.oasdiff.com/checks/api-removed-without-deprecation' \ 'GET /pets5' \ '
2 more' \ 'GET /pets6' \ diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index 399265d..3b5c54b 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -52,7 +52,9 @@ review_comment_changes () { def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - def row: "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"; + def cell: (.text // "") | gsub("\n"; " ") | gsub("\\|"; "\\\\|"); + def row: "| " + (.level | severity) + " | " + cell + + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | `" + endpoint + "` |"; def header: "| severity | change | endpoint |", "| --- | --- | --- |"; sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all | ($all | length) as $total diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index 2abc255..81b5be9 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -73,7 +73,9 @@ review_comment_changes () { def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - def row: "| " + (.level | severity) + " | `" + .id + "` | `" + endpoint + "` |"; + def cell: (.text // "") | gsub("\n"; " ") | gsub("\\|"; "\\\\|"); + def row: "| " + (.level | severity) + " | " + cell + + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | `" + endpoint + "` |"; def header: "| severity | change | endpoint |", "| --- | --- | --- |"; sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all | ($all | length) as $total From 4f92fd3d46150cf833a424f60b54d331226ee8e0 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 18:53:25 +0200 Subject: [PATCH 6/6] Neutralize spec text before it reaches the PR comment Property names and paths come from the spec, which the author of a pull request controls. A name containing a backtick closed the code span the change text opens, so the rest of it rendered as markdown: a mention, a link or HTML, posted under the action's name. Every character markdown reads is now written as an entity, which GitHub prints literally, and the endpoint cell uses a code element instead of a backtick span. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/test-changelog.yaml | 11 ++++++++++- breaking/entrypoint.sh | 15 ++++++++++++--- changelog/entrypoint.sh | 15 ++++++++++++--- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-changelog.yaml b/.github/workflows/test-changelog.yaml index 676eca4..6f554ed 100644 --- a/.github/workflows/test-changelog.yaml +++ b/.github/workflows/test-changelog.yaml @@ -286,7 +286,7 @@ jobs: case "${fmt:-text}" in json) printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}' - printf ',{"id":"response-property-became-optional","text":"became optional","level":1,"operation":"POST","path":"/pets","section":"paths"}' + printf ',{"id":"response-property-became-optional","text":"added @octocat [l](https://e.example) ","level":1,"operation":"POST","path":"/pets","section":"paths"}' i=3 while [ "$i" -le 6 ]; do printf ',{"id":"request-property-added","text":"added","level":1,"operation":"PUT","path":"/pets%s","section":"paths"}' "$i" @@ -349,6 +349,15 @@ jobs: *) echo "FAIL: the comment is missing: $expected" >&2; exit 1 ;; esac done + # Spec text is attacker-controlled on a pull request: it must reach + # the comment inert, with no live mention, link or HTML. + case "$body" in + *'@octocat'*|*'[l]('*|*''*) echo "FAIL: spec text reached the comment as live markdown" >&2; exit 1 ;; + esac + case "$body" in + *'@octocat'*) ;; + *) echo "FAIL: the mention was dropped instead of neutralized" >&2; exit 1 ;; + esac # Severest first, so the row that decides whether the pull request # can merge is never the one that collapses. first_row=$(printf '%s' "$body" | grep -m1 '^| \(error\|warning\|info\) ') diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index 3b5c54b..983d491 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -52,9 +52,18 @@ review_comment_changes () { def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - def cell: (.text // "") | gsub("\n"; " ") | gsub("\\|"; "\\\\|"); - def row: "| " + (.level | severity) + " | " + cell - + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | `" + endpoint + "` |"; + # Property names and paths come from the spec, which the author of a + # pull request controls, so nothing in them may act as markdown: no + # mention, link, image, or break out of the table. Every character + # markdown reads becomes an entity, which GitHub prints literally. + def inert: + gsub("&"; "&") | gsub("<"; "<") | gsub(">"; ">") + | gsub("@"; "@") | gsub("\\["; "[") | gsub("\\]"; "]") + | gsub("`"; "`") | gsub("\\|"; "|") | gsub("\\*"; "*") + | gsub("_"; "_") | gsub("://"; "://") | gsub("www\\."; "www.") + | gsub("[\r\n]+"; " "); + def row: "| " + (.level | severity) + " | " + ((.text // "") | inert) + + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | " + (endpoint | inert) + " |"; def header: "| severity | change | endpoint |", "| --- | --- | --- |"; sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all | ($all | length) as $total diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index 81b5be9..fbdbb87 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -73,9 +73,18 @@ review_comment_changes () { def severity: if . == 3 then "error" elif . == 2 then "warning" else "info" end; def endpoint: if (.operation // "") == "" then (.section // "") else .operation + " " + .path end; - def cell: (.text // "") | gsub("\n"; " ") | gsub("\\|"; "\\\\|"); - def row: "| " + (.level | severity) + " | " + cell - + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | `" + endpoint + "` |"; + # Property names and paths come from the spec, which the author of a + # pull request controls, so nothing in them may act as markdown: no + # mention, link, image, or break out of the table. Every character + # markdown reads becomes an entity, which GitHub prints literally. + def inert: + gsub("&"; "&") | gsub("<"; "<") | gsub(">"; ">") + | gsub("@"; "@") | gsub("\\["; "[") | gsub("\\]"; "]") + | gsub("`"; "`") | gsub("\\|"; "|") | gsub("\\*"; "*") + | gsub("_"; "_") | gsub("://"; "://") | gsub("www\\."; "www.") + | gsub("[\r\n]+"; " "); + def row: "| " + (.level | severity) + " | " + ((.text // "") | inert) + + " [`" + .id + "`](https://www.oasdiff.com/checks/" + .id + ") | " + (endpoint | inert) + " |"; def header: "| severity | change | endpoint |", "| --- | --- | --- |"; sort_by([-.level, (.path // ""), (.operation // ""), .id]) as $all | ($all | length) as $total