From b21a640d1af29114962494a11a5986032d6bd6df Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Sun, 15 Nov 2026 20:53:55 +0200 Subject: [PATCH] Let an approved Pro review satisfy fail-on The job's exit code came from fail-on alone, so a workflow that set fail-on and added an oasdiff-token stayed red after every breaking change was approved. Where that job is a required check, approval could never unblock the merge. With a token, fail-on of ERR or WARN is now satisfied once the review is approved. While it is pending or rejected the job fails as before, with a notice saying how to clear it, so a required job still blocks. INFO and MISSING-VERSION-BUMP are not cleared by approval, and nothing changes when no review could be posted. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/test-breaking.yaml | 77 ++++++++++++++++++++++++++++ README.md | 2 + breaking/entrypoint.sh | 30 ++++++++++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-breaking.yaml b/.github/workflows/test-breaking.yaml index fbbf8e8..7044199 100644 --- a/.github/workflows/test-breaking.yaml +++ b/.github/workflows/test-breaking.yaml @@ -675,3 +675,80 @@ jobs: echo "FAIL: review URL not in step summary" >&2; exit 1 fi echo "PASS" + + oasdiff_breaking_pro_approval_clears_fail_on: + runs-on: ubuntu-latest + name: Test an approved Pro review satisfies fail-on, and a pending one does not + # With a token, a breaking change is something a reviewer can approve. The + # job keeps failing on fail-on while the review is pending, so a job that is + # a required check still blocks the merge, and passes once every breaking + # change is approved. MISSING-VERSION-BUMP is a rule about the release and + # is not cleared by approval. Stubs oasdiff: the report run exits 1 when + # --fail-on is set, and --open reports the review status in STUB_STATUS. + steps: + - uses: actions/checkout@v7 + - name: Stub oasdiff + curl, run the three cases + run: | + set -euo pipefail + mkdir -p /tmp/stub /tmp/run + + cat > /tmp/stub/oasdiff <<'STUB' + #!/bin/sh + for a in "$@"; do + case "$a" in + json) printf '[{"id":"api-removed-without-deprecation","text":"API removed","level":3,"operation":"GET","path":"/pets","section":"paths"}]'; exit 0 ;; + githubactions) echo "::error file=specs/base.yaml,line=11::API removed"; exit 0 ;; + --open) + echo "oasdiff: opening review at https://www.oasdiff.com/review/ep/test-token-uuid#k=test-key" >&2 + echo "oasdiff: review status: ${STUB_STATUS}" >&2 + exit 0 ;; + --fail-on) failing=1 ;; + esac + done + echo "1 changes: 1 error, 0 warning, 0 info" + [ -n "${failing:-}" ] && exit 1 + exit 0 + STUB + chmod +x /tmp/stub/oasdiff + + cat > /tmp/stub/curl <<'STUB' + #!/bin/sh + for a in "$@"; do case "$a" in *"%{http_code}"*) cat > /dev/null; echo "201"; exit 0 ;; esac; done + echo "[]" + STUB + chmod +x /tmp/stub/curl + + export GITHUB_REF=refs/pull/123/merge + export GITHUB_REPOSITORY=acme/widgets + export GITHUB_SHA=deadbeef + export GITHUB_OUTPUT=/tmp/run/out + export GITHUB_STEP_SUMMARY=/tmp/run/summary + printf '{"pull_request":{"head":{"sha":"deadbeef"},"base":{"sha":"baadcafe"}}}' > /tmp/run/event.json + export GITHUB_EVENT_PATH=/tmp/run/event.json + export PATH=/tmp/stub:$PATH + + # run -> sets $rc and $out + run () { + : > "$GITHUB_OUTPUT"; : > "$GITHUB_STEP_SUMMARY" + set +e + out=$(env STUB_STATUS="$1" OASDIFF_ACTION_COMMAND=breaking INPUT_BASE=b INPUT_REVISION=r \ + "INPUT_FAIL-ON=$2" 'INPUT_GITHUB-TOKEN=gh-token' 'INPUT_OASDIFF-TOKEN=oasdiff-token' \ + bash ./breaking/entrypoint.sh 2>&1) + rc=$? + set -e + echo "--- review $1, fail-on $2: exit $rc"; echo "$out" | grep "oasdiff:" || true + } + + run pending WARN + if [ "$rc" -ne 1 ]; then echo "FAIL: a pending review must leave fail-on failing the job" >&2; exit 1; fi + case "$out" in + *"until every breaking change is approved"*) ;; + *) echo "FAIL: no notice saying how to clear the failure" >&2; exit 1 ;; + esac + + run approved WARN + if [ "$rc" -ne 0 ]; then echo "FAIL: an approved review must satisfy fail-on" >&2; exit 1; fi + + run approved MISSING-VERSION-BUMP + if [ "$rc" -ne 1 ]; then echo "FAIL: approval must not clear a missing version bump" >&2; exit 1; fi + echo "PASS" diff --git a/README.md b/README.md index 1e3c5d3..029ec1c 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,8 @@ jobs: The only difference from the free [breaking workflow](#check-for-breaking-changes) is the `oasdiff-token` secret and the `statuses: write` permission. Your specs are still encrypted in CI before upload, so the server can't read them. With `breaking`, the changes to approve are the breaking ones; use `changelog` with the same token to put every change up for review. +If your workflow sets `fail-on`, keep it. The job fails until every breaking change is approved, and passes on the next run after that, so a job that is a required check keeps blocking the merge. For a gate that clears the moment a change is approved, make the `oasdiff` status check the required one: with the GitHub App installed it updates without a new run. `fail-on: MISSING-VERSION-BUMP` is not cleared by approval. + Both actions are documented above; the one added input is: | Input | Default | Description | Accepted values | diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index 045f434..f92c6db 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -301,6 +301,7 @@ pro_review () { return 0 fi + pro_gate="$gate" echo "### 📋 [Review and approve the API changes](${review_url})" >> "$GITHUB_STEP_SUMMARY" pro_comment "$review_url" "$gate" set_commit_status "$owner" "$repo" "$head_sha" "$gate" "$review_url" @@ -421,6 +422,27 @@ pro_review_clear () { "${api}/repos/${owner}/${repo}/issues/comments/${existing_id}" --data-binary @- 2>/dev/null || true } +# finish exits the job. A Pro review makes a breaking change something a +# reviewer can approve, so once every breaking change is approved, a fail-on +# of ERR or WARN is satisfied and the job passes. Until then the job fails as +# it does without a token, so a job that is a required check keeps blocking +# the merge. INFO and MISSING-VERSION-BUMP are left alone: info changes are +# not part of the approval, and a missing version bump is a rule about the +# release, which approving the changes does not answer. +# $1: the exit code of the report run. +finish () { + code="$1" + if [ "$code" -eq 1 ] && [ -n "${pro_gate:-}" ]; then + if [ "$pro_gate" != "approved" ]; then + echo "::notice::oasdiff: fail-on fails this job until every breaking change is approved. Approve them in the review, then re-run the job." + elif [ "$fail_on" != "INFO" ] && [ "$fail_on" != "MISSING-VERSION-BUMP" ] && [ -z "${version_policy_violated:-}" ]; then + echo "oasdiff: every breaking change in this pull request is approved, so fail-on does not fail the job." + code=0 + fi + fi + exit "$code" +} + run_breaking () { echo "running oasdiff breaking... base: $base, revision: $revision, fail_on: $fail_on, include_checks: $include_checks, include_path_params: $include_path_params, deprecation_days_beta: $deprecation_days_beta, deprecation_days_stable: $deprecation_days_stable, exclude_elements: $exclude_elements, filter_extension: $filter_extension, composed: $composed, flatten_allof: $flatten_allof, err_ignore: $err_ignore, warn_ignore: $warn_ignore, output_to_file: $output_to_file" @@ -480,6 +502,8 @@ run_breaking () { _err=$(mktemp) # shellcheck disable=SC2086 # the flags variables hold multiple arguments; splitting is intended breaking_changes=$(oasdiff breaking "$base" "$revision" $flags $fail_on_flag 2>"$_err") || exit_code=$? + version_policy_violated="" + grep -q '^versioning policy: violated' "$_err" && version_policy_violated=1 [ -s "$_err" ] && cat "$_err" >&2 # Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is # success and exit 1 is the intended "breaking changes found" / fail-on result; @@ -606,7 +630,7 @@ run_breaking () { # are no breaking changes (the notice/URL only fire then). echo "review_url=${free_review_url:-${review_url:-}}" >> "$GITHUB_OUTPUT" - exit $exit_code + finish "$exit_code" } run_changelog () { @@ -682,6 +706,8 @@ run_changelog () { fi # shellcheck disable=SC2086 # the flags variables hold multiple arguments; splitting is intended output=$(oasdiff changelog "$base" "$revision" $flags $fmt_flags $fail_on_flag 2>"$_err") || exit_code=$? + version_policy_violated="" + grep -q '^versioning policy: violated' "$_err" && version_policy_violated=1 # Only codes >=2 are real errors. Exit 1 is a fail-on result (the changelog # action has no fail-on input, but .oasdiff.yaml can set one); the changelog # action doesn't gate, so don't let it abort the run or suppress the review below. @@ -800,7 +826,7 @@ run_changelog () { echo "review_url=${free_review_url:-${review_url:-}}" >> "$GITHUB_OUTPUT" # 1 when fail-on was reached, from the input or from .oasdiff.yaml. - exit "$exit_code" + finish "$exit_code" } case "$oasdiff_command" in