diff --git a/.github/scripts/generate-test-summary.sh b/.github/scripts/generate-test-summary.sh index 19fc6b710d..cb0f840b0f 100755 --- a/.github/scripts/generate-test-summary.sh +++ b/.github/scripts/generate-test-summary.sh @@ -63,8 +63,25 @@ status_emoji() { } # --- Fetch job data --- +# `gh api --paginate` asks for 100 items per page unless the URL sets +# per_page, and GitHub answers that page of a large run's job list (e.g. once +# a job has been re-run) with HTTP 502; 50 per page works. A failed listing +# is also retried, since GitHub's 5xx can be transient. +JOBS_PER_PAGE=50 +FETCH_ATTEMPTS=3 +FETCH_RETRY_DELAY="${SUMMARY_FETCH_RETRY_DELAY:-10}" log "Fetching job data for run $RUN_ID..." -jobs_json=$(gh api "/repos/$REPO/actions/runs/$RUN_ID/jobs" --paginate -q '.jobs') +for attempt in $(seq 1 "$FETCH_ATTEMPTS"); do + if jobs_json=$(gh api "/repos/$REPO/actions/runs/$RUN_ID/jobs?per_page=$JOBS_PER_PAGE" --paginate -q '.jobs'); then + break + fi + if [ "$attempt" -eq "$FETCH_ATTEMPTS" ]; then + log "Could not list the jobs of run $RUN_ID after $FETCH_ATTEMPTS attempts" + exit 1 + fi + log "Listing jobs failed (attempt $attempt of $FETCH_ATTEMPTS); retrying in ${FETCH_RETRY_DELAY}s" + sleep "$FETCH_RETRY_DELAY" +done # --- Parse test jobs --- log "Parsing test job statuses..." diff --git a/.github/scripts/tests/test_generate_test_summary.sh b/.github/scripts/tests/test_generate_test_summary.sh index e424dbf100..3c05b77f77 100755 --- a/.github/scripts/tests/test_generate_test_summary.sh +++ b/.github/scripts/tests/test_generate_test_summary.sh @@ -33,12 +33,22 @@ pass() { } # A stub `gh` good enough for this script's two call sites: the jobs listing -# and the ci-outcome artifact download. Real GitHub is never reached. +# and the ci-outcome artifact download. Real GitHub is never reached. Like +# GitHub, the listing fails with HTTP 502 unless it asks for fewer than 100 +# jobs per page (`gh api --paginate` asks for 100 by default), and its first +# $GH_API_FAILURES calls fail regardless, to exercise the retry. STUB_BIN="$TEMP_DIR/stub-bin" mkdir -p "$STUB_BIN" cat > "$STUB_BIN/gh" <<'EOS' #!/usr/bin/env bash if [ "$1" = "api" ]; then + calls=$(( $(cat "$GH_API_CALLS" 2>/dev/null || echo 0) + 1 )) + echo "$calls" > "$GH_API_CALLS" + per_page=$(echo "$2" | sed -n 's/.*[?&]per_page=\([0-9]*\).*/\1/p') + if [ -z "$per_page" ] || [ "$per_page" -ge 100 ] || [ "$calls" -le "${GH_API_FAILURES:-0}" ]; then + echo "gh: Server Error (HTTP 502)" >&2 + exit 1 + fi cat "$GH_JOBS_FIXTURE" exit 0 fi @@ -59,6 +69,7 @@ EOS chmod +x "$STUB_BIN/gh" PATH="$STUB_BIN:$PATH" export PATH +export GH_API_CALLS="$TEMP_DIR/gh-api-calls" SUMMARY_FETCH_RETRY_DELAY=0 write_jobs_fixture() { # write_jobs_fixture @@ -146,6 +157,37 @@ echo "$summary" | grep -q "All 1 test jobs passed" \ || fail "expected the all-passed banner, got: $summary" pass "an all-passing run prints no Failed Jobs section" +# GitHub's 5xx on the job listing can be transient: two failed attempts are +# retried, and the third succeeds. +CASE="$TEMP_DIR/case-listing-retried" +mkdir -p "$CASE/jobs" "$CASE/outcomes" "$CASE/work" +write_jobs_fixture "$CASE/jobs/jobs.json" success +rm -f "$GH_API_CALLS" +( + cd "$CASE/work" + export GH_JOBS_FIXTURE="$CASE/jobs/jobs.json" GH_OUTCOME_FIXTURE_DIR="$CASE/outcomes" GH_API_FAILURES=2 + export GITHUB_REPOSITORY="DataDog/java-profiler" GITHUB_SHA="deadbeefcafef00dfeedfacebeefcafebeefcafe" + "$SCRIPT" 12345 "$CASE/work/summary.md" +) || fail "generate-test-summary.sh gave up on a job listing that succeeded on its third attempt" +grep -q "All 1 test jobs passed" "$CASE/work/summary.md" \ + || fail "expected the all-passed banner after the retried listing, got: $(cat "$CASE/work/summary.md")" +pass "a job listing that fails twice is retried" + +# A listing that keeps failing still fails the script, as before the retry. +CASE="$TEMP_DIR/case-listing-fails" +mkdir -p "$CASE/jobs" "$CASE/outcomes" "$CASE/work" +write_jobs_fixture "$CASE/jobs/jobs.json" success +rm -f "$GH_API_CALLS" +if ( + cd "$CASE/work" + export GH_JOBS_FIXTURE="$CASE/jobs/jobs.json" GH_OUTCOME_FIXTURE_DIR="$CASE/outcomes" GH_API_FAILURES=3 + export GITHUB_REPOSITORY="DataDog/java-profiler" GITHUB_SHA="deadbeefcafef00dfeedfacebeefcafebeefcafe" + "$SCRIPT" 12345 "$CASE/work/summary.md" +) 2>/dev/null; then + fail "generate-test-summary.sh succeeded although every job listing attempt failed" +fi +pass "a job listing that keeps failing fails the script" + # The job-name regex and test_workflow.yml's `name:` expressions are a contract # with no enforcement: a non-matching job is skipped silently, so a rename # renders a summary reporting zero test jobs while the run stays green. Read