|
9 | 9 | # Run inside a clone of easybuild-easyconfigs whose `origin` is <owner/repo>. |
10 | 10 | # The base text of a changed file is read from `develop`. |
11 | 11 | set -euo pipefail |
| 12 | +# GitHub returns at most this many items on one page. |
| 13 | +page_size=100 |
| 14 | +# GitHub lists at most this many changed files for one pull request. |
| 15 | +file_cap=3000 |
| 16 | +# Pages of open pull requests read before the list is taken as unending. |
| 17 | +open_page_limit=1000 |
| 18 | +# Pages of changed files read before the list is taken as unending. |
| 19 | +file_page_limit=40 |
12 | 20 |
|
13 | 21 | repo=$1 |
14 | 22 | out=${2:?usage: fetch-prs.sh <owner/repo> <out-dir>} |
15 | 23 | # Rewrite outputs from scratch on every run, so closed pull requests do not |
16 | 24 | # persist from stale raw/ and prs/ directories. |
17 | | -rm -rf "$out/raw" "$out/prs" |
18 | | -mkdir -p "$out/prs" |
| 25 | +rm -rf "$out/raw" "$out/prs" "$out/trunc" |
| 26 | +mkdir -p "$out/prs" "$out/trunc" |
19 | 27 | rm -f "$out/refspecs" |
20 | 28 | prefix="easybuild/easyconfigs/" |
21 | 29 |
|
22 | 30 | git fetch --quiet --no-tags origin develop |
23 | | -# Every open pull request, 100 per page, with no upper limit. |
24 | | -gh api --method GET --paginate "repos/${repo}/pulls?state=open&per_page=100" \ |
25 | | - --jq '.[] | {number, headRefOid: .head.sha, title}' >"$out/open.jsonl" |
| 31 | +# Every open pull request, 100 per page. Each page is requested on its own, |
| 32 | +# and a short page is the end. There is no stop at 500. |
| 33 | +: >"$out/open.jsonl" |
| 34 | +page=1 |
| 35 | +while true; do |
| 36 | + if ! chunk=$(gh api --method GET \ |
| 37 | + "repos/${repo}/pulls?state=open&per_page=${page_size}&page=${page}" \ |
| 38 | + --jq '.[] | {number, headRefOid: .head.sha, title}' 2>"$out/.gh-err"); then |
| 39 | + cat "$out/.gh-err" >&2 |
| 40 | + echo "fetch-prs.sh: listing open pull requests failed on page ${page}" >&2 |
| 41 | + exit 1 |
| 42 | + fi |
| 43 | + if [[ -z "${chunk//[$' \t\n']/}" ]]; then |
| 44 | + break |
| 45 | + fi |
| 46 | + printf '%s\n' "$chunk" >>"$out/open.jsonl" |
| 47 | + count=0 |
| 48 | + while IFS= read -r line; do |
| 49 | + [[ -n "$line" ]] || continue |
| 50 | + count=$((count + 1)) |
| 51 | + done <<<"$chunk" |
| 52 | + if (( count < page_size )); then |
| 53 | + break |
| 54 | + fi |
| 55 | + page=$((page + 1)) |
| 56 | + if (( page > open_page_limit )); then |
| 57 | + echo "fetch-prs.sh: open pull request list did not end" >&2 |
| 58 | + exit 1 |
| 59 | + fi |
| 60 | +done |
| 61 | +rm -f "$out/.gh-err" |
26 | 62 | python3 - "$out/open.jsonl" "$out/open.json" <<'PY' |
27 | 63 | import json, sys |
28 | 64 | rows = [json.loads(line) for line in open(sys.argv[1]) if line.strip()] |
29 | 65 | json.dump(rows, open(sys.argv[2], "w")) |
30 | 66 | print("open pull requests:", len(rows)) |
31 | 67 | PY |
32 | 68 |
|
| 69 | +# Matching easyconfig paths for pull request $1, one per line. |
| 70 | +# Pages are 100 files. GitHub stops at 3000: a later page answers 422, or |
| 71 | +# another page is non-empty after 3000 names. The pull request is kept and |
| 72 | +# $out/trunc/$1 is created. Paths past that cap are not kept. |
| 73 | +pr_changed_files() { |
| 74 | + local number=$1 |
| 75 | + local page=1 |
| 76 | + local total=0 |
| 77 | + local err="$out/.gh-err" |
| 78 | + while true; do |
| 79 | + if ! chunk=$(gh api --method GET \ |
| 80 | + "repos/${repo}/pulls/${number}/files?per_page=${page_size}&page=${page}" \ |
| 81 | + --jq '.[] | .filename' 2>"$err"); then |
| 82 | + if (( page > 1 )) && grep -Eqi "${file_cap}|pagination is limited|maximum number of files|too many files" "$err"; then |
| 83 | + : >"$out/trunc/${number}" |
| 84 | + return 0 |
| 85 | + fi |
| 86 | + cat "$err" >&2 |
| 87 | + echo "fetch-prs.sh: file list for pull request ${number} failed on page ${page}" >&2 |
| 88 | + return 1 |
| 89 | + fi |
| 90 | + if [[ -z "${chunk//[$' \t\n']/}" ]]; then |
| 91 | + return 0 |
| 92 | + fi |
| 93 | + local count=0 |
| 94 | + local name |
| 95 | + while IFS= read -r name; do |
| 96 | + [[ -n "$name" ]] || continue |
| 97 | + count=$((count + 1)) |
| 98 | + case "$name" in |
| 99 | + "${prefix}"*.eb) printf '%s\n' "$name" ;; |
| 100 | + esac |
| 101 | + done <<<"$chunk" |
| 102 | + total=$((total + count)) |
| 103 | + if (( count < page_size )); then |
| 104 | + return 0 |
| 105 | + fi |
| 106 | + if (( total >= file_cap )); then |
| 107 | + page=$((page + 1)) |
| 108 | + if ! chunk=$(gh api --method GET \ |
| 109 | + "repos/${repo}/pulls/${number}/files?per_page=${page_size}&page=${page}" \ |
| 110 | + --jq '.[] | .filename' 2>"$err"); then |
| 111 | + if grep -Eqi "${file_cap}|pagination is limited|maximum number of files|too many files" "$err"; then |
| 112 | + : >"$out/trunc/${number}" |
| 113 | + return 0 |
| 114 | + fi |
| 115 | + cat "$err" >&2 |
| 116 | + echo "fetch-prs.sh: file list for pull request ${number} failed on page ${page}" >&2 |
| 117 | + return 1 |
| 118 | + fi |
| 119 | + if [[ -n "${chunk//[$' \t\n']/}" ]]; then |
| 120 | + : >"$out/trunc/${number}" |
| 121 | + fi |
| 122 | + return 0 |
| 123 | + fi |
| 124 | + page=$((page + 1)) |
| 125 | + if (( page > file_page_limit )); then |
| 126 | + echo "fetch-prs.sh: file list for pull request ${number} did not end" >&2 |
| 127 | + return 1 |
| 128 | + fi |
| 129 | + done |
| 130 | +} |
| 131 | + |
33 | 132 | python3 - "$out/open.json" <<'PY' | while read -r number sha; do |
34 | 133 | import json, sys |
35 | 134 | for pr in json.load(open(sys.argv[1])): |
36 | 135 | print(pr["number"], pr["headRefOid"]) |
37 | 136 | PY |
38 | 137 | # Changed files, from the API so no diff of fork history is needed. |
39 | | - mapfile -t files < <(gh api --paginate "repos/${repo}/pulls/${number}/files" \ |
40 | | - --jq ".[] | select(.filename | startswith(\"${prefix}\")) | select(.filename | endswith(\".eb\")) | .filename") |
41 | | - [[ ${#files[@]} -eq 0 ]] && continue |
42 | 138 | dir="$out/raw/${number}" |
43 | 139 | mkdir -p "$dir" |
| 140 | + if ! pr_changed_files "$number" >"$dir/names"; then |
| 141 | + exit 1 |
| 142 | + fi |
| 143 | + if [[ -f "$out/trunc/${number}" ]]; then |
| 144 | + mv "$out/trunc/${number}" "$dir/truncated" |
| 145 | + fi |
| 146 | + mapfile -t files <"$dir/names" |
| 147 | + if [[ ${#files[@]} -eq 0 && ! -f "$dir/truncated" ]]; then |
| 148 | + rm -rf "$dir" |
| 149 | + continue |
| 150 | + fi |
44 | 151 | i=0 |
45 | 152 | for f in "${files[@]}"; do |
46 | 153 | printf '%s\n' "$f" >"$dir/${i}.path" |
47 | 154 | i=$((i + 1)) |
48 | 155 | done |
49 | 156 | printf '%s\n' "$sha" >"$dir/head_sha" |
50 | 157 | # GitHub keeps every pull request head, fork or not, at refs/pull/N/head. |
51 | | - printf '+refs/pull/%s/head:refs/remotes/pr/%s\n' "$number" "$number" >>"$out/refspecs" |
| 158 | + if [[ ${#files[@]} -gt 0 ]]; then |
| 159 | + printf '+refs/pull/%s/head:refs/remotes/pr/%s\n' "$number" "$number" >>"$out/refspecs" |
| 160 | + fi |
52 | 161 | done |
53 | 162 |
|
54 | 163 | # One shallow fetch for every head. A shallow fetch per pull request |
@@ -101,6 +210,13 @@ for number in sorted(os.listdir(raw) if os.path.isdir(raw) else [], key=int): |
101 | 210 | i += 1 |
102 | 211 | doc = {"number": int(number), "head_sha": read(f"{d}/head_sha").strip(), |
103 | 212 | "title": titles.get(int(number)), "easyconfigs": easyconfigs} |
| 213 | + if os.path.exists(f"{d}/truncated"): |
| 214 | + doc["file_list_truncated"] = True |
104 | 215 | json.dump(doc, open(f"{out}/prs/{number}.json", "w"), indent=1, sort_keys=True) |
105 | 216 | PY |
106 | | -ls "$out/prs" | wc -l | xargs echo "pull requests with easyconfig changes:" |
| 217 | +rm -rf "$out/trunc" |
| 218 | +rm -f "$out/.gh-err" |
| 219 | +shopt -s nullglob |
| 220 | +prs=("$out"/prs/*.json) |
| 221 | +shopt -u nullglob |
| 222 | +echo "pull requests with easyconfig changes: ${#prs[@]}" |
0 commit comments