diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 4feca0aa5b..840d05bf1e 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -85,28 +85,59 @@ cp_psl () { echo "Could not find the public suffix list at $src" >&2 return 1 fi - python - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' + # Use the newest mongodbtoolchain python on CI hosts; python3 fallback locally. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which this conversion requires." >&2; return 1; } + "$PY" - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' +import os import sys src, dst = sys.argv[1], sys.argv[2] -with open(src, encoding="utf-8") as f, open(dst, "w", encoding="utf-8") as out: - for line in f: - rule = line.strip() - if rule and not rule.startswith("//") and not rule.isascii(): - # "!" and "*." prefixes are not valid IDNA, so encode only the - # domain part and re-attach the prefix afterwards. - prefix = "" - for p in ("!", "*."): - if rule.startswith(p): - prefix, rule = p, rule[len(p) :] - break - try: - rule = rule.encode("idna").decode("ascii") - except UnicodeError: - print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) - sys.exit(1) - line = f"{prefix}{rule.lower()}\n" - out.write(line) + +# Convert in memory and write atomically so failures can't truncate the list. +with open(src, encoding="utf-8") as f: + lines = f.readlines() +if not lines: + print(f"Source public suffix list is empty: {src}", file=sys.stderr) + sys.exit(1) + +converted = [] +for line in lines: + rule = line.strip() + if rule and not rule.startswith("//") and not rule.isascii(): + # "!" and "*." prefixes are not valid IDNA, so encode only the + # domain part and re-attach the prefix afterwards. + prefix = "" + for p in ("!", "*."): + if rule.startswith(p): + prefix, rule = p, rule[len(p) :] + break + try: + rule = rule.encode("idna").decode("ascii") + except UnicodeError: + print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) + sys.exit(1) + line = f"{prefix}{rule.lower()}\n" + converted.append(line) + +if len(converted) != len(lines): + print( + f"Conversion lost lines: {len(lines)} in, {len(converted)} out", + file=sys.stderr, + ) + sys.exit(1) + +# A blank or comment-only source must not replace the real list. +if not any(ln.strip() and not ln.strip().startswith("//") for ln in converted): + print(f"Source public suffix list has no rules: {src}", file=sys.stderr) + sys.exit(1) + +tmp = dst + ".tmp" +with open(tmp, "w", encoding="utf-8") as out: + out.writelines(converted) +os.replace(tmp, dst) EOF } diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 0867ae5487..cffd8dc4de 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -32,12 +32,38 @@ echo "Creating the git checkout..." branch="spec-resync-"$(date '+%m-%d-%Y') git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git -git checkout -b $branch "origin/main" +# -B resets the branch if a reused checkout already has it, keeping reruns idempotent. +git checkout -B "$branch" "origin/main" + +# Attribute the commit to the bot user; fall back to the host identity. +bot_login="mongodb-drivers-pr-bot[bot]" +bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) +if [ -n "$bot_id" ]; then + git config user.name "$bot_login" + git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" +fi + git add ./test git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." -git push origin $branch +# Date-stamped branch: force-push makes same-day reruns idempotent; retry +# absorbs GitHub's intermittent 403s on the receive-pack endpoint. +push_ok=false +for _attempt in 1 2 3; do + if git push --force origin $branch; then + push_ok=true + break + fi + if [ "$_attempt" -lt 3 ]; then + echo "Push attempt ${_attempt} failed; retrying in 30s..." + sleep 30 + fi +done +if [ "$push_ok" != true ]; then + echo "Failed to push $branch after 3 attempts!" >&2 + exit 1 +fi # Build the payload as a file so the body content is always properly # JSON-escaped, rather than interpolated into the command line as text. @@ -49,6 +75,31 @@ jq -n \ --rawfile body "$1" \ '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" +# Reuse an open PR for this branch (same-day rerun) instead of duplicating it. +existing_pr_json=$(curl -gs \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open") +existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') +if [ -n "$existing_pr_url" ]; then + existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') + if ! curl -sfX PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \ + --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null + then + echo "Failed to update PR summary!" >&2 + exit 1 + fi + echo "$existing_pr_url" + echo "Creating the PR... done. (PR already existed; branch and summary were updated)" + rm -rf $tools + exit 0 +fi + resp=$(curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 78afe8a4d8..52c276b8b8 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -28,6 +28,8 @@ def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: ) except CalledProcessError as exc: errored[spec.name] = exc.stderr + # The PR-body summary is lost if PR creation fails. + print(f"Spec sync failed for {spec.name}:\n{exc.stderr}", flush=True) print("Done syncing specs") @@ -57,9 +59,11 @@ def apply_patches(errored): ], check=True, stderr=subprocess.PIPE, + text=True, ) except CalledProcessError as exc: errored[f"{patch}"] = exc.stderr + print(f"Patch application failed for {patch}:\n{exc.stderr}", flush=True) def check_new_spec_directories(directory: pathlib.Path) -> list[str]: @@ -127,9 +131,10 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None) if filename is None: print(f"\n{pr_body}") else: + # Plain text: the consumer embeds this file via jq --rawfile, + # which handles the JSON escaping itself. with open(filename, "w") as f: - # replacements made for proper json - f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) + f.write(pr_body) def main(args: Namespace): diff --git a/.evergreen/scripts/resync-all-specs.sh b/.evergreen/scripts/resync-all-specs.sh index 41e4a2bc73..11dd2e65ba 100755 --- a/.evergreen/scripts/resync-all-specs.sh +++ b/.evergreen/scripts/resync-all-specs.sh @@ -34,7 +34,12 @@ then # we're running locally python3 ./.evergreen/scripts/resync-all-specs.py else - /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" + # Use the newest mongodbtoolchain python; the default python3 may be too old. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which resync-all-specs.py requires." >&2; exit 1; } + "$PY" ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" if [[ -f $PR_DESC ]]; then # changes were made -> call scrypt to create PR for us .evergreen/scripts/create-spec-pr.sh "$PR_DESC"