PYTHON-6132 Re-sync specs PR improvements

.evergreen/resync-specs.sh

@@ -85,28 +85,59 @@ cp_psl () {
       echo "Could not find the public suffix list at $src" >&2
       return 1
     fi
-    python - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF'
+    # Use the newest mongodbtoolchain python on CI hosts; python3 fallback locally.
+    PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1)
+    [ -n "$PY" ] || PY=python3
+    [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \
+        || { echo "Error: $PY is not Python 3.7+, which this conversion requires." >&2; return 1; }
+    "$PY" - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF'
+import os
 import sys
 
 src, dst = sys.argv[1], sys.argv[2]
-with open(src, encoding="utf-8") as f, open(dst, "w", encoding="utf-8") as out:
-    for line in f:
-        rule = line.strip()
-        if rule and not rule.startswith("//") and not rule.isascii():
-            # "!" and "*." prefixes are not valid IDNA, so encode only the
-            # domain part and re-attach the prefix afterwards.
-            prefix = ""
-            for p in ("!", "*."):
-                if rule.startswith(p):
-                    prefix, rule = p, rule[len(p) :]
-                    break
-            try:
-                rule = rule.encode("idna").decode("ascii")
-            except UnicodeError:
-                print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr)
-                sys.exit(1)
-            line = f"{prefix}{rule.lower()}\n"
-        out.write(line)
+
+# Convert in memory and write atomically so failures can't truncate the list.
+with open(src, encoding="utf-8") as f:
+    lines = f.readlines()
+if not lines:
+    print(f"Source public suffix list is empty: {src}", file=sys.stderr)
+    sys.exit(1)
+
+converted = []
+for line in lines:
+    rule = line.strip()
+    if rule and not rule.startswith("//") and not rule.isascii():
+        # "!" and "*." prefixes are not valid IDNA, so encode only the
+        # domain part and re-attach the prefix afterwards.
+        prefix = ""
+        for p in ("!", "*."):
+            if rule.startswith(p):
+                prefix, rule = p, rule[len(p) :]
+                break
+        try:
+            rule = rule.encode("idna").decode("ascii")
+        except UnicodeError:
+            print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr)
+            sys.exit(1)
+        line = f"{prefix}{rule.lower()}\n"
+    converted.append(line)
+
+if len(converted) != len(lines):
+    print(
+        f"Conversion lost lines: {len(lines)} in, {len(converted)} out",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+# A blank or comment-only source must not replace the real list.
+if not any(ln.strip() and not ln.strip().startswith("//") for ln in converted):
+    print(f"Source public suffix list has no rules: {src}", file=sys.stderr)
+    sys.exit(1)
+
+tmp = dst + ".tmp"
+with open(tmp, "w", encoding="utf-8") as out:
+    out.writelines(converted)
+os.replace(tmp, dst)
 EOF
 }
 

.evergreen/scripts/create-spec-pr.sh

@@ -32,12 +32,38 @@ echo "Creating the git checkout..."
 branch="spec-resync-"$(date '+%m-%d-%Y')
 
 git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git
-git checkout -b $branch "origin/main"
+# -B resets the branch if a reused checkout already has it, keeping reruns idempotent.
+git checkout -B "$branch" "origin/main"
+
+# Attribute the commit to the bot user; fall back to the host identity.
+bot_login="mongodb-drivers-pr-bot[bot]"
+bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true)
+if [ -n "$bot_id" ]; then
+    git config user.name "$bot_login"
+    git config user.email "${bot_id}+${bot_login}@users.noreply.github.com"
+fi
+
 git add ./test
 git commit -am "resyncing specs $(date '+%m-%d-%Y')"
 echo "Creating the git checkout... done."
 
-git push origin $branch
+# Date-stamped branch: force-push makes same-day reruns idempotent; retry
+# absorbs GitHub's intermittent 403s on the receive-pack endpoint.
+push_ok=false
+for _attempt in 1 2 3; do
+    if git push --force origin $branch; then
+        push_ok=true
+        break
+    fi
+    if [ "$_attempt" -lt 3 ]; then
+        echo "Push attempt ${_attempt} failed; retrying in 30s..."
+        sleep 30
+    fi
+done
+if [ "$push_ok" != true ]; then
+    echo "Failed to push $branch after 3 attempts!" >&2
+    exit 1
+fi
 
 # Build the payload as a file so the body content is always properly
 # JSON-escaped, rather than interpolated into the command line as text.
@@ -49,6 +75,31 @@ jq -n \
     --rawfile body "$1" \
     '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file"
 
+# Reuse an open PR for this branch (same-day rerun) instead of duplicating it.
+existing_pr_json=$(curl -gs \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $token" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open")
+existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty')
+if [ -n "$existing_pr_url" ]; then
+    existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty')
+    if ! curl -sfX PATCH \
+        -H "Accept: application/vnd.github+json" \
+        -H "Authorization: Bearer $token" \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \
+        --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null
+    then
+        echo "Failed to update PR summary!" >&2
+        exit 1
+    fi
+    echo "$existing_pr_url"
+    echo "Creating the PR... done. (PR already existed; branch and summary were updated)"
+    rm -rf $tools
+    exit 0
+fi
+
 resp=$(curl -L \
     -X POST \
     -H "Accept: application/vnd.github+json" \

.evergreen/scripts/resync-all-specs.py

@@ -28,6 +28,8 @@ def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None:
             )
         except CalledProcessError as exc:
             errored[spec.name] = exc.stderr
+            # The PR-body summary is lost if PR creation fails.
+            print(f"Spec sync failed for {spec.name}:\n{exc.stderr}", flush=True)
     print("Done syncing specs")
 
 
@@ -57,9 +59,11 @@ def apply_patches(errored):
                     ],
                     check=True,
                     stderr=subprocess.PIPE,
+                    text=True,
                 )
             except CalledProcessError as exc:
                 errored[f"{patch}"] = exc.stderr
+                print(f"Patch application failed for {patch}:\n{exc.stderr}", flush=True)
 
 
 def check_new_spec_directories(directory: pathlib.Path) -> list[str]:
@@ -127,9 +131,10 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None)
         if filename is None:
             print(f"\n{pr_body}")
         else:
+            # Plain text: the consumer embeds this file via jq --rawfile,
+            # which handles the JSON escaping itself.
             with open(filename, "w") as f:
-                # replacements made for proper json
-                f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t"))
+                f.write(pr_body)
 
 
 def main(args: Namespace):

.evergreen/scripts/resync-all-specs.sh

@@ -34,7 +34,12 @@ then
   # we're running locally
   python3 ./.evergreen/scripts/resync-all-specs.py
 else
-  /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC"
+  # Use the newest mongodbtoolchain python; the default python3 may be too old.
+  PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1)
+  [ -n "$PY" ] || PY=python3
+  [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \
+      || { echo "Error: $PY is not Python 3.7+, which resync-all-specs.py requires." >&2; exit 1; }
+  "$PY" ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC"
   if [[ -f $PR_DESC ]]; then
     # changes were made -> call scrypt to create PR for us
     .evergreen/scripts/create-spec-pr.sh "$PR_DESC"