Use Componentized Custodian bot

.github/scripts/create-commit.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Create a commit of the staged changes with the GitHub REST API, signed by GitHub, for a GitHub App
+# token that can't push.
+#
+#   GH_TOKEN=<token> APP_SLUG=<slug> .github/scripts/create-commit.sh <base> <message>
+#
+# Shared with the other componentized repositories, e.g. componentized/static-config.
+#
+# Prints the commit, a child of <base>, the commit the changes are staged against. No branch is moved
+# to it. The changes are only staged locally to find the changed files and their modes.
+#
+# The commit is authored, committed and signed off (DCO) by the app's bot. GitHub only signs commits
+# it attributes entirely to the app, so the author and committer are left for GitHub to fill in. The
+# user who triggered the workflow is credited as a co-author, with their GitHub noreply email so their
+# email address isn't exposed. A run started by the app has no one else to credit.
+#
+# GH_TOKEN           a token for the app, with contents write permission, and workflows write
+#                    permission to change workflow files
+# APP_SLUG           the slug of the app, e.g. custodian
+# GITHUB_REPOSITORY  the repository to create the commit in, e.g. componentized/http
+# GITHUB_ACTOR       the user who triggered the workflow, credited as a co-author
+# GITHUB_ACTOR_ID    the id of the user
+# RUNNER_TEMP        a directory for temporary files
+
+set -euo pipefail
+
+base="${1:?usage: $0 <base> <message>}"
+subject="${2:?usage: $0 <base> <message>}"
+: "${GH_TOKEN:?GH_TOKEN is undefined}"
+: "${APP_SLUG:?APP_SLUG is undefined}"
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is undefined}"
+
+api="repos/${GITHUB_REPOSITORY}"
+temp="${RUNNER_TEMP:-$(mktemp -d)}"
+
+# a blob for each changed file, or a null sha for a deleted file
+entries="${temp}/tree-entries.json"
+echo '[]' > "${entries}"
+git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do
+  if [ "${status}" = "D" ] ; then
+    entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' )
+  else
+    mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 )
+    sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha )
+    entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' )
+  fi
+  jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}"
+  echo "${status} ${path}" >&2
+done
+tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha )
+
+bot="${APP_SLUG}[bot]"
+bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com"
+message=$( printf '%s\n\nSigned-off-by: %s <%s>' "${subject}" "${bot}" "${bot_email}" )
+if [ -n "${GITHUB_ACTOR:-}" ] && [ "${GITHUB_ACTOR}" != "${bot}" ] ; then
+  name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' )
+  name="${name:-${GITHUB_ACTOR}}"
+  email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
+  message=$( printf '%s\nCo-authored-by: %s <%s>' "${message}" "${name}" "${email}" )
+fi
+read -r commit verified < <( jq -n \
+  --arg message "${message}" \
+  --arg tree "${tree}" --arg parent "${base}" \
+  '{message: $message, tree: $tree, parents: [$parent]}' \
+  | gh api --method POST "${api}/git/commits" --input - --jq '"\(.sha) \(.verification.verified)"' )
+echo "created commit ${commit}" >&2
+# the branch requires signed commits, fail before pushing rather than after
+if [ "${verified}" != "true" ] ; then
+  echo "::error::the commit '${commit}' was not created, or was not signed by GitHub" >&2
+  exit 1
+fi
+
+echo "${commit}"

.github/workflows/preview-cleanup.yaml

@@ -6,7 +6,7 @@ name: Preview cleanup
 # Cloudflare secrets for PRs from forks too. It must never check out or run the
 # PR's code: it only uses the PR number from the event.
 #
-# Needs the same secrets and variable as the Preview workflow.
+# Needs the same secrets and variable as the Preview workflow, and comments as the custodian app.
 
 on:
   pull_request_target:
@@ -18,9 +18,8 @@ permissions: {}
 jobs:
   cleanup:
     runs-on: ubuntu-latest
-    permissions:
-      # mark the preview comment as removed
-      pull-requests: write
+    # the custodian app marks the preview comment as removed
+    permissions: {}
     steps:
       - name: Delete the PR's deployments
         env:
@@ -51,15 +50,26 @@ jobs:
           done
           echo "Deleted ${#ids[@]} deployments of $BRANCH"
 
+      - name: Create custodian app token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }}
+          private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }}
+          # only this repository, with only the permission the comment needs
+          repositories: ${{ github.event.repository.name }}
+          permission-pull-requests: write
       - name: Update the preview comment
         env:
-          GH_TOKEN: ${{ github.token }}
+          # the custodian app comments, rather than github-actions
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          BOT: ${{ steps.app-token.outputs.app-slug }}[bot]
           REPO: ${{ github.repository }}
           PR: ${{ github.event.number }}
         run: |
           marker='<!-- preview -->'
           id="$(gh api --paginate "repos/$REPO/issues/$PR/comments" \
-            --jq "map(select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)"
+            --jq "map(select(.user.login == \"$BOT\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)"
           if [[ -n "$id" ]]; then
             body="$(printf '%s\n**Preview:** removed now that this PR is closed.' "$marker")"
             gh api -X PATCH "repos/$REPO/issues/comments/$id" -f body="$body" > /dev/null

.github/workflows/preview.yaml

@@ -13,6 +13,8 @@ name: Preview
 # Needs:
 # - secrets.CLOUDFLARE_API_TOKEN: an account token with Cloudflare Pages: Edit
 # - secrets.CLOUDFLARE_ACCOUNT_ID
+# - secrets.CUSTODIAN_CLIENT_ID and secrets.CUSTODIAN_PRIVATE_KEY: the componentized custodian
+#   GitHub App, which comments the preview on the PR
 # - vars.CLOUDFLARE_PAGES_PROJECT: an existing Pages project, created once with
 #   `wrangler pages project create <name> --production-branch=main`
 
@@ -36,10 +38,9 @@ jobs:
       github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     permissions:
-      # download the site from the CI run
+      # download the site from the CI run, and read the PR; the custodian app comments
       actions: read
-      # comment the preview's URL on the PR
-      pull-requests: write
+      pull-requests: read
     steps:
       - uses: actions/download-artifact@v8
         with:
@@ -89,10 +90,22 @@ jobs:
             --commit-hash=${{ github.event.workflow_run.head_sha }}
             --commit-dirty=true
 
+      - name: Create custodian app token
+        id: app-token
+        if: steps.pr.outputs.number
+        uses: actions/create-github-app-token@v3
+        with:
+          client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }}
+          private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }}
+          # only this repository, with only the permission the comment needs
+          repositories: ${{ github.event.repository.name }}
+          permission-pull-requests: write
       - name: Comment the preview
         if: steps.pr.outputs.number
         env:
-          GH_TOKEN: ${{ github.token }}
+          # the custodian app comments, rather than github-actions
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          BOT: ${{ steps.app-token.outputs.app-slug }}[bot]
           REPO: ${{ github.repository }}
           PR: ${{ steps.pr.outputs.number }}
           SHA: ${{ github.event.workflow_run.head_sha }}
@@ -103,7 +116,7 @@ jobs:
           body="$(printf '%s\n**Preview:** %s\n\nBuilt from %s ([this deploy](%s)).' \
             "$marker" "${ALIAS_URL:-$DEPLOY_URL}" "$SHA" "$DEPLOY_URL")"
           id="$(gh api --paginate "repos/$REPO/issues/$PR/comments" \
-            --jq "map(select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)"
+            --jq "map(select(.user.login == \"$BOT\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)"
           if [[ -n "$id" ]]; then
             gh api -X PATCH "repos/$REPO/issues/comments/$id" -f body="$body" > /dev/null
           else

.github/workflows/scan.yaml

@@ -11,6 +11,8 @@ on:
     types:
       - release
 
+# commits the new versions as the componentized custodian GitHub App, from
+# secrets.CUSTODIAN_CLIENT_ID and secrets.CUSTODIAN_PRIVATE_KEY
 permissions: {}
 
 concurrency:
@@ -21,53 +23,71 @@ jobs:
   scan:
     runs-on: ubuntu-latest
     permissions:
-      # commit the new versions to the index
-      contents: write
+      # the index is committed by the custodian app, this job's token only reads
+      contents: read
       # discover the org's packages with the GitHub Packages API
       packages: read
-    outputs:
-      changed: ${{ steps.commit.outputs.changed }}
-      sha: ${{ steps.commit.outputs.sha }}
     steps:
       - uses: actions/checkout@v7
       - uses: actions-rust-lang/setup-rust-toolchain@v2
       - name: Scan
         id: scan
+        # without pipefail, the step would take tee's status and pass when the scan fails
         run: |
+          set -o pipefail
           cargo run --release --locked -- scan | tee "${RUNNER_TEMP}/added.txt"
         env:
           GITHUB_TOKEN: ${{ github.token }}
-      - name: Commit
-        id: commit
+      - name: Stage
+        id: stage
         run: |
           git add --all
           if git diff --cached --quiet ; then
             echo "no new versions"
             echo "changed=false" >> "${GITHUB_OUTPUT}"
-            exit 0
+          else
+            echo "changed=true" >> "${GITHUB_OUTPUT}"
           fi
+      - name: Create custodian app token
+        id: app-token
+        if: steps.stage.outputs.changed == 'true'
+        uses: actions/create-github-app-token@v3
+        with:
+          client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }}
+          private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }}
+          # only this repository, with only the permission the commit needs
+          repositories: ${{ github.event.repository.name }}
+          permission-contents: write
+      # main requires signed commits, which the custodian app's commits created with the REST API
+      # are, and the app bypasses its required checks. A push by the app starts the Pages workflow,
+      # which deploys the new index.
+      - name: Commit
+        if: steps.stage.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
+          # a run started by hand credits whoever started it as a co-author; a scheduled run, or one
+          # a release asked for, credits no one
+          GITHUB_ACTOR: ${{ github.event_name == 'workflow_dispatch' && github.actor || '' }}
+        run: |
           added=$( grep -v '^::' "${RUNNER_TEMP}/added.txt" || true )
           count=$( printf '%s\n' "${added}" | grep -c . || true )
-          {
-            echo "Index ${count} new version(s)"
-            echo
-            printf '%s\n' "${added}" | sed 's/^/- /'
-          } > "${RUNNER_TEMP}/message.txt"
-          git -c user.name="github-actions[bot]" \
-              -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
-              commit --quiet --file "${RUNNER_TEMP}/message.txt"
-          git push
-          echo "changed=true" >> "${GITHUB_OUTPUT}"
-          echo "sha=$( git rev-parse HEAD )" >> "${GITHUB_OUTPUT}"
-
-  # a push with the GITHUB_TOKEN does not start other workflows, deploy the new index from here
-  deploy:
-    needs: scan
-    if: needs.scan.outputs.changed == 'true'
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-    uses: ./.github/workflows/pages.yaml
-    with:
-      ref: ${{ needs.scan.outputs.sha }}
+          if [ "${count}" -gt 0 ] ; then
+            {
+              echo "Index ${count} new version(s)"
+              echo
+              printf '%s\n' "${added}" | sed 's/^/- /'
+            } > "${RUNNER_TEMP}/message.txt"
+          else
+            # no new versions, but those indexed changed, e.g. linked to their GitHub releases
+            echo "Update the indexed versions" > "${RUNNER_TEMP}/message.txt"
+          fi
+          base=$( git rev-parse HEAD )
+          commit=$( .github/scripts/create-commit.sh "${base}" "$( cat "${RUNNER_TEMP}/message.txt" )" )
+          # fast forwards main to the commit, failing rather than losing commits pushed since the
+          # scan started
+          if ! gh api --method PATCH "repos/${GITHUB_REPOSITORY}/git/refs/heads/${GITHUB_REF_NAME}" -f sha="${commit}" -F force=false --silent ; then
+            echo "::error::failed to push ${commit} to ${GITHUB_REF_NAME}, either the branch moved since ${base} and the scan should be run again, or the custodian app is not allowed to bypass the branch's ruleset"
+            exit 1
+          fi
+          echo "pushed ${commit} to ${GITHUB_REF_NAME}"

src/http.rs

@@ -291,6 +291,17 @@ pub fn oci_location(metadata: serde_json::Value) -> Result<OciLocation> {
     })
 }
 
+/// The explanation GitHub gives with an error, e.g. `: You need at least read:packages scope`, or
+/// nothing when it gives none.
+fn github_message(response: Response) -> String {
+    response
+        .json::<serde_json::Value>()
+        .ok()
+        .and_then(|body| body.get("message")?.as_str().map(str::to_string))
+        .map(|message| format!(": {message}"))
+        .unwrap_or_default()
+}
+
 /// The few GitHub API calls discovery needs.
 pub struct GitHub {
     client: Client,
@@ -319,7 +330,9 @@ impl GitHub {
             | StatusCode::FORBIDDEN
             | StatusCode::UNAUTHORIZED
             | StatusCode::UNPROCESSABLE_ENTITY => Ok(None),
-            status if !status.is_success() => bail!("GET {url}: {status}"),
+            status if !status.is_success() => {
+                bail!("GET {url}: {status}{}", github_message(response))
+            }
             _ => Ok(Some(response.json()?)),
         }
     }
@@ -343,7 +356,9 @@ impl GitHub {
                 StatusCode::NOT_FOUND | StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED => {
                     return Ok(None);
                 }
-                status if !status.is_success() => bail!("GET {url}: {status}"),
+                status if !status.is_success() => {
+                    bail!("GET {url}: {status}{}", github_message(response))
+                }
                 _ => {}
             }
             let next = next_link(response.headers());

src/scan.rs

@@ -284,8 +284,10 @@ fn discover(github: &GitHub, org: &str) -> Result<BTreeSet<String>> {
     }
 
     let mut found = BTreeSet::new();
-    match github.list::<Package>(&format!("/orgs/{org}/packages?package_type=container"))? {
-        Some(packages) => {
+    // the Packages API is optional: a token that can't read it, or an error reading it, falls back
+    // to discovering the packages from the repositories
+    match github.list::<Package>(&format!("/orgs/{org}/packages?package_type=container")) {
+        Ok(Some(packages)) => {
             eprintln!(
                 "found {} packages with the GitHub Packages API",
                 packages.len()
@@ -297,9 +299,15 @@ fn discover(github: &GitHub, org: &str) -> Result<BTreeSet<String>> {
                     .map(|p| p.name),
             );
         }
-        None => eprintln!(
+        Ok(None) => eprintln!(
             "the GitHub Packages API is not readable, `packages: read`, discovering packages from the repositories"
         ),
+        Err(err) => {
+            println!(
+                "::warning::the GitHub Packages API failed, discovering packages from the repositories: {err:#}"
+            );
+            eprintln!("the GitHub Packages API failed: {err:#}");
+        }
     }
     let repos = github
         .list::<Repo>(&format!("/orgs/{org}/repos?type=public"))?