From 66b5f3f1dea25067a671c7bb5f4a8f137026c0b0 Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Fri, 13 Nov 2026 12:38:11 +0000 Subject: [PATCH] Use Componentized Custodian bot Git commits and comments will appear as the custodian bot. Signed-off-by: Scott Andrews --- .github/scripts/create-commit.sh | 74 +++++++++++++++++++++++ .github/workflows/preview-cleanup.yaml | 22 +++++-- .github/workflows/preview.yaml | 23 ++++++-- .github/workflows/scan.yaml | 82 ++++++++++++++++---------- src/http.rs | 19 +++++- src/scan.rs | 14 ++++- 6 files changed, 187 insertions(+), 47 deletions(-) create mode 100755 .github/scripts/create-commit.sh diff --git a/.github/scripts/create-commit.sh b/.github/scripts/create-commit.sh new file mode 100755 index 0000000..c5bb681 --- /dev/null +++ b/.github/scripts/create-commit.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Create a commit of the staged changes with the GitHub REST API, signed by GitHub, for a GitHub App +# token that can't push. +# +# GH_TOKEN= APP_SLUG= .github/scripts/create-commit.sh +# +# Shared with the other componentized repositories, e.g. componentized/static-config. +# +# Prints the commit, a child of , the commit the changes are staged against. No branch is moved +# to it. The changes are only staged locally to find the changed files and their modes. +# +# The commit is authored, committed and signed off (DCO) by the app's bot. GitHub only signs commits +# it attributes entirely to the app, so the author and committer are left for GitHub to fill in. The +# user who triggered the workflow is credited as a co-author, with their GitHub noreply email so their +# email address isn't exposed. A run started by the app has no one else to credit. +# +# GH_TOKEN a token for the app, with contents write permission, and workflows write +# permission to change workflow files +# APP_SLUG the slug of the app, e.g. custodian +# GITHUB_REPOSITORY the repository to create the commit in, e.g. componentized/http +# GITHUB_ACTOR the user who triggered the workflow, credited as a co-author +# GITHUB_ACTOR_ID the id of the user +# RUNNER_TEMP a directory for temporary files + +set -euo pipefail + +base="${1:?usage: $0 }" +subject="${2:?usage: $0 }" +: "${GH_TOKEN:?GH_TOKEN is undefined}" +: "${APP_SLUG:?APP_SLUG is undefined}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is undefined}" + +api="repos/${GITHUB_REPOSITORY}" +temp="${RUNNER_TEMP:-$(mktemp -d)}" + +# a blob for each changed file, or a null sha for a deleted file +entries="${temp}/tree-entries.json" +echo '[]' > "${entries}" +git diff --cached --no-renames --name-status "${base}" | while IFS=$'\t' read -r status path ; do + if [ "${status}" = "D" ] ; then + entry=$( jq -n --arg path "${path}" '{path: $path, mode: "100644", type: "blob", sha: null}' ) + else + mode=$( git ls-files --stage -- "${path}" | cut -d' ' -f1 ) + sha=$( base64 < "${path}" | tr -d '\n' | jq -Rs '{encoding: "base64", content: .}' | gh api --method POST "${api}/git/blobs" --input - --jq .sha ) + entry=$( jq -n --arg path "${path}" --arg mode "${mode}" --arg sha "${sha}" '{path: $path, mode: $mode, type: "blob", sha: $sha}' ) + fi + jq --argjson entry "${entry}" '. + [$entry]' "${entries}" > "${entries}.tmp" && mv "${entries}.tmp" "${entries}" + echo "${status} ${path}" >&2 +done +tree=$( jq --arg base "$( git rev-parse "${base}^{tree}" )" '{base_tree: $base, tree: .}' "${entries}" | gh api --method POST "${api}/git/trees" --input - --jq .sha ) + +bot="${APP_SLUG}[bot]" +bot_email="$( gh api "users/${bot}" --jq .id )+${bot}@users.noreply.github.com" +message=$( printf '%s\n\nSigned-off-by: %s <%s>' "${subject}" "${bot}" "${bot_email}" ) +if [ -n "${GITHUB_ACTOR:-}" ] && [ "${GITHUB_ACTOR}" != "${bot}" ] ; then + name=$( gh api "users/${GITHUB_ACTOR}" --jq '.name // .login' ) + name="${name:-${GITHUB_ACTOR}}" + email="${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + message=$( printf '%s\nCo-authored-by: %s <%s>' "${message}" "${name}" "${email}" ) +fi +read -r commit verified < <( jq -n \ + --arg message "${message}" \ + --arg tree "${tree}" --arg parent "${base}" \ + '{message: $message, tree: $tree, parents: [$parent]}' \ + | gh api --method POST "${api}/git/commits" --input - --jq '"\(.sha) \(.verification.verified)"' ) +echo "created commit ${commit}" >&2 +# the branch requires signed commits, fail before pushing rather than after +if [ "${verified}" != "true" ] ; then + echo "::error::the commit '${commit}' was not created, or was not signed by GitHub" >&2 + exit 1 +fi + +echo "${commit}" diff --git a/.github/workflows/preview-cleanup.yaml b/.github/workflows/preview-cleanup.yaml index 0bde9b7..ba9ec84 100644 --- a/.github/workflows/preview-cleanup.yaml +++ b/.github/workflows/preview-cleanup.yaml @@ -6,7 +6,7 @@ name: Preview cleanup # Cloudflare secrets for PRs from forks too. It must never check out or run the # PR's code: it only uses the PR number from the event. # -# Needs the same secrets and variable as the Preview workflow. +# Needs the same secrets and variable as the Preview workflow, and comments as the custodian app. on: pull_request_target: @@ -18,9 +18,8 @@ permissions: {} jobs: cleanup: runs-on: ubuntu-latest - permissions: - # mark the preview comment as removed - pull-requests: write + # the custodian app marks the preview comment as removed + permissions: {} steps: - name: Delete the PR's deployments env: @@ -51,15 +50,26 @@ jobs: done echo "Deleted ${#ids[@]} deployments of $BRANCH" + - name: Create custodian app token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + # only this repository, with only the permission the comment needs + repositories: ${{ github.event.repository.name }} + permission-pull-requests: write - name: Update the preview comment env: - GH_TOKEN: ${{ github.token }} + # the custodian app comments, rather than github-actions + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BOT: ${{ steps.app-token.outputs.app-slug }}[bot] REPO: ${{ github.repository }} PR: ${{ github.event.number }} run: | marker='' id="$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ - --jq "map(select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)" + --jq "map(select(.user.login == \"$BOT\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)" if [[ -n "$id" ]]; then body="$(printf '%s\n**Preview:** removed now that this PR is closed.' "$marker")" gh api -X PATCH "repos/$REPO/issues/comments/$id" -f body="$body" > /dev/null diff --git a/.github/workflows/preview.yaml b/.github/workflows/preview.yaml index 4cc89a1..92b8842 100644 --- a/.github/workflows/preview.yaml +++ b/.github/workflows/preview.yaml @@ -13,6 +13,8 @@ name: Preview # Needs: # - secrets.CLOUDFLARE_API_TOKEN: an account token with Cloudflare Pages: Edit # - secrets.CLOUDFLARE_ACCOUNT_ID +# - secrets.CUSTODIAN_CLIENT_ID and secrets.CUSTODIAN_PRIVATE_KEY: the componentized custodian +# GitHub App, which comments the preview on the PR # - vars.CLOUDFLARE_PAGES_PROJECT: an existing Pages project, created once with # `wrangler pages project create --production-branch=main` @@ -36,10 +38,9 @@ jobs: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest permissions: - # download the site from the CI run + # download the site from the CI run, and read the PR; the custodian app comments actions: read - # comment the preview's URL on the PR - pull-requests: write + pull-requests: read steps: - uses: actions/download-artifact@v8 with: @@ -89,10 +90,22 @@ jobs: --commit-hash=${{ github.event.workflow_run.head_sha }} --commit-dirty=true + - name: Create custodian app token + id: app-token + if: steps.pr.outputs.number + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + # only this repository, with only the permission the comment needs + repositories: ${{ github.event.repository.name }} + permission-pull-requests: write - name: Comment the preview if: steps.pr.outputs.number env: - GH_TOKEN: ${{ github.token }} + # the custodian app comments, rather than github-actions + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BOT: ${{ steps.app-token.outputs.app-slug }}[bot] REPO: ${{ github.repository }} PR: ${{ steps.pr.outputs.number }} SHA: ${{ github.event.workflow_run.head_sha }} @@ -103,7 +116,7 @@ jobs: body="$(printf '%s\n**Preview:** %s\n\nBuilt from %s ([this deploy](%s)).' \ "$marker" "${ALIAS_URL:-$DEPLOY_URL}" "$SHA" "$DEPLOY_URL")" id="$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ - --jq "map(select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)" + --jq "map(select(.user.login == \"$BOT\" and (.body | startswith(\"$marker\")))) | .[0].id // empty" | head -n1)" if [[ -n "$id" ]]; then gh api -X PATCH "repos/$REPO/issues/comments/$id" -f body="$body" > /dev/null else diff --git a/.github/workflows/scan.yaml b/.github/workflows/scan.yaml index f886b1a..a6ab1a7 100644 --- a/.github/workflows/scan.yaml +++ b/.github/workflows/scan.yaml @@ -11,6 +11,8 @@ on: types: - release +# commits the new versions as the componentized custodian GitHub App, from +# secrets.CUSTODIAN_CLIENT_ID and secrets.CUSTODIAN_PRIVATE_KEY permissions: {} concurrency: @@ -21,53 +23,71 @@ jobs: scan: runs-on: ubuntu-latest permissions: - # commit the new versions to the index - contents: write + # the index is committed by the custodian app, this job's token only reads + contents: read # discover the org's packages with the GitHub Packages API packages: read - outputs: - changed: ${{ steps.commit.outputs.changed }} - sha: ${{ steps.commit.outputs.sha }} steps: - uses: actions/checkout@v7 - uses: actions-rust-lang/setup-rust-toolchain@v2 - name: Scan id: scan + # without pipefail, the step would take tee's status and pass when the scan fails run: | + set -o pipefail cargo run --release --locked -- scan | tee "${RUNNER_TEMP}/added.txt" env: GITHUB_TOKEN: ${{ github.token }} - - name: Commit - id: commit + - name: Stage + id: stage run: | git add --all if git diff --cached --quiet ; then echo "no new versions" echo "changed=false" >> "${GITHUB_OUTPUT}" - exit 0 + else + echo "changed=true" >> "${GITHUB_OUTPUT}" fi + - name: Create custodian app token + id: app-token + if: steps.stage.outputs.changed == 'true' + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.CUSTODIAN_CLIENT_ID }} + private-key: ${{ secrets.CUSTODIAN_PRIVATE_KEY }} + # only this repository, with only the permission the commit needs + repositories: ${{ github.event.repository.name }} + permission-contents: write + # main requires signed commits, which the custodian app's commits created with the REST API + # are, and the app bypasses its required checks. A push by the app starts the Pages workflow, + # which deploys the new index. + - name: Commit + if: steps.stage.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + # a run started by hand credits whoever started it as a co-author; a scheduled run, or one + # a release asked for, credits no one + GITHUB_ACTOR: ${{ github.event_name == 'workflow_dispatch' && github.actor || '' }} + run: | added=$( grep -v '^::' "${RUNNER_TEMP}/added.txt" || true ) count=$( printf '%s\n' "${added}" | grep -c . || true ) - { - echo "Index ${count} new version(s)" - echo - printf '%s\n' "${added}" | sed 's/^/- /' - } > "${RUNNER_TEMP}/message.txt" - git -c user.name="github-actions[bot]" \ - -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ - commit --quiet --file "${RUNNER_TEMP}/message.txt" - git push - echo "changed=true" >> "${GITHUB_OUTPUT}" - echo "sha=$( git rev-parse HEAD )" >> "${GITHUB_OUTPUT}" - - # a push with the GITHUB_TOKEN does not start other workflows, deploy the new index from here - deploy: - needs: scan - if: needs.scan.outputs.changed == 'true' - permissions: - contents: read - pages: write - id-token: write - uses: ./.github/workflows/pages.yaml - with: - ref: ${{ needs.scan.outputs.sha }} + if [ "${count}" -gt 0 ] ; then + { + echo "Index ${count} new version(s)" + echo + printf '%s\n' "${added}" | sed 's/^/- /' + } > "${RUNNER_TEMP}/message.txt" + else + # no new versions, but those indexed changed, e.g. linked to their GitHub releases + echo "Update the indexed versions" > "${RUNNER_TEMP}/message.txt" + fi + base=$( git rev-parse HEAD ) + commit=$( .github/scripts/create-commit.sh "${base}" "$( cat "${RUNNER_TEMP}/message.txt" )" ) + # fast forwards main to the commit, failing rather than losing commits pushed since the + # scan started + if ! gh api --method PATCH "repos/${GITHUB_REPOSITORY}/git/refs/heads/${GITHUB_REF_NAME}" -f sha="${commit}" -F force=false --silent ; then + echo "::error::failed to push ${commit} to ${GITHUB_REF_NAME}, either the branch moved since ${base} and the scan should be run again, or the custodian app is not allowed to bypass the branch's ruleset" + exit 1 + fi + echo "pushed ${commit} to ${GITHUB_REF_NAME}" diff --git a/src/http.rs b/src/http.rs index 33a05d2..f63fafe 100644 --- a/src/http.rs +++ b/src/http.rs @@ -291,6 +291,17 @@ pub fn oci_location(metadata: serde_json::Value) -> Result { }) } +/// The explanation GitHub gives with an error, e.g. `: You need at least read:packages scope`, or +/// nothing when it gives none. +fn github_message(response: Response) -> String { + response + .json::() + .ok() + .and_then(|body| body.get("message")?.as_str().map(str::to_string)) + .map(|message| format!(": {message}")) + .unwrap_or_default() +} + /// The few GitHub API calls discovery needs. pub struct GitHub { client: Client, @@ -319,7 +330,9 @@ impl GitHub { | StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED | StatusCode::UNPROCESSABLE_ENTITY => Ok(None), - status if !status.is_success() => bail!("GET {url}: {status}"), + status if !status.is_success() => { + bail!("GET {url}: {status}{}", github_message(response)) + } _ => Ok(Some(response.json()?)), } } @@ -343,7 +356,9 @@ impl GitHub { StatusCode::NOT_FOUND | StatusCode::FORBIDDEN | StatusCode::UNAUTHORIZED => { return Ok(None); } - status if !status.is_success() => bail!("GET {url}: {status}"), + status if !status.is_success() => { + bail!("GET {url}: {status}{}", github_message(response)) + } _ => {} } let next = next_link(response.headers()); diff --git a/src/scan.rs b/src/scan.rs index afb7a45..01ef079 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -284,8 +284,10 @@ fn discover(github: &GitHub, org: &str) -> Result> { } let mut found = BTreeSet::new(); - match github.list::(&format!("/orgs/{org}/packages?package_type=container"))? { - Some(packages) => { + // the Packages API is optional: a token that can't read it, or an error reading it, falls back + // to discovering the packages from the repositories + match github.list::(&format!("/orgs/{org}/packages?package_type=container")) { + Ok(Some(packages)) => { eprintln!( "found {} packages with the GitHub Packages API", packages.len() @@ -297,9 +299,15 @@ fn discover(github: &GitHub, org: &str) -> Result> { .map(|p| p.name), ); } - None => eprintln!( + Ok(None) => eprintln!( "the GitHub Packages API is not readable, `packages: read`, discovering packages from the repositories" ), + Err(err) => { + println!( + "::warning::the GitHub Packages API failed, discovering packages from the repositories: {err:#}" + ); + eprintln!("the GitHub Packages API failed: {err:#}"); + } } let repos = github .list::(&format!("/orgs/{org}/repos?type=public"))?