From c99b9eebc70471cceb7f8b20719426c1a9bc920a Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Fri, 4 Sep 2026 12:12:51 -0400 Subject: [PATCH 1/2] Report build status via the GitHub Checks API using the halide-ci App GitHubStatusPush only ever POSTs to the legacy Statuses API, which has no "in progress" state, so a running build shows a static pending dot instead of GitHub's spinner. Add GitHubAppCheckPush, a small subclass that reports through the Checks API instead, authenticating as the halide-ci GitHub App (the Checks API rejects plain PATs). AppInstallationToken mints and caches installation access tokens as a buildbot IRenderable, so it plugs into GitHubStatusPush's existing `token=` argument unchanged. --- .gitignore | 1 + .pre-commit-config.yaml | 12 ++-- docker-compose.yml | 3 + master/github_app_check_push.py | 109 ++++++++++++++++++++++++++++++++ master/master.cfg | 16 ++++- pyproject.toml | 18 +++++- 6 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 master/github_app_check_push.py diff --git a/.gitignore b/.gitignore index 57bc0eee..5f95edad 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ venv/ .venv/ secrets/*.txt +secrets/*.pem http.log twistd.hostname twistd.log diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93443d39..53c7c461 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,17 +10,17 @@ repos: hooks: - id: ruff-check args: [--fix] - files: ^(master/(master\.cfg|custom_steps\.py|buildbot\.tac)|worker/buildbot\.tac)$ + files: ^(master/(master\.cfg|custom_steps\.py|github_app_check_push\.py|buildbot\.tac)|worker/buildbot\.tac)$ types_or: [python, text] - id: ruff-format - files: ^(master/(master\.cfg|custom_steps\.py|buildbot\.tac)|worker/buildbot\.tac)$ + files: ^(master/(master\.cfg|custom_steps\.py|github_app_check_push\.py|buildbot\.tac)|worker/buildbot\.tac)$ types_or: [python, text] - repo: https://github.com/PyCQA/bandit rev: 1.9.3 hooks: - id: bandit - args: ["-c", "pyproject.toml", "master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"] + args: ["-c", "pyproject.toml", "master/master.cfg", "master/custom_steps.py", "master/github_app_check_push.py", "master/buildbot.tac", "worker/buildbot.tac"] pass_filenames: false always_run: true additional_dependencies: [ "bandit[toml]" ] @@ -34,7 +34,7 @@ repos: rev: v2.14 hooks: - id: vulture - args: ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"] + args: ["master/master.cfg", "master/custom_steps.py", "master/github_app_check_push.py", "master/buildbot.tac", "worker/buildbot.tac"] pass_filenames: false always_run: true @@ -42,10 +42,10 @@ repos: hooks: - id: ty-check name: ty - entry: uv run --package master ty check --error-on-warning master/master.cfg master/custom_steps.py + entry: uv run --package master ty check --error-on-warning master/master.cfg master/custom_steps.py master/github_app_check_push.py language: system pass_filenames: false - files: ^master/(master\.cfg|custom_steps\.py)$ + files: ^master/(master\.cfg|custom_steps\.py|github_app_check_push\.py)$ # caddy-bin ships the caddy binary via pip, so no Docker is required. # `caddy fmt --diff` prints the diff and exits 1 if the file isn't diff --git a/docker-compose.yml b/docker-compose.yml index 9ec63f7b..7443e71a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: secrets: - db_password.txt - github_token.txt + - github_app_private_key.pem - halide_bb_pass.txt - webhook_token.txt - buildbot_www_pass.txt @@ -70,6 +71,8 @@ secrets: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/db_password.txt github_token.txt: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/github_token.txt + github_app_private_key.pem: + file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/github_app_private_key.pem halide_bb_pass.txt: file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/halide_bb_pass.txt webhook_token.txt: diff --git a/master/github_app_check_push.py b/master/github_app_check_push.py new file mode 100644 index 00000000..bdfb9071 --- /dev/null +++ b/master/github_app_check_push.py @@ -0,0 +1,109 @@ +import time + +import jwt +import requests +from buildbot.interfaces import IRenderable +from buildbot.reporters.github import GitHubStatusPush +from twisted.internet import defer, threads +from zope.interface import implementer + +__all__ = ["AppInstallationToken", "GitHubAppCheckPush"] + + +@implementer(IRenderable) +class AppInstallationToken: + """Renders to a GitHub App installation access token, refreshed as needed. Pass an instance + as GitHubAppCheckPush's `token=`; buildbot re-renders it on every request, so refreshes + happen transparently. + """ + + def __init__(self, client_id, private_key, installation_id): + self._client_id = client_id + self._private_key = private_key + self._installation_id = installation_id + self._token = None + self._expires = 0 + self._lock = defer.DeferredLock() + + async def getRenderingFor(self, _iprops): + if time.time() > self._expires: + async with self._lock: + if time.time() > self._expires: + self._token = await threads.deferToThread(self._fetch) + self._expires = time.time() + 55 * 60 + return self._token + + def _fetch(self): + now = int(time.time()) + app_jwt = jwt.encode( + {"iat": now - 60, "exp": now + 570, "iss": self._client_id}, + self._private_key, + algorithm="RS256", + ) + resp = requests.post( + f"https://api.github.com/app/installations/{self._installation_id}/access_tokens", + headers={"Authorization": f"Bearer {app_jwt}", "Accept": "application/vnd.github+json"}, + timeout=10, + ) + resp.raise_for_status() + return resp.json()["token"] + + +class GitHubAppCheckPush(GitHubStatusPush): + """Like GitHubStatusPush, but reports through the Checks API instead of the legacy Statuses + API, so a build in progress shows GitHub's spinner instead of a static pending dot. Requires + a GitHub App: pass an AppInstallationToken as `token=` (the Statuses API's PAT-based token + doesn't work here; only the Checks API used by this class requires App auth). + """ + + @defer.inlineCallbacks + def _get_auth_header(self, props): + token = yield props.render(self.token) + return {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"} + + @defer.inlineCallbacks + def createStatus( + self, repo_user, repo_name, sha, state, props, target_url=None, context=None, issue=None, description=None + ): + headers = yield self._get_auth_header(props) + base = f"/repos/{repo_user}/{repo_name}/check-runs" + output = {"title": context, "summary": description or ""} + + if state == "pending": + payload = { + "name": context, + "head_sha": sha, + "status": "in_progress", + "details_url": target_url, + "output": output, + "external_id": issue, + } + return (yield self._http.post(base, json=payload, headers=headers)) + + # The check run's id isn't threaded through from the "pending" call above, so look it up + # by name instead of tracking build-run state; one extra GET, but no persisted state. + resp = yield self._http.get( + f"/repos/{repo_user}/{repo_name}/commits/{sha}/check-runs", + params={"check_name": context}, + headers=headers, + ) + runs = (yield resp.json())["check_runs"] + if not runs: + return None + + # GitHubStatusPush.sendMessage() already collapsed several build results into "error"; + # both "failure" and "error" map to the same GitHub conclusion. + conclusion = "success" if state == "success" else "failure" + payload = { + "status": "completed", + "conclusion": conclusion, + "details_url": target_url, + "output": output, + } + # HTTPSession has no patch() wrapper (only get/put/post/delete); the Checks API update + # endpoint is PATCH-only, so fall through to the generic dispatcher it's built on. + return ( + yield self._http.http._do_request( + self._http, "patch", f"{base}/{runs[0]['id']}", json=payload, headers=headers + ) + ) diff --git a/master/master.cfg b/master/master.cfg index 110a4836..19069b1b 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -18,7 +18,6 @@ from buildbot.config import BuilderConfig from buildbot.locks import WorkerLock from buildbot.process.factory import BuildFactory from buildbot.process.properties import Interpolate, Properties, Property, Transform, renderer -from buildbot.reporters.github import GitHubStatusPush from buildbot.schedulers.basic import AnyBranchScheduler from buildbot.schedulers.canceller import OldBuildCanceller from buildbot.schedulers.forcesched import ForceScheduler @@ -35,6 +34,7 @@ from buildbot.www.authz import Authz from buildbot.www.authz.roles import RolesFromUsername from buildbot.www.hooks.github import GitHubEventHandler from custom_steps import CTest +from github_app_check_push import AppInstallationToken, GitHubAppCheckPush from twisted.internet.defer import inlineCallbacks from twisted.python import log @@ -78,9 +78,15 @@ SECRETS_DIR = REPO_DIR / os.environ.get("HALIDE_BB_MASTER_SECRETS_DIR", "secrets BUILDBOT_WWW_PASS = (SECRETS_DIR / "buildbot_www_pass.txt").read_text().strip() GITHUB_TOKEN = (SECRETS_DIR / "github_token.txt").read_text().strip() +GITHUB_APP_PRIVATE_KEY = (SECRETS_DIR / "github_app_private_key.pem").read_text().strip() HALIDE_BB_PASS = (SECRETS_DIR / "halide_bb_pass.txt").read_text().strip() WEBHOOK_TOKEN = (SECRETS_DIR / "webhook_token.txt").read_text().strip() +# Not secret (the client ID and installation ID are public identifiers, visible in the app's +# installation URL), but specific to the "halide-ci" GitHub App installation on the halide org. +GITHUB_APP_CLIENT_ID = "Iv23licXRWCliqhA0UNM" +GITHUB_APP_INSTALLATION_ID = 114824380 + DB_URL = os.environ.get("HALIDE_BB_MASTER_DB_URL", "sqlite:///state.sqlite") if "{DB_PASSWORD}" in DB_URL: DB_PASSWORD = (SECRETS_DIR / "db_password.txt").read_text().strip() @@ -1352,8 +1358,12 @@ c["logCompressionMethod"] = "zstd" # GitHub Integration c["services"] = [ - GitHubStatusPush( - token=GITHUB_TOKEN, + GitHubAppCheckPush( + token=AppInstallationToken( + client_id=GITHUB_APP_CLIENT_ID, + private_key=GITHUB_APP_PRIVATE_KEY, + installation_id=GITHUB_APP_INSTALLATION_ID, + ), verbose=True, ), OldBuildCanceller( diff --git a/pyproject.toml b/pyproject.toml index 8305a0f2..46f87a5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,16 +22,28 @@ ignore = [ extra-paths = ["master"] [tool.ty.src] -include = ["master/custom_steps.py"] +include = ["master/custom_steps.py", "master/github_app_check_push.py"] [tool.bandit] -targets = ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"] +targets = [ + "master/master.cfg", + "master/custom_steps.py", + "master/github_app_check_push.py", + "master/buildbot.tac", + "worker/buildbot.tac", +] skips = [ "B101", # assert_used: asserts are intentional for config validation ] [tool.vulture] -paths = ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"] +paths = [ + "master/master.cfg", + "master/custom_steps.py", + "master/github_app_check_push.py", + "master/buildbot.tac", + "worker/buildbot.tac", +] min_confidence = 80 [tool.codespell] From edcefa78f7b19a7ebcd8c45a68b61223b22b9093 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Fri, 4 Sep 2026 16:29:02 -0400 Subject: [PATCH 2/2] Stub the App private key secret in CI's checkconfig job master.cfg now reads secrets/github_app_private_key.pem unconditionally, but the checkconfig job only stubbed the pre-existing secrets, so this PR's own CI run failed with FileNotFoundError. --- .github/workflows/validations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validations.yml b/.github/workflows/validations.yml index 9809de36..7d92f108 100644 --- a/.github/workflows/validations.yml +++ b/.github/workflows/validations.yml @@ -17,6 +17,7 @@ jobs: openssl rand -hex 20 > secrets/buildbot_www_pass.txt openssl rand -hex 20 > secrets/db_password.txt echo "$GITHUB_TOKEN" > secrets/github_token.txt + openssl genrsa -out secrets/github_app_private_key.pem 2048 openssl rand -hex 20 > secrets/halide_bb_pass.txt openssl rand -hex 20 > secrets/webhook_token.txt uv run --package master --python 3.12 buildbot checkconfig master