Report build status via the GitHub Checks API using the halide-ci App

.github/workflows/validations.yml

@@ -17,6 +17,7 @@ jobs:
           openssl rand -hex 20 > secrets/buildbot_www_pass.txt
           openssl rand -hex 20 > secrets/db_password.txt
           echo "$GITHUB_TOKEN" > secrets/github_token.txt
+          openssl genrsa -out secrets/github_app_private_key.pem 2048
           openssl rand -hex 20 > secrets/halide_bb_pass.txt
           openssl rand -hex 20 > secrets/webhook_token.txt
           uv run --package master --python 3.12 buildbot checkconfig master

.gitignore

@@ -3,6 +3,7 @@
 venv/
 .venv/
 secrets/*.txt
+secrets/*.pem
 http.log
 twistd.hostname
 twistd.log

.pre-commit-config.yaml

@@ -10,17 +10,17 @@ repos:
     hooks:
       - id: ruff-check
         args: [--fix]
-        files: ^(master/(master\.cfg|custom_steps\.py|buildbot\.tac)|worker/buildbot\.tac)$
+        files: ^(master/(master\.cfg|custom_steps\.py|github_app_check_push\.py|buildbot\.tac)|worker/buildbot\.tac)$
         types_or: [python, text]
       - id: ruff-format
-        files: ^(master/(master\.cfg|custom_steps\.py|buildbot\.tac)|worker/buildbot\.tac)$
+        files: ^(master/(master\.cfg|custom_steps\.py|github_app_check_push\.py|buildbot\.tac)|worker/buildbot\.tac)$
         types_or: [python, text]
 
   - repo: https://github.com/PyCQA/bandit
     rev: 1.9.3
     hooks:
       - id: bandit
-        args: ["-c", "pyproject.toml", "master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"]
+        args: ["-c", "pyproject.toml", "master/master.cfg", "master/custom_steps.py", "master/github_app_check_push.py", "master/buildbot.tac", "worker/buildbot.tac"]
         pass_filenames: false
         always_run: true
         additional_dependencies: [ "bandit[toml]" ]
@@ -34,18 +34,18 @@ repos:
     rev: v2.14
     hooks:
       - id: vulture
-        args: ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"]
+        args: ["master/master.cfg", "master/custom_steps.py", "master/github_app_check_push.py", "master/buildbot.tac", "worker/buildbot.tac"]
         pass_filenames: false
         always_run: true
 
   - repo: local
     hooks:
       - id: ty-check
         name: ty
-        entry: uv run --package master ty check --error-on-warning master/master.cfg master/custom_steps.py
+        entry: uv run --package master ty check --error-on-warning master/master.cfg master/custom_steps.py master/github_app_check_push.py
         language: system
         pass_filenames: false
-        files: ^master/(master\.cfg|custom_steps\.py)$
+        files: ^master/(master\.cfg|custom_steps\.py|github_app_check_push\.py)$
 
       # caddy-bin ships the caddy binary via pip, so no Docker is required.
       # `caddy fmt --diff` prints the diff and exits 1 if the file isn't

docker-compose.yml

@@ -29,6 +29,7 @@ services:
     secrets:
       - db_password.txt
       - github_token.txt
+      - github_app_private_key.pem
       - halide_bb_pass.txt
       - webhook_token.txt
       - buildbot_www_pass.txt
@@ -70,6 +71,8 @@ secrets:
     file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/db_password.txt
   github_token.txt:
     file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/github_token.txt
+  github_app_private_key.pem:
+    file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/github_app_private_key.pem
   halide_bb_pass.txt:
     file: ${HALIDE_BB_MASTER_SECRETS_DIR:-./secrets}/halide_bb_pass.txt
   webhook_token.txt:

master/github_app_check_push.py

@@ -0,0 +1,109 @@
+import time
+
+import jwt
+import requests
+from buildbot.interfaces import IRenderable
+from buildbot.reporters.github import GitHubStatusPush
+from twisted.internet import defer, threads
+from zope.interface import implementer
+
+__all__ = ["AppInstallationToken", "GitHubAppCheckPush"]
+
+
+@implementer(IRenderable)
+class AppInstallationToken:
+    """Renders to a GitHub App installation access token, refreshed as needed. Pass an instance
+    as GitHubAppCheckPush's `token=`; buildbot re-renders it on every request, so refreshes
+    happen transparently.
+    """
+
+    def __init__(self, client_id, private_key, installation_id):
+        self._client_id = client_id
+        self._private_key = private_key
+        self._installation_id = installation_id
+        self._token = None
+        self._expires = 0
+        self._lock = defer.DeferredLock()
+
+    async def getRenderingFor(self, _iprops):
+        if time.time() > self._expires:
+            async with self._lock:
+                if time.time() > self._expires:
+                    self._token = await threads.deferToThread(self._fetch)
+                    self._expires = time.time() + 55 * 60
+        return self._token
+
+    def _fetch(self):
+        now = int(time.time())
+        app_jwt = jwt.encode(
+            {"iat": now - 60, "exp": now + 570, "iss": self._client_id},
+            self._private_key,
+            algorithm="RS256",
+        )
+        resp = requests.post(
+            f"https://api.github.com/app/installations/{self._installation_id}/access_tokens",
+            headers={"Authorization": f"Bearer {app_jwt}", "Accept": "application/vnd.github+json"},
+            timeout=10,
+        )
+        resp.raise_for_status()
+        return resp.json()["token"]
+
+
+class GitHubAppCheckPush(GitHubStatusPush):
+    """Like GitHubStatusPush, but reports through the Checks API instead of the legacy Statuses
+    API, so a build in progress shows GitHub's spinner instead of a static pending dot. Requires
+    a GitHub App: pass an AppInstallationToken as `token=` (the Statuses API's PAT-based token
+    doesn't work here; only the Checks API used by this class requires App auth).
+    """
+
+    @defer.inlineCallbacks
+    def _get_auth_header(self, props):
+        token = yield props.render(self.token)
+        return {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}
+
+    @defer.inlineCallbacks
+    def createStatus(
+        self, repo_user, repo_name, sha, state, props, target_url=None, context=None, issue=None, description=None
+    ):
+        headers = yield self._get_auth_header(props)
+        base = f"/repos/{repo_user}/{repo_name}/check-runs"
+        output = {"title": context, "summary": description or ""}
+
+        if state == "pending":
+            payload = {
+                "name": context,
+                "head_sha": sha,
+                "status": "in_progress",
+                "details_url": target_url,
+                "output": output,
+                "external_id": issue,
+            }
+            return (yield self._http.post(base, json=payload, headers=headers))
+
+        # The check run's id isn't threaded through from the "pending" call above, so look it up
+        # by name instead of tracking build-run state; one extra GET, but no persisted state.
+        resp = yield self._http.get(
+            f"/repos/{repo_user}/{repo_name}/commits/{sha}/check-runs",
+            params={"check_name": context},
+            headers=headers,
+        )
+        runs = (yield resp.json())["check_runs"]
+        if not runs:
+            return None
+
+        # GitHubStatusPush.sendMessage() already collapsed several build results into "error";
+        # both "failure" and "error" map to the same GitHub conclusion.
+        conclusion = "success" if state == "success" else "failure"
+        payload = {
+            "status": "completed",
+            "conclusion": conclusion,
+            "details_url": target_url,
+            "output": output,
+        }
+        # HTTPSession has no patch() wrapper (only get/put/post/delete); the Checks API update
+        # endpoint is PATCH-only, so fall through to the generic dispatcher it's built on.
+        return (
+            yield self._http.http._do_request(
+                self._http, "patch", f"{base}/{runs[0]['id']}", json=payload, headers=headers
+            )
+        )

master/master.cfg

@@ -18,7 +18,6 @@ from buildbot.config import BuilderConfig
 from buildbot.locks import WorkerLock
 from buildbot.process.factory import BuildFactory
 from buildbot.process.properties import Interpolate, Properties, Property, Transform, renderer
-from buildbot.reporters.github import GitHubStatusPush
 from buildbot.schedulers.basic import AnyBranchScheduler
 from buildbot.schedulers.canceller import OldBuildCanceller
 from buildbot.schedulers.forcesched import ForceScheduler
@@ -35,6 +34,7 @@ from buildbot.www.authz import Authz
 from buildbot.www.authz.roles import RolesFromUsername
 from buildbot.www.hooks.github import GitHubEventHandler
 from custom_steps import CTest
+from github_app_check_push import AppInstallationToken, GitHubAppCheckPush
 from twisted.internet.defer import inlineCallbacks
 from twisted.python import log
 
@@ -78,9 +78,15 @@ SECRETS_DIR = REPO_DIR / os.environ.get("HALIDE_BB_MASTER_SECRETS_DIR", "secrets
 
 BUILDBOT_WWW_PASS = (SECRETS_DIR / "buildbot_www_pass.txt").read_text().strip()
 GITHUB_TOKEN = (SECRETS_DIR / "github_token.txt").read_text().strip()
+GITHUB_APP_PRIVATE_KEY = (SECRETS_DIR / "github_app_private_key.pem").read_text().strip()
 HALIDE_BB_PASS = (SECRETS_DIR / "halide_bb_pass.txt").read_text().strip()
 WEBHOOK_TOKEN = (SECRETS_DIR / "webhook_token.txt").read_text().strip()
 
+# Not secret (the client ID and installation ID are public identifiers, visible in the app's
+# installation URL), but specific to the "halide-ci" GitHub App installation on the halide org.
+GITHUB_APP_CLIENT_ID = "Iv23licXRWCliqhA0UNM"
+GITHUB_APP_INSTALLATION_ID = 114824380
+
 DB_URL = os.environ.get("HALIDE_BB_MASTER_DB_URL", "sqlite:///state.sqlite")
 if "{DB_PASSWORD}" in DB_URL:
     DB_PASSWORD = (SECRETS_DIR / "db_password.txt").read_text().strip()
@@ -1352,8 +1358,12 @@ c["logCompressionMethod"] = "zstd"
 # GitHub Integration
 
 c["services"] = [
-    GitHubStatusPush(
-        token=GITHUB_TOKEN,
+    GitHubAppCheckPush(
+        token=AppInstallationToken(
+            client_id=GITHUB_APP_CLIENT_ID,
+            private_key=GITHUB_APP_PRIVATE_KEY,
+            installation_id=GITHUB_APP_INSTALLATION_ID,
+        ),
         verbose=True,
     ),
     OldBuildCanceller(

pyproject.toml

@@ -22,16 +22,28 @@ ignore = [
 extra-paths = ["master"]
 
 [tool.ty.src]
-include = ["master/custom_steps.py"]
+include = ["master/custom_steps.py", "master/github_app_check_push.py"]
 
 [tool.bandit]
-targets = ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"]
+targets = [
+    "master/master.cfg",
+    "master/custom_steps.py",
+    "master/github_app_check_push.py",
+    "master/buildbot.tac",
+    "worker/buildbot.tac",
+]
 skips = [
     "B101",  # assert_used: asserts are intentional for config validation
 ]
 
 [tool.vulture]
-paths = ["master/master.cfg", "master/custom_steps.py", "master/buildbot.tac", "worker/buildbot.tac"]
+paths = [
+    "master/master.cfg",
+    "master/custom_steps.py",
+    "master/github_app_check_push.py",
+    "master/buildbot.tac",
+    "worker/buildbot.tac",
+]
 min_confidence = 80
 
 [tool.codespell]