From 75498edb4b57fad07808340349a696dc8a51b29b Mon Sep 17 00:00:00 2001 From: Cheolmin Jo Date: Wed, 11 Nov 2026 13:16:36 +0900 Subject: [PATCH 1/3] ci: add the release packaging driver with unit-tested naming and checksum rules tools/ci_release.py derives tag, archive and checksum names (reproducing the four published releases), refuses anything but a full commit SHA and a clean tree, builds with the pinned published CLI and public SDK through ci_bootstrap, assembles the bootstrap resources and packs twice to require byte-identical archives. Co-Authored-By: Claude Sonnet 5.5 --- tools/ci_release.py | 175 +++++++++++++++++++++++++++++++++++++++ tools/test_ci_release.py | 76 +++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 tools/ci_release.py create mode 100644 tools/test_ci_release.py diff --git a/tools/ci_release.py b/tools/ci_release.py new file mode 100644 index 0000000..f57eb37 --- /dev/null +++ b/tools/ci_release.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Release CI driver for crt-cli; a build-host orchestrator, never a CLI runtime dependency. + + package pinned published CLI + pinned public SDK -> build the commit -> bootstrap resources -> deterministic archive + verify extract the archive on a fresh runner, check inventory, run the packaged executable and a real SDK acquisition + publish re-check the artifact digests, stage a draft GitHub release, verify it, publish it as a non-latest prerelease + +Naming and note rules are pure functions so they are unit-tested; publication itself is the only step with write +permission and it sits behind a protected environment. CI success is not publication authority. +""" +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(Path(__file__).resolve().parent)) + + +class ReleaseError(Exception): + pass + + +# host key -> (tag architecture, asset machine name, SHA256SUMS suffix); the tag carries the OS-architecture, the asset the machine name. +HOSTS = { + 'linux-x64': ('linux-x86_64', 'linux-x86_64', 'linux-x86_64'), + 'linux-arm64': ('linux-arm64', 'linux-aarch64', 'linux-aarch64'), + 'macos-arm64': ('macos-arm64', 'macos-aarch64', 'macos-aarch64'), + 'win-x64': ('windows-x86_64', 'windows-x86_64', 'windows-x86_64'), +} +PRODUCT_VERSION = '0.1.0-dev' +DATE = re.compile(r'^20\d{6}$') +SHA40 = re.compile(r'^[0-9a-f]{40}$') + + +def names(host, date, serial): + """Tag, archive and checksum-file names for one release, e.g. v0.1.0-preview.20261111.linux-arm64.1.""" + if host not in HOSTS: + raise ReleaseError('unsupported release host: %s' % host) + if not DATE.match(str(date)): + raise ReleaseError('date must look like 20261111') + if type(serial) is not int or serial < 1: + raise ReleaseError('serial must be a positive integer') + tag_arch, machine, sums = HOSTS[host] + base = '0.1.0-preview.%s' % date + return { + 'tag': 'v%s.%s.%d' % (base, tag_arch, serial), + 'archive': 'crt-cli-%s-%s-preview.%s.%d.tar.xz' % (PRODUCT_VERSION, machine, date, serial), + 'sums': 'SHA256SUMS-%s' % sums, + 'title': 'crt-cli Preview %s (%s)' % (date, {'linux-x64': 'Linux x86_64', 'linux-arm64': 'Linux aarch64', + 'macos-arm64': 'macOS arm64', 'win-x64': 'Windows x86_64'}[host]), + } + + +def sums_line(archive_name, digest): + if not re.fullmatch(r'[0-9a-f]{64}', digest): + raise ReleaseError('digest is not a lowercase SHA-256') + return '%s %s\n' % (digest, archive_name) + + +def parse_sums(text): + entries = {} + for line in text.splitlines(): + match = re.fullmatch(r'([0-9a-f]{64}) (\S+)', line) + if not match: + raise ReleaseError('malformed checksum line: %r' % line) + if match.group(2) in entries: + raise ReleaseError('duplicate checksum entry: %s' % match.group(2)) + entries[match.group(2)] = match.group(1) + if not entries: + raise ReleaseError('checksum file is empty') + return entries + + +def require_commit(sha): + if not SHA40.match(sha or ''): + raise ReleaseError('a release is built from a full lowercase 40-character commit SHA, never a branch or tag') + return sha + + +def require_artifact_match(archive_path, sums_text, archive_name): + """The bytes about to be uploaded must equal the digest the verify job saw.""" + entries = parse_sums(sums_text) + if set(entries) != {archive_name}: + raise ReleaseError('checksum file must list exactly %s, found %s' % (archive_name, sorted(entries))) + actual = hashlib.sha256(Path(archive_path).read_bytes()).hexdigest() + if actual != entries[archive_name]: + raise ReleaseError('archive digest %s differs from the verified digest %s' % (actual, entries[archive_name])) + return actual + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def run(label, argv, **kwargs): + log('[%s] %s' % (label, ' '.join(map(str, argv)))) + done = subprocess.run([str(a) for a in argv], **kwargs) + if done.returncode != 0: + raise ReleaseError('%s failed with exit %d' % (label, done.returncode)) + return done + + +def command_package(args): + """Build the exact commit with the pinned published CLI and the pinned public SDK, then pack deterministically. + + The heavy lifting is tools/ci_bootstrap.py (bootstrap, SDK acquisition, stage 1 and stage 2); this adds the resource + assembly and the archive, and refuses a tree that is not the requested commit. + """ + commit = require_commit(args.commit) + head = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], capture_output=True, text=True).stdout.strip() + if head != commit: + raise ReleaseError('checked-out commit %s is not the requested %s' % (head, commit)) + if subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain'], capture_output=True, text=True).stdout.strip(): + raise ReleaseError('the working tree is not clean; a release is built from a committed tree only') + n = names(args.host, args.date, args.serial) + work = Path(args.work).resolve() + work.mkdir(parents=True, exist_ok=False) + ci_work = work / 'ci' + argv = [sys.executable, ROOT / 'tools/ci_bootstrap.py', '--work', ci_work, '--host', args.host, '--native-test-jobs', '2'] + if args.llvm_bin: + argv += ['--llvm-bin', args.llvm_bin] + run('ci-bootstrap', argv) + summary = json.loads((ci_work / 'artifact/ci-summary.json').read_text(encoding='utf-8')) + if summary.get('result') != 'pass' or summary.get('source_commit') != commit: + raise ReleaseError('the CI driver did not pass for the requested commit') + exe = next(p for p in (ci_work / 'artifact').rglob('crt-cli') if p.is_file()) + sdk_root = next(p.parent for p in (ci_work).rglob('manifest.json') if p.parent.name != 'crt-cli' and (p.parent / 'share/licenses').is_dir()) + package = work / 'package' / 'crt-cli' + run('assemble', [sys.executable, ROOT / 'tools/create_bootstrap_resources.py', '--binary', exe, '--sdk-root', sdk_root, '--output', package]) + out = work / 'release' + out.mkdir() + archive = out / n['archive'] + run('pack', [sys.executable, ROOT / 'tools/package_cli_archive.py', 'pack', '--package', package, '--output', archive]) + twice = work / 'second.tar.xz' + run('pack-again', [sys.executable, ROOT / 'tools/package_cli_archive.py', 'pack', '--package', package, '--output', twice]) + if archive.read_bytes() != twice.read_bytes(): + raise ReleaseError('packing the same package twice gave different bytes') + twice.unlink() + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + (out / n['sums']).write_text(sums_line(n['archive'], digest), encoding='ascii') + (out / 'release.json').write_text(json.dumps({'commit': commit, 'host': args.host, 'names': n, 'archive_sha256': digest, + 'archive_bytes': archive.stat().st_size, 'executable_sha256': summary['stage2']['sha256'], + 'sdk': summary['sdk'], 'bootstrap': summary['bootstrap']}, indent=2) + '\n', encoding='utf-8') + log('packaged %s (%d bytes) sha256=%s' % (n['archive'], archive.stat().st_size, digest)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest='command', required=True) + for name in ('package',): + p = sub.add_parser(name) + p.add_argument('--host', choices=sorted(HOSTS), required=True) + p.add_argument('--commit', required=True) + p.add_argument('--date', required=True) + p.add_argument('--serial', type=int, required=True) + p.add_argument('--work', required=True) + p.add_argument('--llvm-bin') + args = parser.parse_args() + try: + if args.command == 'package': + command_package(args) + except ReleaseError as error: + log('RELEASE FAIL: %s' % error) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/test_ci_release.py b/tools/test_ci_release.py new file mode 100644 index 0000000..85a8b34 --- /dev/null +++ b/tools/test_ci_release.py @@ -0,0 +1,76 @@ +"""Release naming, checksum and commit rules: a mistake here would publish the wrong bytes under the wrong name.""" +import hashlib +import tempfile +import unittest +from pathlib import Path + +from ci_release import ReleaseError, names, parse_sums, require_artifact_match, require_commit, sums_line + + +class NamesTest(unittest.TestCase): + def test_reproduces_the_published_linux_aarch64_release(self): + n = names('linux-arm64', '20261111', 1) + self.assertEqual(n['tag'], 'v0.1.0-preview.20261111.linux-arm64.1') + self.assertEqual(n['archive'], 'crt-cli-0.1.0-dev-linux-aarch64-preview.20261111.1.tar.xz') + self.assertEqual(n['sums'], 'SHA256SUMS-linux-aarch64') + self.assertEqual(n['title'], 'crt-cli Preview 20261111 (Linux aarch64)') + + def test_reproduces_the_other_published_releases(self): + self.assertEqual(names('linux-x64', '20261110', 1)['tag'], 'v0.1.0-preview.20261110.linux-x86_64.1') + self.assertEqual(names('linux-x64', '20261110', 1)['archive'], 'crt-cli-0.1.0-dev-linux-x86_64-preview.20261110.1.tar.xz') + self.assertEqual(names('macos-arm64', '20261110', 1)['tag'], 'v0.1.0-preview.20261110.macos-arm64.1') + self.assertEqual(names('macos-arm64', '20261110', 1)['archive'], 'crt-cli-0.1.0-dev-macos-aarch64-preview.20261110.1.tar.xz') + self.assertEqual(names('win-x64', '20261110', 1)['tag'], 'v0.1.0-preview.20261110.windows-x86_64.1') + self.assertEqual(names('win-x64', '20261110', 1)['sums'], 'SHA256SUMS-windows-x86_64') + + def test_bad_inputs_are_refused(self): + for args in [('freebsd', '20261111', 1), ('linux-x64', '2026-11-11', 1), ('linux-x64', '20261111', 0), ('linux-x64', '20261111', '1')]: + with self.subTest(args=args), self.assertRaises(ReleaseError): + names(*args) + + +class CommitTest(unittest.TestCase): + def test_only_a_full_lowercase_sha_is_a_commit(self): + self.assertEqual(require_commit('6a008856d4a92fafbea1221b0f6c88a4559ccc29'), '6a008856d4a92fafbea1221b0f6c88a4559ccc29') + for bad in ['main', 'v0.1.0', '6a00885', '6A008856D4A92FAFBEA1221B0F6C88A4559CCC29', '', None, '6a008856d4a92fafbea1221b0f6c88a4559ccc2']: + with self.subTest(bad=bad), self.assertRaises(ReleaseError): + require_commit(bad) + + +class ChecksumTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + self.archive = self.dir / 'a.tar.xz' + self.archive.write_bytes(b'payload') + self.digest = hashlib.sha256(b'payload').hexdigest() + + def tearDown(self): + self.tmp.cleanup() + + def test_matching_archive_passes(self): + self.assertEqual(require_artifact_match(self.archive, sums_line('a.tar.xz', self.digest), 'a.tar.xz'), self.digest) + + def test_changed_bytes_are_refused(self): + self.archive.write_bytes(b'payloaX') + with self.assertRaisesRegex(ReleaseError, 'differs'): + require_artifact_match(self.archive, sums_line('a.tar.xz', self.digest), 'a.tar.xz') + + def test_wrong_name_extra_entry_malformed_and_empty_are_refused(self): + with self.assertRaisesRegex(ReleaseError, 'exactly'): + require_artifact_match(self.archive, sums_line('b.tar.xz', self.digest), 'a.tar.xz') + two = sums_line('a.tar.xz', self.digest) + sums_line('b.tar.xz', self.digest) + with self.assertRaisesRegex(ReleaseError, 'exactly'): + require_artifact_match(self.archive, two, 'a.tar.xz') + with self.assertRaisesRegex(ReleaseError, 'malformed'): + parse_sums('not a checksum line\n') + with self.assertRaisesRegex(ReleaseError, 'empty'): + parse_sums('') + with self.assertRaisesRegex(ReleaseError, 'duplicate'): + parse_sums(sums_line('a.tar.xz', self.digest) * 2) + with self.assertRaisesRegex(ReleaseError, 'lowercase'): + sums_line('a.tar.xz', self.digest.upper()) + + +if __name__ == '__main__': + unittest.main() From 0331904a34dfbc9aeccea6d9ef41ab3cbcc659b7 Mon Sep 17 00:00:00 2001 From: Cheolmin Jo Date: Wed, 11 Nov 2026 13:21:58 +0900 Subject: [PATCH 2/3] ci: locate the release executable by digest and the SDK by its recorded identity The first local run stopped with a bare StopIteration because the artifact is named crt-cli-. The executable is now the artifact whose bytes equal the verified stage-2 digest, and the SDK is the directory named by the summary's version and content hash; zero or ambiguous matches are controlled errors with unit tests. Co-Authored-By: Claude Sonnet 5.5 --- tools/ci_release.py | 22 ++++++++++++++++++++-- tools/test_ci_release.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/tools/ci_release.py b/tools/ci_release.py index f57eb37..5856d84 100644 --- a/tools/ci_release.py +++ b/tools/ci_release.py @@ -94,6 +94,24 @@ def require_artifact_match(archive_path, sums_text, archive_name): return actual +def locate_executable(artifact_dir, expected_sha256): + """The artifact file whose bytes are the verified stage-2 executable; names are never guessed.""" + matches = [p for p in Path(artifact_dir).iterdir() + if p.is_file() and not p.name.startswith('SHA256SUMS') and p.suffix != '.json' + and hashlib.sha256(p.read_bytes()).hexdigest() == expected_sha256] + if len(matches) != 1: + raise ReleaseError('expected exactly one artifact with the verified executable digest, found %d' % len(matches)) + return matches[0] + + +def locate_sdk(store, sdk): + """The installed SDK directory named by the summary's version and content identity.""" + directory = Path(store) / ('%s-%s' % (sdk['version'], sdk['content_sha256'][:12])) + if not (directory / 'manifest.json').is_file() or not (directory / 'share/licenses').is_dir(): + raise ReleaseError('the installed SDK %s is not where the summary says' % directory.name) + return directory + + def log(message): print(message, file=sys.stderr, flush=True) @@ -129,8 +147,8 @@ def command_package(args): summary = json.loads((ci_work / 'artifact/ci-summary.json').read_text(encoding='utf-8')) if summary.get('result') != 'pass' or summary.get('source_commit') != commit: raise ReleaseError('the CI driver did not pass for the requested commit') - exe = next(p for p in (ci_work / 'artifact').rglob('crt-cli') if p.is_file()) - sdk_root = next(p.parent for p in (ci_work).rglob('manifest.json') if p.parent.name != 'crt-cli' and (p.parent / 'share/licenses').is_dir()) + exe = locate_executable(ci_work / 'artifact', summary['stage2']['sha256']) + sdk_root = locate_sdk(ci_work / 'home' / '.crt' / 'sdks', summary['sdk']) package = work / 'package' / 'crt-cli' run('assemble', [sys.executable, ROOT / 'tools/create_bootstrap_resources.py', '--binary', exe, '--sdk-root', sdk_root, '--output', package]) out = work / 'release' diff --git a/tools/test_ci_release.py b/tools/test_ci_release.py index 85a8b34..9b038dc 100644 --- a/tools/test_ci_release.py +++ b/tools/test_ci_release.py @@ -4,7 +4,8 @@ import unittest from pathlib import Path -from ci_release import ReleaseError, names, parse_sums, require_artifact_match, require_commit, sums_line +from ci_release import (ReleaseError, locate_executable, locate_sdk, names, parse_sums, require_artifact_match, require_commit, + sums_line) class NamesTest(unittest.TestCase): @@ -72,5 +73,37 @@ def test_wrong_name_extra_entry_malformed_and_empty_are_refused(self): sums_line('a.tar.xz', self.digest.upper()) +class LocateTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_executable_is_found_by_digest_not_by_name(self): + (self.dir / 'crt-cli-host').write_bytes(b'binary') + (self.dir / 'other').write_bytes(b'something else') + (self.dir / 'SHA256SUMS-host').write_text('x') + (self.dir / 'ci-summary.json').write_text('{}') + self.assertEqual(locate_executable(self.dir, hashlib.sha256(b'binary').hexdigest()).name, 'crt-cli-host') + + def test_zero_or_two_matching_executables_are_refused(self): + (self.dir / 'a').write_bytes(b'same') + with self.assertRaisesRegex(ReleaseError, 'found 0'): + locate_executable(self.dir, '0' * 64) + (self.dir / 'b').write_bytes(b'same') + with self.assertRaisesRegex(ReleaseError, 'found 2'): + locate_executable(self.dir, hashlib.sha256(b'same').hexdigest()) + + def test_sdk_is_the_directory_the_summary_names(self): + sdk = {'version': 'v0.0.1-x', 'content_sha256': 'cd2ab657615c0fa52a1804f301a57dfa21b1067e3e5ed5a9f9a07d86ae050cc6'} + (self.dir / 'v0.0.1-x-cd2ab657615c/share/licenses').mkdir(parents=True) + (self.dir / 'v0.0.1-x-cd2ab657615c/manifest.json').write_text('{}') + self.assertEqual(locate_sdk(self.dir, sdk).name, 'v0.0.1-x-cd2ab657615c') + with self.assertRaisesRegex(ReleaseError, 'not where'): + locate_sdk(self.dir, dict(sdk, content_sha256='f' * 64)) + + if __name__ == '__main__': unittest.main() From e4c145dbc957cf5ae9e892dc3f87fbe8bcafe294 Mon Sep 17 00:00:00 2001 From: Cheolmin Jo Date: Wed, 11 Nov 2026 13:30:05 +0900 Subject: [PATCH 3/3] ci: add the protected release workflow with package, verify and approval-gated publish release.yml (manual) builds exactly a requested commit that is on main and has a passing CI gate, packs a deterministic archive, verifies it on a separate runner (including a real GitHub SDK acquisition and the C and C++ workflows) and publishes only through the `release` environment, which requires a reviewer. publish is the only job with contents: write; it re-checks the digest, stages a draft, compares the downloaded draft byte for byte and refuses an existing tag. dry_run (default) stops at the verified draft. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 188 ++++++++++++++++++++++++++++++++++ tools/ci_release.py | 137 ++++++++++++++++++++++++- 2 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4be096f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,188 @@ +name: Release + +# Manually started, protected packaging of a CLI Preview from a tested commit. CI success is not publication authority: +# package builds exactly the requested commit with the pinned published CLI and the pinned public SDK, assembles the +# notices and packs a deterministic archive (no write permission); +# verify extracts that archive on a separate fresh runner and runs the packaged executable, including a real GitHub +# SDK acquisition and the C and C++ workflows (no write permission); +# publish waits for approval on the `release` environment, re-checks the artifact digest, stages a draft, compares the +# downloaded draft byte for byte and only then publishes a non-latest prerelease. It is the only job with +# `contents: write` and never rebuilds. +# `dry_run: true` stops at the verified draft; delete that draft by hand. Release notes are supplied as a reviewed file. + +on: + workflow_dispatch: + inputs: + host: + description: Host to release + type: choice + options: [linux-x64, linux-arm64] + required: true + commit: + description: Full 40-character commit SHA on main to build (a branch or tag is refused) + type: string + required: true + date: + description: Preview date, for example 20261111 + type: string + required: true + serial: + description: Serial within that date and host, starting at 1 + type: string + required: true + default: '1' + notes_file: + description: Path of the reviewed release notes in the repository at that commit + type: string + required: true + dry_run: + description: Stop at the verified draft instead of publishing + type: boolean + default: true + +permissions: + contents: read + +concurrency: + group: release-${{ inputs.host }} + cancel-in-progress: false + +jobs: + package: + name: package (${{ inputs.host }}) + runs-on: ${{ inputs.host == 'linux-arm64' && 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} + timeout-minutes: 120 + steps: + - name: Check out exactly the requested commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.commit }} + fetch-depth: 0 + persist-credentials: false + + - name: Require the commit to be on main and to have passed CI + env: + GH_TOKEN: ${{ github.token }} + COMMIT: ${{ inputs.commit }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + git fetch --quiet origin main + git merge-base --is-ancestor "$COMMIT" origin/main || { echo "::error::$COMMIT is not on main"; exit 1; } + state="$(gh api "repos/$REPO/commits/$COMMIT/check-runs" --jq '[.check_runs[]|select(.name=="CI gate")|.conclusion]|first // "none"')" + echo "CI gate on $COMMIT: $state" + [ "$state" = success ] || { echo "::error::CI gate has not succeeded for $COMMIT"; exit 1; } + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Install LLVM 21 + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends clang-21 lld-21 llvm-21 binutils + + - name: Check the driver's rules + run: python -m unittest discover -s tools -p test_ci_release.py + + - name: Build, assemble and pack + env: + HOST: ${{ inputs.host }} + COMMIT: ${{ inputs.commit }} + DATE: ${{ inputs.date }} + SERIAL: ${{ inputs.serial }} + run: | + python tools/ci_release.py package --host "$HOST" --commit "$COMMIT" --date "$DATE" --serial "$SERIAL" \ + --work "$RUNNER_TEMP/release-work" --llvm-bin /usr/lib/llvm-21/bin + + - name: Upload the release files + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-${{ inputs.host }}-${{ inputs.date }}-${{ inputs.serial }} + path: ${{ runner.temp }}/release-work/release/ + if-no-files-found: error + retention-days: 14 + + verify: + name: verify (${{ inputs.host }}) + needs: package + runs-on: ${{ inputs.host == 'linux-arm64' && 'ubuntu-26.04-arm' || 'ubuntu-26.04' }} + timeout-minutes: 60 + steps: + - name: Check out the driver at the requested commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.commit }} + persist-credentials: false + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Install LLVM 21 + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends clang-21 lld-21 llvm-21 binutils + + - name: Download the release files from the package job + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-${{ inputs.host }}-${{ inputs.date }}-${{ inputs.serial }} + path: ${{ runner.temp }}/release-files + + - name: Extract and run the packaged executable + env: + HOST: ${{ inputs.host }} + DATE: ${{ inputs.date }} + SERIAL: ${{ inputs.serial }} + run: | + python tools/ci_release.py verify --host "$HOST" --date "$DATE" --serial "$SERIAL" \ + --release-dir "$RUNNER_TEMP/release-files" --work "$RUNNER_TEMP/verify-work" --llvm-bin /usr/lib/llvm-21/bin + + publish: + name: publish (${{ inputs.host }}) + needs: verify + runs-on: ubuntu-24.04 + timeout-minutes: 30 + # Required reviewers are configured on this environment; the job does not start until a reviewer approves. + environment: release + permissions: + contents: write + steps: + - name: Check out the driver and the reviewed release notes at the requested commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.commit }} + persist-credentials: false + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Download the verified release files + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: release-${{ inputs.host }}-${{ inputs.date }}-${{ inputs.serial }} + path: ${{ runner.temp }}/release-files + + - name: Stage, compare and publish + env: + GH_TOKEN: ${{ github.token }} + HOST: ${{ inputs.host }} + COMMIT: ${{ inputs.commit }} + DATE: ${{ inputs.date }} + SERIAL: ${{ inputs.serial }} + NOTES: ${{ inputs.notes_file }} + REPO: ${{ github.repository }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + args=(--host "$HOST" --commit "$COMMIT" --date "$DATE" --serial "$SERIAL" --repo "$REPO" + --release-dir "$RUNNER_TEMP/release-files" --notes-file "$NOTES" --work "$RUNNER_TEMP/draft-check") + if [ "$DRY_RUN" = true ]; then args+=(--draft-only); fi + python tools/ci_release.py publish "${args[@]}" diff --git a/tools/ci_release.py b/tools/ci_release.py index 5856d84..50913b7 100644 --- a/tools/ci_release.py +++ b/tools/ci_release.py @@ -168,21 +168,152 @@ def command_package(args): log('packaged %s (%d bytes) sha256=%s' % (n['archive'], archive.stat().st_size, digest)) +def command_verify(args): + """Independent check of the packed archive on a fresh runner: only the archive, its sums file and the pinned SDK source.""" + n = names(args.host, args.date, args.serial) + release = Path(args.release_dir).resolve() + archive = release / n['archive'] + digest = require_artifact_match(archive, (release / n['sums']).read_text(encoding='ascii'), n['archive']) + work = Path(args.work).resolve() + work.mkdir(parents=True, exist_ok=False) + extracted = work / 'extracted' + result = subprocess.run([sys.executable, ROOT / 'tools/package_cli_archive.py', 'verify', '--archive', archive, '--extract-to', extracted], + capture_output=True, text=True) + if result.returncode != 0: + raise ReleaseError('archive verification failed: %s' % result.stderr.strip()[-400:]) + info = json.loads(result.stdout) + root = Path(info['root']) + exe = root / 'crt-cli' + env = {'PATH': str(work / 'nowhere'), 'HOME': str(work / 'home'), 'LC_ALL': 'C'} + (work / 'home').mkdir() + def cli(label, *argv, expect=0): + done = subprocess.run([str(exe), *map(str, argv)], capture_output=True, text=True, env=env, timeout=1800) + log('[%s] exit=%d' % (label, done.returncode)) + if (done.returncode == 0) != (expect == 0): + raise ReleaseError('%s: unexpected exit %d: %s' % (label, done.returncode, (done.stdout + done.stderr)[-300:])) + return done + version = cli('version', '--version').stdout + if PRODUCT_VERSION not in version: + raise ReleaseError('the packaged executable reports an unexpected version: %s' % version.strip()) + pin = info['manifest_sha256'] + doctor = json.loads(cli('self-doctor', '--json', 'doctor', '--scope', 'self', '--resources', root, '--resources-sha256', pin).stdout) + if doctor['data']['checks'].get('resources') != 'pass' or doctor['data'].get('publisher_authentication') not in (None, 'not_verified'): + raise ReleaseError('self doctor did not pass the resources check') + cli('wrong-pin', '--json', 'doctor', '--scope', 'self', '--resources', root, '--resources-sha256', '0' * 64, expect=1) + # Real network: the packaged executable acquires the pinned public SDK through the github-release redirect policy. + pins = json.loads((ROOT / '.github/ci/pins.json').read_text(encoding='utf-8'))['hosts'][args.host]['sdk'] + roots = next((c for c in ('/etc/ssl/certs/ca-certificates.crt', '/etc/pki/tls/certs/ca-bundle.crt') if Path(c).is_file()), None) + if roots is None: + raise ReleaseError('no system CA bundle found for the SDK acquisition') + base = pins['release_base'] + source = {'schema': 'crt-cli.sdk-source', 'format': 0, 'ca_file': roots, 'ca_sha256': hashlib.sha256(Path(roots).read_bytes()).hexdigest(), + 'manifest': {'url': base + pins['manifest']['name'], 'sha256': pins['manifest']['sha256'], 'size': pins['manifest']['size']}, + 'archive': {'name': pins['archive'], 'url': base + pins['archive']}, 'contract_url': base + pins['contract'], + 'sdk': {'version': pins['version'], 'content_sha256': pins['content_sha256'], 'contract_sha256': pins['contract_sha256']}, + 'redirect_policy': 'github-release', 'timeout_seconds': 300} + (work / 'sdk-source.json').write_text(json.dumps(source, indent=2) + '\n', encoding='utf-8') + acquired = json.loads(cli('sdk-acquire', '--json', 'sdk', 'acquire', '--source', work / 'sdk-source.json').stdout) + if acquired['data']['identity']['content_sha256'] != pins['content_sha256']: + raise ReleaseError('the acquired SDK identity differs from the pin') + llvm = Path(args.llvm_bin or '/usr/lib/llvm-21/bin') + flags = ['--cc', llvm / 'clang', '--cxx', llvm / 'clang++', '--linker', llvm / 'ld.lld', '--ar', llvm / 'llvm-ar'] + for language in ('c', 'cxx'): + project = work / 'projects' / ('app_' + language) + project.parent.mkdir(exist_ok=True) + cli(language + '-new', 'new', project, '--language', language) + cli(language + '-build', 'build', '--project', project, '--sdk', pins['content_sha256'], *flags) + cli(language + '-test', 'test', '--project', project) + ran = cli(language + '-run', 'run', '--project', project) + if 'Hello from CRT' not in (ran.stdout + ran.stderr): + raise ReleaseError('%s: the built program did not print its greeting' % language) + cli(language + '-warm', 'build', '--project', project) + log('archive %s verified: sha256=%s manifest=%s' % (n['archive'], digest, pin)) + (work / 'verify.json').write_text(json.dumps({'archive_sha256': digest, 'manifest_sha256': pin, 'files': info['files'], + 'executable_version': version.strip()}, indent=2) + '\n', encoding='utf-8') + + +def gh(*argv, check=True): + done = subprocess.run(['gh', *map(str, argv)], capture_output=True, text=True) + if check and done.returncode != 0: + raise ReleaseError('gh %s failed: %s' % (' '.join(map(str, argv[:3])), (done.stderr or done.stdout).strip()[-300:])) + return done + + +def command_publish(args): + """Stage a draft from the verified bytes, check it, and publish a non-latest prerelease (or stop at the draft). + + The only step with write permission. It never rebuilds: it uploads the files the verify job saw, after re-checking + their digest, and it refuses a tag that already has a release. + """ + commit = require_commit(args.commit) + n = names(args.host, args.date, args.serial) + release = Path(args.release_dir).resolve() + archive, sums = release / n['archive'], release / n['sums'] + digest = require_artifact_match(archive, sums.read_text(encoding='ascii'), n['archive']) + repo = args.repo + if gh('release', 'view', n['tag'], '--repo', repo, check=False).returncode == 0: + raise ReleaseError('a release for %s already exists; releases are never overwritten' % n['tag']) + notes = Path(args.notes_file).resolve() + if not notes.is_file() or not notes.read_text(encoding='utf-8').strip(): + raise ReleaseError('the release notes file is missing or empty') + if digest not in notes.read_text(encoding='utf-8'): + raise ReleaseError('the release notes do not state the archive SHA-256 %s' % digest) + gh('release', 'create', n['tag'], '--repo', repo, '--draft', '--prerelease', '--latest=false', '--target', commit, + '--title', n['title'], '--notes-file', notes, archive, sums) + log('draft %s created' % n['tag']) + check = Path(args.work).resolve() + check.mkdir(parents=True, exist_ok=False) + gh('release', 'download', n['tag'], '--repo', repo, '--dir', check) + require_artifact_match(check / n['archive'], (check / n['sums']).read_text(encoding='ascii'), n['archive']) + if (check / n['archive']).read_bytes() != archive.read_bytes() or (check / n['sums']).read_bytes() != sums.read_bytes(): + raise ReleaseError('the draft assets differ from the verified files') + view = json.loads(gh('release', 'view', n['tag'], '--repo', repo, '--json', 'isDraft,isPrerelease,targetCommitish,assets').stdout) + if not view['isDraft'] or not view['isPrerelease'] or view['targetCommitish'] != commit or len(view['assets']) != 2: + raise ReleaseError('the staged draft does not match the request: %s' % json.dumps(view)[:300]) + log('draft verified byte for byte') + if args.draft_only: + log('draft-only: stopping before publication') + return + gh('release', 'edit', n['tag'], '--repo', repo, '--draft=false', '--prerelease', '--latest=false') + final = json.loads(gh('release', 'view', n['tag'], '--repo', repo, '--json', 'isDraft,isPrerelease,tagName,url').stdout) + if final['isDraft'] or not final['isPrerelease']: + raise ReleaseError('the published release is not a non-draft prerelease: %s' % final) + log('published %s' % final['url']) + + def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest='command', required=True) - for name in ('package',): + for name in ('package', 'verify'): p = sub.add_parser(name) p.add_argument('--host', choices=sorted(HOSTS), required=True) - p.add_argument('--commit', required=True) p.add_argument('--date', required=True) p.add_argument('--serial', type=int, required=True) p.add_argument('--work', required=True) - p.add_argument('--llvm-bin') + if name == 'package': + p.add_argument('--commit', required=True) + p.add_argument('--llvm-bin') + else: + p.add_argument('--release-dir', required=True) + p.add_argument('--llvm-bin') + p = sub.add_parser('publish') + p.add_argument('--host', choices=sorted(HOSTS), required=True) + p.add_argument('--commit', required=True) + p.add_argument('--date', required=True) + p.add_argument('--serial', type=int, required=True) + p.add_argument('--release-dir', required=True) + p.add_argument('--notes-file', required=True) + p.add_argument('--repo', required=True) + p.add_argument('--work', required=True) + p.add_argument('--draft-only', action='store_true', help='stage and verify the draft but do not publish') args = parser.parse_args() try: if args.command == 'package': command_package(args) + elif args.command == 'verify': + command_verify(args) + elif args.command == 'publish': + command_publish(args) except ReleaseError as error: log('RELEASE FAIL: %s' % error) return 1