diff --git a/.github/ci/integration-pins.json b/.github/ci/integration-pins.json new file mode 100644 index 0000000..b5ccbc9 --- /dev/null +++ b/.github/ci/integration-pins.json @@ -0,0 +1,19 @@ +{ + "schema": "crt-cli.ci-integration-pins", + "format": 0, + "note": "Exact inputs of the daily/manual integration workflow. An empty (null) pin is a failure, never a skip: a skipped suite looks like a pass. Fill a pin only in a reviewed commit that cites the release or source record and re-checks the digests against the published bytes.", + "zlib_source": { + "url": "https://zlib.net/fossils/zlib-1.3.1.tar.gz", + "name": "zlib-1.3.1.tar.gz", + "sha256": "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23", + "size": 1512791 + }, + "hosts": { + "linux-x64": { + "host_tools": {"url": null, "name": null, "sha256": null, "size": null, "manifest_sha256": null, "version": null} + }, + "linux-arm64": { + "host_tools": {"url": null, "name": null, "sha256": null, "size": null, "manifest_sha256": null, "version": null} + } + } +} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..634bde6 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,110 @@ +name: Integration + +# Input-gated integration checks that are too slow or too external for the required pull-request gate in ci.yml. +# Every input is pinned in .github/ci/integration-pins.json; an unpinned input fails the job instead of skipping it, +# because a skipped suite looks like a pass. This workflow is not a required check and never publishes anything. +# The scheduled trigger is intentionally absent until the host-tools pin exists: a daily run that is red for a known, +# unfixable-from-here reason teaches people to ignore red. Add `schedule:` in the commit that fills the pin. + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: integration-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + pins: + name: pins (${{ matrix.host }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + runner: ubuntu-26.04 + - host: linux-arm64 + runner: ubuntu-26.04-arm + steps: + - name: Check out the sources under test + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Check the driver's pin gates + run: python -m unittest discover -s tools -p test_ci_integration.py + + - name: Require every integration input to be pinned + run: python tools/ci_integration.py pins --host "${{ matrix.host }}" + + redirect: + name: redirect (${{ matrix.host }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + runner: ubuntu-26.04 + - host: linux-arm64 + runner: ubuntu-26.04-arm + steps: + - name: Check out the sources under test + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Restricted-redirect replay over the pinned public SDK bytes + run: python tools/ci_integration.py redirect --host "${{ matrix.host }}" --work "$RUNNER_TEMP/redirect" + + - name: Upload the replay result + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: redirect-${{ matrix.host }}-${{ github.sha }} + path: ${{ runner.temp }}/redirect/redirect-replay/redirect-replay.json + if-no-files-found: warn + retention-days: 14 + + ports: + name: ports (${{ matrix.host }}) + needs: pins + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - host: linux-x64 + runner: ubuntu-26.04 + - host: linux-arm64 + runner: ubuntu-26.04-arm + steps: + - name: Check out the sources under test + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python for the driver + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + # Not wired yet: fails explicitly rather than reporting a pass for work that did not run. + - name: Port suites + run: python tools/ci_integration.py ports --host "${{ matrix.host }}" diff --git a/tools/ci_integration.py b/tools/ci_integration.py new file mode 100644 index 0000000..27b7338 --- /dev/null +++ b/tools/ci_integration.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Daily/manual integration driver for crt-cli; a build-host orchestrator, never a CLI runtime dependency. + +Reads .github/ci/integration-pins.json, downloads every pinned input, checks its size and SHA-256, and only then +runs the replays. A missing (null) pin or a digest mismatch is a failure: a skipped suite looks like a pass. +Subcommands: + pins check the pin file only (what a scheduled run needs to know before spending runner time) + redirect restricted-redirect replay: the pinned published CLI against the pinned public SDK bytes + ports the five input-gated port suites through the pinned host tools and zlib source (fails while unpinned) +""" +import argparse +import hashlib +import json +import platform +import shutil +import ssl +import subprocess +import sys +import tarfile +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +class IntegrationError(Exception): + pass + + +HOST_TOOL_FIELDS = ('url', 'name', 'sha256', 'size', 'manifest_sha256', 'version') +ZLIB_FIELDS = ('url', 'name', 'sha256', 'size') + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for block in iter(lambda: stream.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def _require_fields(label, entry, fields): + if not isinstance(entry, dict): + raise IntegrationError('%s: pin is not an object' % label) + missing = [name for name in fields if entry.get(name) in (None, '')] + if missing: + raise IntegrationError('%s: unpinned %s (an empty pin is a failure, not a skip)' % (label, ', '.join(missing))) + if not str(entry['url']).startswith('https://'): + raise IntegrationError('%s: url must be https' % label) + for name in ('sha256', 'manifest_sha256'): + if name in entry and (len(entry[name]) != 64 or any(c not in '0123456789abcdef' for c in entry[name])): + raise IntegrationError('%s: %s is not a lowercase SHA-256' % (label, name)) + if type(entry['size']) is not int or entry['size'] <= 0: + raise IntegrationError('%s: size must be a positive integer' % label) + + +def load_pins(path, host): + pins = json.loads(Path(path).read_text(encoding='utf-8')) + if pins.get('schema') != 'crt-cli.ci-integration-pins' or pins.get('format') != 0: + raise IntegrationError('unsupported integration pin file') + if host not in pins.get('hosts', {}): + raise IntegrationError('no integration pins for host %s' % host) + _require_fields('zlib_source', pins.get('zlib_source'), ZLIB_FIELDS) + _require_fields('%s host_tools' % host, pins['hosts'][host].get('host_tools'), HOST_TOOL_FIELDS) + return pins['zlib_source'], pins['hosts'][host]['host_tools'] + + +def verify_download(label, path, entry): + path = Path(path) + if not path.is_file(): + raise IntegrationError('%s: download is missing' % label) + if path.stat().st_size != entry['size']: + raise IntegrationError('%s: size %d differs from the pin %d' % (label, path.stat().st_size, entry['size'])) + actual = sha256_file(path) + if actual != entry['sha256']: + raise IntegrationError('%s: SHA-256 %s differs from the pin %s' % (label, actual, entry['sha256'])) + + +def log(message): + print(message, file=sys.stderr, flush=True) + + +def host_key(system=None, machine=None): + system = system if system is not None else sys.platform + machine = (machine if machine is not None else platform.machine()).lower() + if system.startswith('linux') and machine in ('x86_64', 'amd64'): + return 'linux-x64' + if system.startswith('linux') and machine in ('aarch64', 'arm64'): + return 'linux-arm64' + raise IntegrationError('integration runs support Linux/x86_64 and Linux/aarch64 only: %s/%s' % (system, machine)) + + +def download(url, destination, attempts=3): + """Bounded retries; the SHA-256 pin, not TLS, authenticates the bytes.""" + destination = Path(destination) + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(url, timeout=180, context=ssl.create_default_context()) as src, destination.open('wb') as dst: + shutil.copyfileobj(src, dst) + return + except OSError as error: + log('download attempt %d of %s failed: %s' % (attempt, url, error)) + if attempt == attempts: + raise IntegrationError('cannot download %s' % url) + time.sleep(5 * attempt) + + +def fetch_pinned(label, url, destination, entry): + download(url, destination) + verify_download(label, destination, entry) + + +def run(label, argv, timeout=3600, **kwargs): + log('[%s] %s' % (label, ' '.join(map(str, argv)))) + done = subprocess.run([str(a) for a in argv], timeout=timeout, **kwargs) + if done.returncode != 0: + raise IntegrationError('%s failed with exit %d' % (label, done.returncode)) + return done + + +def published_cli(host, work): + """The same pinned, published Preview that bootstraps ci.yml; its pin lives in .github/ci/pins.json.""" + ci = json.loads((ROOT / '.github/ci/pins.json').read_text(encoding='utf-8'))['hosts'][host] + pin = ci['bootstrap'] + archive = Path(work) / pin['asset'] + fetch_pinned('bootstrap CLI', 'https://github.com/webos21/crt-cli/releases/download/%s/%s' % (pin['release'], pin['asset']), archive, pin) + target = Path(work) / 'cli' + target.mkdir() + with tarfile.open(archive) as bundle: + bundle.extractall(target, filter='data') + found = [p for p in target.rglob('crt-cli') if p.is_file()] + if len(found) != 1: + raise IntegrationError('expected one crt-cli in the bootstrap archive, found %d' % len(found)) + found[0].chmod(found[0].stat().st_mode | 0o111) + return found[0], ci['sdk'] + + +def sdk_release_dir(sdk, work): + """The three public SDK assets, each checked against the ci pins, laid out as replay_redirect_linux.py expects.""" + release = Path(work) / 'sdk-release' + release.mkdir() + base = sdk['release_base'] + manifest = sdk['manifest'] + fetch_pinned('SDK manifest', base + manifest['name'], release / manifest['name'], manifest) + contract = release / sdk['contract'] + download(base + sdk['contract'], contract) + if sha256_file(contract) != sdk['contract_sha256']: + raise IntegrationError('SDK contract does not match its pin') + archive = release / sdk['archive'] + download(base + sdk['archive'], archive) + manifest_data = json.loads((release / manifest['name']).read_text(encoding='utf-8')) + entry = next((a for a in manifest_data['assets'] if a.get('name') == sdk['archive']), None) + if entry is None: + raise IntegrationError('the SDK manifest does not list the pinned archive') + verify_download('SDK archive', archive, {'size': entry['size'], 'sha256': entry['sha256']}) + return release + + +def command_redirect(args, host): + work = Path(args.work).resolve() + work.mkdir(parents=True, exist_ok=False) + exe, sdk = published_cli(host, work) + release = sdk_release_dir(sdk, work) + output = work / 'redirect-replay' + run('redirect-replay', [sys.executable, ROOT / 'tools/replay_redirect_linux.py', '--cli', exe, '--release-dir', release, '--output', output]) + result = json.loads((output / 'redirect-replay.json').read_text(encoding='utf-8')) + if result.get('passed') is not True or not result.get('cases'): + raise IntegrationError('redirect replay did not report a complete pass') + log('redirect replay passed: %d cases' % len(result['cases'])) + + +def command_ports(args, host): + # Every input is checked before any runner time is spent on the replay. + zlib, tools = load_pins(ROOT / '.github/ci/integration-pins.json', host) + raise IntegrationError('port suites are not wired to a Linux replay driver yet (inputs are pinned: zlib %s, host tools %s)' + % (zlib['sha256'][:12], tools['version'])) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('command', choices=['pins', 'redirect', 'ports']) + parser.add_argument('--host', choices=['linux-x64', 'linux-arm64'], help='default: detect') + parser.add_argument('--work', help='new directory for downloads and results') + args = parser.parse_args() + try: + host = args.host or host_key() + if args.command == 'pins': + load_pins(ROOT / '.github/ci/integration-pins.json', host) + log('integration pins complete for %s' % host) + elif args.command == 'redirect': + if not args.work: + parser.error('--work is required') + command_redirect(args, host) + else: + command_ports(args, host) + except IntegrationError as error: + log('INTEGRATION FAIL: %s' % error) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/test_ci_integration.py b/tools/test_ci_integration.py new file mode 100644 index 0000000..2e8053a --- /dev/null +++ b/tools/test_ci_integration.py @@ -0,0 +1,84 @@ +"""Integration pin gates: an empty or wrong pin must fail, never skip.""" +import json +import tempfile +import unittest +from pathlib import Path + +from ci_integration import IntegrationError, load_pins, sha256_file, verify_download + +GOOD_ZLIB = {'url': 'https://example.invalid/zlib.tar.gz', 'name': 'zlib.tar.gz', 'sha256': 'a' * 64, 'size': 10} +GOOD_TOOLS = {'url': 'https://example.invalid/tools.tar.xz', 'name': 'tools.tar.xz', 'sha256': 'b' * 64, 'size': 20, + 'manifest_sha256': 'c' * 64, 'version': 'v0'} + + +def write(directory, zlib=None, tools=None, schema='crt-cli.ci-integration-pins'): + path = Path(directory) / 'pins.json' + path.write_text(json.dumps({'schema': schema, 'format': 0, 'zlib_source': zlib or dict(GOOD_ZLIB), + 'hosts': {'linux-x64': {'host_tools': tools or dict(GOOD_TOOLS)}}})) + return path + + +class PinTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_complete_pins_load(self): + zlib, tools = load_pins(write(self.dir), 'linux-x64') + self.assertEqual((zlib['size'], tools['version']), (10, 'v0')) + + def test_every_empty_host_tool_pin_fails(self): + for name in GOOD_TOOLS: + tools = dict(GOOD_TOOLS, **{name: None}) + with self.subTest(field=name), self.assertRaisesRegex(IntegrationError, 'unpinned'): + load_pins(write(self.dir, tools=tools), 'linux-x64') + + def test_empty_zlib_pin_fails(self): + with self.assertRaisesRegex(IntegrationError, 'unpinned size'): + load_pins(write(self.dir, zlib=dict(GOOD_ZLIB, size=None)), 'linux-x64') + + def test_http_malformed_digest_and_bad_size_fail(self): + for tools, message in [(dict(GOOD_TOOLS, url='http://example.invalid/t'), 'https'), + (dict(GOOD_TOOLS, sha256='B' * 64), 'lowercase SHA-256'), + (dict(GOOD_TOOLS, manifest_sha256='abc'), 'lowercase SHA-256'), + (dict(GOOD_TOOLS, size='20'), 'positive integer'), + (dict(GOOD_TOOLS, size=0), 'positive integer')]: + with self.subTest(message=message), self.assertRaisesRegex(IntegrationError, message): + load_pins(write(self.dir, tools=tools), 'linux-x64') + + def test_unknown_host_schema_and_missing_file_fail(self): + with self.assertRaisesRegex(IntegrationError, 'no integration pins for host'): + load_pins(write(self.dir), 'win-x64') + with self.assertRaisesRegex(IntegrationError, 'unsupported'): + load_pins(write(self.dir, schema='other'), 'linux-x64') + with self.assertRaises(FileNotFoundError): + load_pins(self.dir / 'absent.json', 'linux-x64') + + def test_download_must_match_size_and_digest(self): + blob = self.dir / 'blob' + blob.write_bytes(b'hello') + entry = {'size': 5, 'sha256': sha256_file(blob)} + verify_download('blob', blob, entry) + for bad, message in [(dict(entry, size=6), 'size'), (dict(entry, sha256='0' * 64), 'SHA-256')]: + with self.subTest(message=message), self.assertRaisesRegex(IntegrationError, message): + verify_download('blob', blob, bad) + with self.assertRaisesRegex(IntegrationError, 'missing'): + verify_download('blob', self.dir / 'nope', entry) + + def test_committed_pins_fail_until_host_tools_are_pinned(self): + root = Path(__file__).resolve().parent.parent / '.github/ci/integration-pins.json' + pins = json.loads(root.read_text()) + # The zlib source is pinned to bytes the recipe also pins; the host tools stay unpinned until CRT publishes them. + recipe = json.loads((Path(__file__).resolve().parent.parent / 'src/test/data/zlib.recipe.json').read_text()) + self.assertEqual(pins['zlib_source']['sha256'], recipe['source']['sha256']) + for host in pins['hosts']: + if any(v in (None, '') for v in pins['hosts'][host]['host_tools'].values()): + with self.assertRaisesRegex(IntegrationError, 'unpinned'): + load_pins(root, host) + + +if __name__ == '__main__': + unittest.main()