From 0773c02c3844ffc9218574f8a696fc94e3a600f1 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Fri, 30 Oct 2026 10:44:42 -0400 Subject: [PATCH] Deploy successful Fief main images and verify running containers --- .github/workflows/docker-image.yml | 42 ++++++- scripts/deploy/README.md | 24 ++++ scripts/deploy/fief-release-launch.sh | 11 ++ .../deploy/opensensor-fief-release.service | 12 ++ scripts/deploy/opensensor-fief-release.timer | 11 ++ scripts/hosting_release.py | 104 ++++++++++++++++++ scripts/hosting_release_manifest.py | 26 +++++ scripts/hosting_runtime.py | 64 +++++++++++ scripts/test_hosting_release.py | 32 ++++++ scripts/wait_hosting_release.py | 27 +++++ 10 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 scripts/deploy/README.md create mode 100644 scripts/deploy/fief-release-launch.sh create mode 100644 scripts/deploy/opensensor-fief-release.service create mode 100644 scripts/deploy/opensensor-fief-release.timer create mode 100644 scripts/hosting_release.py create mode 100644 scripts/hosting_release_manifest.py create mode 100644 scripts/hosting_runtime.py create mode 100644 scripts/test_hosting_release.py create mode 100644 scripts/wait_hosting_release.py diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 879b0f4b..5786bddf 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -7,6 +7,8 @@ on: tags: - 'v*' workflow_dispatch: + pull_request: + paths: ["scripts/hosting_*.py", "scripts/test_hosting_release.py", "scripts/wait_hosting_release.py", ".github/workflows/docker-image.yml"] env: REGISTRY: ghcr.io @@ -16,8 +18,15 @@ jobs: checks: uses: ./.github/workflows/checks.yml + release-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: python3 -m unittest discover -s scripts -p test_hosting_release.py -v + docker-build-and-push: - needs: checks + if: github.event_name != 'pull_request' + needs: [checks, release-tests] runs-on: ubuntu-latest permissions: contents: read @@ -50,6 +59,7 @@ jobs: type=sha - name: Build and push Docker image + id: build uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . @@ -62,3 +72,33 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Record this main build's exact platform image + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + BUILD_DIGEST: ${{ steps.build.outputs.digest }} + run: python3 scripts/hosting_release_manifest.py + - uses: actions/upload-artifact@v4 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + with: + name: hosting-release-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} + path: hosting-release.json + if-no-files-found: error + retention-days: 30 + + deploy: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: docker-build-and-push + runs-on: ubuntu-latest + permissions: + contents: read + statuses: read + timeout-minutes: 40 + concurrency: + group: fief-production-verification + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + - name: Verify this image is running in production + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/wait_hosting_release.py diff --git a/scripts/deploy/README.md b/scripts/deploy/README.md new file mode 100644 index 00000000..17001d8e --- /dev/null +++ b/scripts/deploy/README.md @@ -0,0 +1,24 @@ +# Fief production releases + +Every successful main build emits its exact platform image digest. The operator +host follows main and deploys that receipt automatically; CI waits for the +matching deployment status. Existing environment and secret references are +preserved. Verification checks source revision, actual container image ID, +readiness, and absence of old pods. Older workflow runs cannot roll back a newer +completed release. + +Install the timer on the existing operator host: + +```sh +install -Dm755 scripts/deploy/fief-release-launch.sh ~/.local/lib/opensensor-hosting/fief-release-launch.sh +install -Dm644 scripts/deploy/opensensor-fief-release.service ~/.config/systemd/user/opensensor-fief-release.service +install -Dm644 scripts/deploy/opensensor-fief-release.timer ~/.config/systemd/user/opensensor-fief-release.timer +systemctl --user daemon-reload +systemctl --user enable --now opensensor-fief-release.timer +``` + +Existing host GitHub and Kubernetes credentials are used. Inspect failures with +`journalctl --user -u opensensor-fief-release.service`; retries are automatic. +The companion storefront change stops GrowMax deployments from reapplying a +historical Fief image. Merge this Fief change first so its release helper is +available to the storefront's manual recovery workflow. diff --git a/scripts/deploy/fief-release-launch.sh b/scripts/deploy/fief-release-launch.sh new file mode 100644 index 00000000..bb3de44f --- /dev/null +++ b/scripts/deploy/fief-release-launch.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +release_checkout="$HOME/.cache/opensensor-hosting/fief-release" +if [[ ! -d "$release_checkout/.git" ]]; then + git clone --branch main --single-branch https://github.com/opensensor/fief.git "$release_checkout" +fi +git -C "$release_checkout" fetch origin main +git -C "$release_checkout" merge --ff-only origin/main +agent="$release_checkout/scripts/hosting_release.py" +[[ -f "$agent" ]] || exit 0 +exec /usr/bin/python3 "$agent" --apply diff --git a/scripts/deploy/opensensor-fief-release.service b/scripts/deploy/opensensor-fief-release.service new file mode 100644 index 00000000..5fbc7624 --- /dev/null +++ b/scripts/deploy/opensensor-fief-release.service @@ -0,0 +1,12 @@ +[Unit] +Description=Deploy latest main fief build to production +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +Environment=PATH=/snap/bin:/usr/local/bin:/usr/bin:/bin +ExecStart=/usr/bin/bash %h/.local/lib/opensensor-hosting/fief-release-launch.sh +TimeoutStartSec=35min +UMask=0077 +Nice=5 diff --git a/scripts/deploy/opensensor-fief-release.timer b/scripts/deploy/opensensor-fief-release.timer new file mode 100644 index 00000000..61c8fe86 --- /dev/null +++ b/scripts/deploy/opensensor-fief-release.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Watch for new main fief images + +[Timer] +OnBootSec=30s +OnUnitInactiveSec=1min +AccuracySec=5s +Unit=opensensor-fief-release.service + +[Install] +WantedBy=timers.target diff --git a/scripts/hosting_release.py b/scripts/hosting_release.py new file mode 100644 index 00000000..c899c68b --- /dev/null +++ b/scripts/hosting_release.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Consume the latest successful main image build and resume paired deployments.""" +import argparse +import fcntl +import io +import json +import os +from pathlib import Path +import re +import subprocess +import time +import zipfile + +from hosting_runtime import deploy, save_progress + +REPO = 'opensensor/fief' +WORKFLOW = '.github/workflows/docker-image.yml' +CONTEXT = 'hosting/fief-release' +STATE = Path.home() / '.config/opensensor-hosting/fief-release' +IMAGE = re.compile(r'ghcr\.io/opensensor/fief@sha256:[a-f0-9]{64}') + + +def gh(path, **fields): + args = ['gh', 'api', '-H', 'Cache-Control: no-cache', path] + for key, value in fields.items():args += ['-f', key + '=' + str(value)] + return json.loads(subprocess.check_output(args, timeout=90)) + + +def validate(target, workflow): + if (workflow['event'] != 'push' or workflow['head_branch'] != 'main' + or workflow['path'] != WORKFLOW or workflow['repository']['full_name'] != REPO + or set(target) != {'format', 'repository', 'commit', 'run_id', 'run_attempt', 'image'} + or target['format'] != 'fief-release-v1' or target['repository'] != REPO + or target['commit'] != workflow['head_sha'] or target['run_id'] != workflow['id'] + or target['run_attempt'] != workflow['run_attempt'] + or not re.fullmatch('[a-f0-9]{40}', target['commit']) or not IMAGE.fullmatch(target['image'])): + raise ValueError('Release must identify this main build and its immutable image') + return target + + +def latest(): + runs = gh(f'repos/{REPO}/actions/workflows/docker-image.yml/runs?branch=main&event=push&per_page=20')['workflow_runs'] + for workflow in sorted(runs, key=lambda r:(r['run_number'], r['run_attempt']), reverse=True): + jobs = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/attempts/{workflow["run_attempt"]}/jobs?per_page=100')['jobs'] + if not any(j['name']=='docker-build-and-push' and j['conclusion']=='success' for j in jobs):continue + artifacts = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/artifacts?per_page=100')['artifacts'] + name = f'hosting-release-{workflow["head_sha"]}-{workflow["id"]}-{workflow["run_attempt"]}' + matches = [a for a in artifacts if a['name']==name and not a['expired']] + if len(matches)!=1:raise RuntimeError('Successful build is missing its release artifact') + raw = subprocess.check_output(['gh','api',f'repos/{REPO}/actions/artifacts/{matches[0]["id"]}/zip'],timeout=90) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + if archive.namelist()!=['hosting-release.json'] or archive.infolist()[0].file_size>8192: + raise ValueError('Unexpected release artifact') + return validate(json.loads(archive.read('hosting-release.json')),workflow) + return None + + +def report(target, state, description): + gh(f'repos/{REPO}/statuses/{target["commit"]}',state=state, + context=f'{CONTEXT}/{target["run_id"]}/{target["run_attempt"]}',description=description[:140], + target_url=f'https://github.com/{REPO}/actions/runs/{target["run_id"]}') + + +def choose(pending, candidate, previous): + # Finish a handoff already in progress before starting another release. + if pending:return pending + if not candidate:return None + if previous: + before=(previous['run_id'],previous['run_attempt']) + after=(candidate['run_id'],candidate['run_attempt']) + if aftertarget['run_id']:raise RuntimeError('A newer Fief build is already deployed') + annotations.update({'opensensor.io/source-revision':target['commit'],'opensensor.io/build-run':str(target['run_id'])}) + containers=template['spec']['containers'] + matches=[c for c in containers if c['name']==NAME] + if len(matches)!=1:raise RuntimeError('Fief container identity changed') + matches[0]['image']=target['image'] + patch=[{'op':'test','path':'/metadata/resourceVersion','value':deployment['metadata']['resourceVersion']}, + {'op':'replace','path':'/spec/template','value':template}] + kube('patch','deployment',NAME,'--type=json','-p',json.dumps(patch)) + kube('rollout','status','deployment/'+NAME,'--timeout=600s') + deadline=time.monotonic()+120 + while True: + actual=get('deployment',NAME) + selector=','.join(k+'='+v for k,v in actual['spec']['selector']['matchLabels'].items()) + try:verify(actual,get('pods','-l',selector)['items'],target['image'],target['commit']);break + except RuntimeError: + if time.monotonic()>=deadline:raise + time.sleep(3) + return {'production_ready':True} + + +def save_progress(path,state): + temp=path.with_suffix('.tmp');temp.write_text(json.dumps(state));temp.replace(path) diff --git a/scripts/test_hosting_release.py b/scripts/test_hosting_release.py new file mode 100644 index 00000000..946c4d97 --- /dev/null +++ b/scripts/test_hosting_release.py @@ -0,0 +1,32 @@ +import copy +import unittest +import hosting_release as agent +from hosting_runtime import verify,NAME +from hosting_release_manifest import platform_digest + +IMAGE='ghcr.io/opensensor/fief@sha256:'+'a'*64 +COMMIT='b'*40 +TARGET=dict(format='fief-release-v1',repository=agent.REPO,commit=COMMIT,run_id=100,run_attempt=1,image=IMAGE) +WORKFLOW=dict(event='push',head_branch='main',path=agent.WORKFLOW,repository={'full_name':agent.REPO},head_sha=COMMIT,id=100,run_attempt=1) + +class ReleaseTests(unittest.TestCase): + def test_release_belongs_to_main_build(self): + self.assertEqual(agent.validate(TARGET,WORKFLOW),TARGET) + for key,value in [('run_id',99),('run_attempt',2),('image','ghcr.io/opensensor/fief:latest')]: + with self.subTest(key=key),self.assertRaises(ValueError):agent.validate({**TARGET,key:value},WORKFLOW) + with self.assertRaises(ValueError):agent.validate(TARGET,{**WORKFLOW,'event':'pull_request'}) + def test_older_build_cannot_replace_newer_build(self): + with self.assertRaises(RuntimeError):agent.choose({},TARGET,{**TARGET,'run_id':101,'state':'success'}) + def test_ready_old_image_is_rejected(self): + deployment={'metadata':{'generation':1},'spec':{'replicas':1},'status':{'observedGeneration':1,'replicas':1,'updatedReplicas':1,'readyReplicas':1}} + pod={'metadata':{'annotations':{'opensensor.io/source-revision':COMMIT}},'spec':{'containers':[{'name':NAME,'image':IMAGE}]},'status':{'containerStatuses':[{'name':NAME,'ready':True,'imageID':IMAGE}]}} + verify(deployment,[pod],IMAGE,COMMIT) + old=copy.deepcopy(pod);old['status']['containerStatuses'][0]['imageID']='old' + with self.assertRaises(RuntimeError):verify(deployment,[old],IMAGE,COMMIT) + with self.assertRaises(RuntimeError):verify(deployment,[pod,old],IMAGE,COMMIT) + def test_platform_digest_excludes_attestation(self): + doc={'manifests':[{'platform':{'os':'unknown','architecture':'unknown'},'digest':'other'}, + {'platform':{'os':'linux','architecture':'amd64'},'digest':'runtime'}]} + self.assertEqual(platform_digest(doc,'index'),'runtime') + +if __name__=='__main__':unittest.main() diff --git a/scripts/wait_hosting_release.py b/scripts/wait_hosting_release.py new file mode 100644 index 00000000..ecf729bc --- /dev/null +++ b/scripts/wait_hosting_release.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Keep CI pending until this build is verified on production.""" +import json +import os +import subprocess +import time + + +def main(): + repo, commit = os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_SHA'] + context = f'hosting/fief-release/{os.environ["GITHUB_RUN_ID"]}/{os.environ["GITHUB_RUN_ATTEMPT"]}' + deadline = time.monotonic() + 2100 + while time.monotonic() < deadline: + raw = subprocess.check_output(['gh','api','--paginate','--slurp','-H','Cache-Control: no-cache', + f'repos/{repo}/commits/{commit}/statuses?per_page=100'],timeout=60) + statuses = [s for page in json.loads(raw) for s in page if s['context']==context] + if statuses: + status = max(statuses,key=lambda s:s['id']) + print(status['state']+': '+status['description'],flush=True) + if status['state']=='success':return + # The host resumes failed handoffs automatically; keep this run open for its retry. + else:print('Waiting for the production release agent',flush=True) + time.sleep(20) + raise SystemExit('This build has not been verified in production; inspect hosting/fief-release') + + +if __name__=='__main__':main()