Automatically deploy and verify successful Fief main builds

.github/workflows/docker-image.yml

@@ -7,6 +7,8 @@ on:
     tags:
       - 'v*'
   workflow_dispatch:
+  pull_request:
+    paths: ["scripts/hosting_*.py", "scripts/test_hosting_release.py", "scripts/wait_hosting_release.py", ".github/workflows/docker-image.yml"]
 
 env:
   REGISTRY: ghcr.io
@@ -16,8 +18,15 @@ jobs:
   checks:
     uses: ./.github/workflows/checks.yml
 
+  release-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: python3 -m unittest discover -s scripts -p test_hosting_release.py -v
+
   docker-build-and-push:
-    needs: checks
+    if: github.event_name != 'pull_request'
+    needs: [checks, release-tests]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -50,6 +59,7 @@ jobs:
             type=sha
 
       - name: Build and push Docker image
+        id: build
         uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
@@ -62,3 +72,33 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+      - name: Record this main build's exact platform image
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          BUILD_DIGEST: ${{ steps.build.outputs.digest }}
+        run: python3 scripts/hosting_release_manifest.py
+      - uses: actions/upload-artifact@v4
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        with:
+          name: hosting-release-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: hosting-release.json
+          if-no-files-found: error
+          retention-days: 30
+
+  deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: docker-build-and-push
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      statuses: read
+    timeout-minutes: 40
+    concurrency:
+      group: fief-production-verification
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify this image is running in production
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 scripts/wait_hosting_release.py

scripts/deploy/README.md

@@ -0,0 +1,24 @@
+# Fief production releases
+
+Every successful main build emits its exact platform image digest. The operator
+host follows main and deploys that receipt automatically; CI waits for the
+matching deployment status. Existing environment and secret references are
+preserved. Verification checks source revision, actual container image ID,
+readiness, and absence of old pods. Older workflow runs cannot roll back a newer
+completed release.
+
+Install the timer on the existing operator host:
+
+```sh
+install -Dm755 scripts/deploy/fief-release-launch.sh ~/.local/lib/opensensor-hosting/fief-release-launch.sh
+install -Dm644 scripts/deploy/opensensor-fief-release.service ~/.config/systemd/user/opensensor-fief-release.service
+install -Dm644 scripts/deploy/opensensor-fief-release.timer ~/.config/systemd/user/opensensor-fief-release.timer
+systemctl --user daemon-reload
+systemctl --user enable --now opensensor-fief-release.timer
+```
+
+Existing host GitHub and Kubernetes credentials are used. Inspect failures with
+`journalctl --user -u opensensor-fief-release.service`; retries are automatic.
+The companion storefront change stops GrowMax deployments from reapplying a
+historical Fief image. Merge this Fief change first so its release helper is
+available to the storefront's manual recovery workflow.

scripts/deploy/fief-release-launch.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+release_checkout="$HOME/.cache/opensensor-hosting/fief-release"
+if [[ ! -d "$release_checkout/.git" ]]; then
+    git clone --branch main --single-branch https://github.com/opensensor/fief.git "$release_checkout"
+fi
+git -C "$release_checkout" fetch origin main
+git -C "$release_checkout" merge --ff-only origin/main
+agent="$release_checkout/scripts/hosting_release.py"
+[[ -f "$agent" ]] || exit 0
+exec /usr/bin/python3 "$agent" --apply

scripts/deploy/opensensor-fief-release.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Deploy latest main fief build to production
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=PATH=/snap/bin:/usr/local/bin:/usr/bin:/bin
+ExecStart=/usr/bin/bash %h/.local/lib/opensensor-hosting/fief-release-launch.sh
+TimeoutStartSec=35min
+UMask=0077
+Nice=5

scripts/deploy/opensensor-fief-release.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Watch for new main fief images
+
+[Timer]
+OnBootSec=30s
+OnUnitInactiveSec=1min
+AccuracySec=5s
+Unit=opensensor-fief-release.service
+
+[Install]
+WantedBy=timers.target

scripts/hosting_release.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Consume the latest successful main image build and resume paired deployments."""
+import argparse
+import fcntl
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import time
+import zipfile
+
+from hosting_runtime import deploy, save_progress
+
+REPO = 'opensensor/fief'
+WORKFLOW = '.github/workflows/docker-image.yml'
+CONTEXT = 'hosting/fief-release'
+STATE = Path.home() / '.config/opensensor-hosting/fief-release'
+IMAGE = re.compile(r'ghcr\.io/opensensor/fief@sha256:[a-f0-9]{64}')
+
+
+def gh(path, **fields):
+    args = ['gh', 'api', '-H', 'Cache-Control: no-cache', path]
+    for key, value in fields.items():args += ['-f', key + '=' + str(value)]
+    return json.loads(subprocess.check_output(args, timeout=90))
+
+
+def validate(target, workflow):
+    if (workflow['event'] != 'push' or workflow['head_branch'] != 'main'
+            or workflow['path'] != WORKFLOW or workflow['repository']['full_name'] != REPO
+            or set(target) != {'format', 'repository', 'commit', 'run_id', 'run_attempt', 'image'}
+            or target['format'] != 'fief-release-v1' or target['repository'] != REPO
+            or target['commit'] != workflow['head_sha'] or target['run_id'] != workflow['id']
+            or target['run_attempt'] != workflow['run_attempt']
+            or not re.fullmatch('[a-f0-9]{40}', target['commit']) or not IMAGE.fullmatch(target['image'])):
+        raise ValueError('Release must identify this main build and its immutable image')
+    return target
+
+
+def latest():
+    runs = gh(f'repos/{REPO}/actions/workflows/docker-image.yml/runs?branch=main&event=push&per_page=20')['workflow_runs']
+    for workflow in sorted(runs, key=lambda r:(r['run_number'], r['run_attempt']), reverse=True):
+        jobs = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/attempts/{workflow["run_attempt"]}/jobs?per_page=100')['jobs']
+        if not any(j['name']=='docker-build-and-push' and j['conclusion']=='success' for j in jobs):continue
+        artifacts = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/artifacts?per_page=100')['artifacts']
+        name = f'hosting-release-{workflow["head_sha"]}-{workflow["id"]}-{workflow["run_attempt"]}'
+        matches = [a for a in artifacts if a['name']==name and not a['expired']]
+        if len(matches)!=1:raise RuntimeError('Successful build is missing its release artifact')
+        raw = subprocess.check_output(['gh','api',f'repos/{REPO}/actions/artifacts/{matches[0]["id"]}/zip'],timeout=90)
+        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
+            if archive.namelist()!=['hosting-release.json'] or archive.infolist()[0].file_size>8192:
+                raise ValueError('Unexpected release artifact')
+            return validate(json.loads(archive.read('hosting-release.json')),workflow)
+    return None
+
+
+def report(target, state, description):
+    gh(f'repos/{REPO}/statuses/{target["commit"]}',state=state,
+       context=f'{CONTEXT}/{target["run_id"]}/{target["run_attempt"]}',description=description[:140],
+       target_url=f'https://github.com/{REPO}/actions/runs/{target["run_id"]}')
+
+
+def choose(pending, candidate, previous):
+    # Finish a handoff already in progress before starting another release.
+    if pending:return pending
+    if not candidate:return None
+    if previous:
+        before=(previous['run_id'],previous['run_attempt'])
+        after=(candidate['run_id'],candidate['run_attempt'])
+        if after<before:raise RuntimeError('Refuse an older release')
+        if after==before and previous.get('state')=='success':return None
+    return candidate
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--apply',action='store_true')
+    args=parser.parse_args()
+    os.umask(0o077);STATE.mkdir(parents=True,exist_ok=True)
+    with (STATE/'agent.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        read=lambda name:json.loads((STATE/name).read_text()) if (STATE/name).exists() else {}
+        pending, previous=read('pending.json'),read('latest.json')
+        target=choose(pending,None if pending else latest(),previous)
+        if not target:return
+        if not args.apply:print(json.dumps(target));return
+        save_progress(STATE/'pending.json',target)
+        directory=STATE/f'{target["run_id"]}-{target["run_attempt"]}'
+        directory.mkdir(exist_ok=True)
+        report(target,'pending','Automatically deploying this Fief build to production')
+        try:
+            result=deploy(target,directory)
+            receipt={**target,**result,'state':'success','updated_at':time.time()}
+            report(target,'success','Exact Fief image verified in production')
+            save_progress(STATE/'latest.json',receipt)
+            (STATE/'pending.json').unlink()
+        except Exception as error:
+            save_progress(STATE/'latest.json',{**target,'state':'failure','error_type':type(error).__name__,'updated_at':time.time()})
+            report(target,'failure','Automatic deployment will retry: '+type(error).__name__)
+            raise
+
+
+if __name__=='__main__':main()

scripts/hosting_release_manifest.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Bind a release to this build's amd64 manifest, excluding provenance attestations."""
+import json
+import os
+import subprocess
+
+
+def platform_digest(document, digest):
+    if 'manifests' not in document:return digest
+    matches=[m['digest'] for m in document['manifests'] if m.get('platform')=={'architecture':'amd64','os':'linux'}]
+    if len(matches)!=1:raise ValueError('Expected exactly one linux/amd64 image')
+    return matches[0]
+
+
+def main():
+    digest=os.environ['BUILD_DIGEST']
+    repository='ghcr.io/opensensor/fief'
+    document=json.loads(subprocess.check_output(['docker','buildx','imagetools','inspect',repository+'@'+digest,'--raw']))
+    target={'format':'fief-release-v1','repository':os.environ['GITHUB_REPOSITORY'],
+        'commit':os.environ['GITHUB_SHA'],'run_id':int(os.environ['GITHUB_RUN_ID']),
+        'run_attempt':int(os.environ['GITHUB_RUN_ATTEMPT']),
+        'image':repository+'@'+platform_digest(document,digest)}
+    with open('hosting-release.json','w') as out:json.dump(target,out)
+
+
+if __name__=='__main__':main()

scripts/hosting_runtime.py

@@ -0,0 +1,64 @@
+"""Roll out the latest Fief build without replacing its production configuration."""
+import json
+import subprocess
+import time
+
+CLOUD='do-nyc1-k8s-1-33-1-do-1-nyc1-1752275240114'
+NS='whitewhale'
+NAME='opensensor-fief'
+
+
+def kube(*args):
+    return subprocess.check_output(['kubectl','--context',CLOUD,'-n',NS,*args],text=True,timeout=650)
+
+
+def get(*args):return json.loads(kube('get',*args,'-o','json'))
+
+
+def verify(deployment,pods,image,commit):
+    spec,status=deployment['spec'],deployment.get('status',{})
+    count=spec.get('replicas',0)
+    if count<1 or status.get('observedGeneration',0)<deployment['metadata']['generation']:
+        raise RuntimeError('Fief rollout not observed')
+    if any(status.get(k,0)!=count for k in ('replicas','updatedReplicas','readyReplicas')):
+        raise RuntimeError('Fief rollout not complete')
+    if len(pods)!=count:raise RuntimeError('Old Fief pods remain')
+    for pod in pods:
+        if pod['metadata'].get('deletionTimestamp'):raise RuntimeError('Fief pod still terminating')
+        if pod['metadata'].get('annotations',{}).get('opensensor.io/source-revision')!=commit:
+            raise RuntimeError('Wrong Fief source revision')
+        containers={c['name']:c for c in pod['spec']['containers']}
+        statuses={c['name']:c for c in pod.get('status',{}).get('containerStatuses',[])}
+        c=containers.get(NAME,{});s=statuses.get(NAME,{})
+        if c.get('image')!=image or not s.get('ready') or not s.get('imageID','').endswith(image.split('@')[1]):
+            raise RuntimeError('Fief actual running image differs from this build')
+
+
+def deploy(target,directory):
+    deployment=get('deployment',NAME)
+    template=deployment['spec']['template']
+    annotations=template.setdefault('metadata',{}).setdefault('annotations',{})
+    current=int(annotations.get('opensensor.io/build-run','0'))
+    if current>target['run_id']:raise RuntimeError('A newer Fief build is already deployed')
+    annotations.update({'opensensor.io/source-revision':target['commit'],'opensensor.io/build-run':str(target['run_id'])})
+    containers=template['spec']['containers']
+    matches=[c for c in containers if c['name']==NAME]
+    if len(matches)!=1:raise RuntimeError('Fief container identity changed')
+    matches[0]['image']=target['image']
+    patch=[{'op':'test','path':'/metadata/resourceVersion','value':deployment['metadata']['resourceVersion']},
+           {'op':'replace','path':'/spec/template','value':template}]
+    kube('patch','deployment',NAME,'--type=json','-p',json.dumps(patch))
+    kube('rollout','status','deployment/'+NAME,'--timeout=600s')
+    deadline=time.monotonic()+120
+    while True:
+        actual=get('deployment',NAME)
+        selector=','.join(k+'='+v for k,v in actual['spec']['selector']['matchLabels'].items())
+        try:verify(actual,get('pods','-l',selector)['items'],target['image'],target['commit']);break
+        except RuntimeError:
+            if time.monotonic()>=deadline:raise
+            time.sleep(3)
+    return {'production_ready':True}
+
+
+def save_progress(path,state):
+    temp=path.with_suffix('.tmp');temp.write_text(json.dumps(state));temp.replace(path)

scripts/test_hosting_release.py

@@ -0,0 +1,32 @@
+import copy
+import unittest
+import hosting_release as agent
+from hosting_runtime import verify,NAME
+from hosting_release_manifest import platform_digest
+
+IMAGE='ghcr.io/opensensor/fief@sha256:'+'a'*64
+COMMIT='b'*40
+TARGET=dict(format='fief-release-v1',repository=agent.REPO,commit=COMMIT,run_id=100,run_attempt=1,image=IMAGE)
+WORKFLOW=dict(event='push',head_branch='main',path=agent.WORKFLOW,repository={'full_name':agent.REPO},head_sha=COMMIT,id=100,run_attempt=1)
+
+class ReleaseTests(unittest.TestCase):
+    def test_release_belongs_to_main_build(self):
+        self.assertEqual(agent.validate(TARGET,WORKFLOW),TARGET)
+        for key,value in [('run_id',99),('run_attempt',2),('image','ghcr.io/opensensor/fief:latest')]:
+            with self.subTest(key=key),self.assertRaises(ValueError):agent.validate({**TARGET,key:value},WORKFLOW)
+        with self.assertRaises(ValueError):agent.validate(TARGET,{**WORKFLOW,'event':'pull_request'})
+    def test_older_build_cannot_replace_newer_build(self):
+        with self.assertRaises(RuntimeError):agent.choose({},TARGET,{**TARGET,'run_id':101,'state':'success'})
+    def test_ready_old_image_is_rejected(self):
+        deployment={'metadata':{'generation':1},'spec':{'replicas':1},'status':{'observedGeneration':1,'replicas':1,'updatedReplicas':1,'readyReplicas':1}}
+        pod={'metadata':{'annotations':{'opensensor.io/source-revision':COMMIT}},'spec':{'containers':[{'name':NAME,'image':IMAGE}]},'status':{'containerStatuses':[{'name':NAME,'ready':True,'imageID':IMAGE}]}}
+        verify(deployment,[pod],IMAGE,COMMIT)
+        old=copy.deepcopy(pod);old['status']['containerStatuses'][0]['imageID']='old'
+        with self.assertRaises(RuntimeError):verify(deployment,[old],IMAGE,COMMIT)
+        with self.assertRaises(RuntimeError):verify(deployment,[pod,old],IMAGE,COMMIT)
+    def test_platform_digest_excludes_attestation(self):
+        doc={'manifests':[{'platform':{'os':'unknown','architecture':'unknown'},'digest':'other'},
+            {'platform':{'os':'linux','architecture':'amd64'},'digest':'runtime'}]}
+        self.assertEqual(platform_digest(doc,'index'),'runtime')
+
+if __name__=='__main__':unittest.main()

scripts/wait_hosting_release.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Keep CI pending until this build is verified on production."""
+import json
+import os
+import subprocess
+import time
+
+
+def main():
+    repo, commit = os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_SHA']
+    context = f'hosting/fief-release/{os.environ["GITHUB_RUN_ID"]}/{os.environ["GITHUB_RUN_ATTEMPT"]}'
+    deadline = time.monotonic() + 2100
+    while time.monotonic() < deadline:
+        raw = subprocess.check_output(['gh','api','--paginate','--slurp','-H','Cache-Control: no-cache',
+            f'repos/{repo}/commits/{commit}/statuses?per_page=100'],timeout=60)
+        statuses = [s for page in json.loads(raw) for s in page if s['context']==context]
+        if statuses:
+            status = max(statuses,key=lambda s:s['id'])
+            print(status['state']+': '+status['description'],flush=True)
+            if status['state']=='success':return
+            # The host resumes failed handoffs automatically; keep this run open for its retry.
+        else:print('Waiting for the production release agent',flush=True)
+        time.sleep(20)
+    raise SystemExit('This build has not been verified in production; inspect hosting/fief-release')
+
+
+if __name__=='__main__':main()