feat: airgaps

Makefile

@@ -17,7 +17,35 @@ cache-keys: ## Download Red Hat signing keys from official sources to ~/.coco-pa
 	@echo "Done. Verify fingerprints at https://access.redhat.com/security/team/key/"
 
 
+.PHONY: cache-registry-ca
+cache-registry-ca: ## Copy registry CA certificate chain to ~/.coco-pattern/
+	@mkdir -p ~/.coco-pattern
+	@if [ -f ~/quay-ca-chain.pem ]; then \
+		cp ~/quay-ca-chain.pem ~/.coco-pattern/quay-ca-cert.pem; \
+		echo "  Cached at ~/.coco-pattern/quay-ca-cert.pem"; \
+	else \
+		echo "ERROR: ~/quay-ca-chain.pem not found."; \
+		echo "  Copy your registry CA certificate chain to ~/quay-ca-chain.pem first."; \
+		exit 1; \
+	fi
+
+.PHONY: pck-register
+pck-register: ## Register PCK certificates with Intel PCS (requires INTEL_PCS_API_KEY)
+	@if [ -z "$(INTEL_PCS_API_KEY)" ]; then \
+		echo "ERROR: Set INTEL_PCS_API_KEY environment variable"; \
+		echo "  Usage: make pck-register INTEL_PCS_API_KEY=<key>"; \
+		exit 1; \
+	fi
+	@PCS_TOOL="$(HOME)/confidential-computing.tee.dcap/tools/PcsClientTool/pcsclient.py"; \
+	if [ ! -f "$$PCS_TOOL" ]; then \
+		echo "ERROR: PCS Client Tool not found at $$PCS_TOOL"; \
+		echo "  Clone: git clone https://github.com/intel/confidential-computing.tee.dcap ~/confidential-computing.tee.dcap"; \
+		exit 1; \
+	fi; \
+	cd "$$(dirname $$PCS_TOOL)" && python3 pcsclient.py -t register -k "$(INTEL_PCS_API_KEY)"
+
 ##@ Reference Value Collection
+
 .PHONY: collect-firmware-refvals
 collect-firmware-refvals: ## Collect firmware reference values (bare metal, default)
 	@scripts/collect-firmware-refvals.sh
@@ -26,6 +54,106 @@ collect-firmware-refvals: ## Collect firmware reference values (bare metal, defa
 collect-azure-refvals: ## Collect PCR reference values (Azure)
 	@scripts/collect-firmware-refvals.sh --platform azure
 
+.PHONY: collect-dcap-collateral
+collect-dcap-collateral: ## Collect TDX DCAP collateral from Intel PCS (API key via OS keyring)
+	@scripts/collect-dcap-collateral.sh
+
+.PHONY: dcap-offline-provision
+dcap-offline-provision: ## Full DCAP offline provisioning workflow (collect collateral + load secrets)
+	$(MAKE) collect-dcap-collateral
+	$(MAKE) load-secrets
+
+##@ AMD SEV-SNP VCEK Provisioning
+
+.PHONY: snp-collect-vcek-urls
+snp-collect-vcek-urls: ## Collect VCEK URLs from AMD SNP nodes (requires KUBECONFIG)
+	@scripts/collect-snp-vcek-urls.sh
+
+.PHONY: snp-download-vcek
+snp-download-vcek: ## Download VCEK certs from AMD KDS (requires internet)
+	@scripts/download-snp-vcek.sh
+
+.PHONY: snp-gen-overrides
+snp-gen-overrides: ## Generate SNP VCEK values override from cached certs (local only)
+	@scripts/gen-snp-vcek-overrides.sh
+
+.PHONY: snp-offline-provision
+snp-offline-provision: snp-collect-vcek-urls snp-download-vcek snp-gen-overrides ## Full SNP VCEK offline provisioning
+
+##@ Disconnected Deployment
+MIRROR_REGISTRY ?= quay.example.com:443/mirror
+IMAGESET_CONFIG ?= airgap/imageset-config.yaml
+OC_MIRROR_WORKSPACE ?= file://$(HOME)/oc-mirror-workspace
+
+.PHONY: airgap-mirror
+airgap-mirror: ## Mirror content to disconnected registry (requires MIRROR_REGISTRY)
+	oc-mirror -c $(IMAGESET_CONFIG) \
+		--workspace $(OC_MIRROR_WORKSPACE) \
+		--dest-tls-verify=false \
+		docker://$(MIRROR_REGISTRY) --v2
+
+.PHONY: airgap-post-install
+airgap-post-install: ## Post-install bootstrap (after airgap-mirror + labctl apply-mirror-resources)
+	@scripts/airgap-post-install.sh
+
+.PHONY: airgap-deploy-pattern
+airgap-deploy-pattern: ## Deploy Pattern CR directly (skip pattern.sh, use values-global.yaml config)
+	@scripts/airgap-post-install.sh --deploy-pattern
+
+.PHONY: airgap-fix-manifests
+airgap-fix-manifests: ## Fix oc-mirror manifest list failures with fallback mirroring
+	@scripts/airgap-post-install.sh --fix-manifest-lists
+
+.PHONY: airgap-sync-repos
+airgap-sync-repos: ## Push working copy changes to bare HTTP repos and restart git server
+	@scripts/airgap-post-install.sh --sync-repos-only
+
+ARGOCD_CLI_DIR ?= $(HOME)/.local/bin
+
+.PHONY: argocd-install
+argocd-install: ## Download argocd CLI from the cluster's ArgoCD image into ~/.local/bin
+	@mkdir -p $(ARGOCD_CLI_DIR)
+	@ARGOCD_NS=$$(oc get argocd -A -o jsonpath='{.items[0].metadata.namespace}') || \
+		{ echo "ERROR: No ArgoCD instance found. Is KUBECONFIG set and the pattern deployed?"; exit 1; }; \
+	ARGOCD_IMG=$$(oc get deployment -n $$ARGOCD_NS -l app.kubernetes.io/component=server,app.kubernetes.io/part-of=argocd \
+		-o jsonpath='{.items[0].spec.template.spec.containers[0].image}') || \
+		{ echo "ERROR: Could not find ArgoCD server deployment in $$ARGOCD_NS"; exit 1; }; \
+	echo "Extracting argocd CLI from $$ARGOCD_IMG ..."; \
+	POD_NAME="argocd-cli-extract-$$$$"; \
+	oc run "$$POD_NAME" -n $$ARGOCD_NS --image="$$ARGOCD_IMG" \
+		--restart=Never --command -- sleep 300; \
+	echo "Waiting for extract pod..."; \
+	oc wait --for=condition=Ready pod/"$$POD_NAME" -n $$ARGOCD_NS --timeout=120s; \
+	oc cp "$$ARGOCD_NS/$$POD_NAME:/usr/local/bin/argocd" "$(ARGOCD_CLI_DIR)/argocd"; \
+	chmod +x "$(ARGOCD_CLI_DIR)/argocd"; \
+	oc delete pod "$$POD_NAME" -n $$ARGOCD_NS --force --grace-period=0 2>/dev/null; \
+	echo "Installed: $(ARGOCD_CLI_DIR)/argocd"; \
+	$(ARGOCD_CLI_DIR)/argocd version --client | head -1
+
+.PHONY: argocd-login
+argocd-login: ## Extract ArgoCD credentials from cluster and log in with argocd CLI
+	@ARGOCD_NS=$$(oc get argocd -A -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null) && \
+	ARGOCD_NAME=$$(oc get argocd -n $$ARGOCD_NS -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
+	ARGOCD_ROUTE=$$(oc get route $${ARGOCD_NAME}-server -n $$ARGOCD_NS -o jsonpath='{.spec.host}' 2>/dev/null) && \
+	ARGOCD_PASS=$$(oc get secret $${ARGOCD_NAME}-cluster -n $$ARGOCD_NS -o jsonpath='{.data.admin\.password}' 2>/dev/null | base64 -d) && \
+	echo "ArgoCD URL:  https://$$ARGOCD_ROUTE" && \
+	echo "Username:    admin" && \
+	echo "Password:    $$ARGOCD_PASS" && \
+	echo "" && \
+	if command -v argocd >/dev/null 2>&1; then \
+		oc config set-context --current --namespace=$$ARGOCD_NS && \
+		argocd login --core && \
+		echo "" && \
+		echo "Logged in (core mode, namespace $$ARGOCD_NS)." && \
+		echo "Usage: argocd app list"; \
+	else \
+		echo "argocd CLI not installed. To install:" && \
+		echo "  curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64" && \
+		echo "  chmod +x /usr/local/bin/argocd" && \
+		echo "" && \
+		echo "Or use the web UI at: https://$$ARGOCD_ROUTE"; \
+	fi
+
 ##@ Hardware Detection
 .PHONY: detect-hardware
 detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECONFIG or oc login)
@@ -59,3 +187,16 @@ detect-hardware: ## Detect hardware profile from cluster nodes (requires KUBECON
 	fi && \
 	echo "" && \
 	echo "To apply: edit values-global.yaml and set global.hardware.profile to the recommended value."
+
+##@ Chart Management
+KYVERNO_VERSION ?= 3.7.2
+KYVERNO_REPO ?= https://kyverno.github.io/kyverno/
+
+.PHONY: update-kyverno-chart
+update-kyverno-chart: ## Pull and embed upstream Kyverno Helm chart (requires internet)
+	@echo "Pulling kyverno chart v$(KYVERNO_VERSION)..."
+	@rm -rf charts/all/kyverno
+	@helm pull kyverno --repo $(KYVERNO_REPO) --version $(KYVERNO_VERSION) --untar -d charts/all/
+	@find charts/all/kyverno -name "README.md" -o -name "README.md.gotmpl" | xargs rm -f
+	@echo "Embedded charts/all/kyverno (v$(KYVERNO_VERSION))"
+	@echo "Next: commit, push, and sync ArgoCD"

airgap/imageset-config.yaml

@@ -0,0 +1,162 @@
+---
+# ImageSetConfiguration for CoCo Validated Pattern airgap deployment
+# Target: quay.apac-tech-lab.net
+# OCP: 4.21.24
+# Topology: baremetal (full operator set from values-baremetal.yaml)
+#
+# Usage:
+#   # Mirror to mirror (jump host has both internet and quay access):
+#   oc-mirror -c airgap/imageset-config.yaml \
+#     --workspace file:///home/chbutler/oc-mirror-workspace \
+#     docker://quay.apac-tech-lab.net/ocp-mirror --v2
+#
+#   # Or mirror to disk first, then disk to mirror:
+#   oc-mirror -c airgap/imageset-config.yaml \
+#     file:///home/chbutler/oc-mirror-archive --v2
+#   oc-mirror -c airgap/imageset-config.yaml \
+#     --from file:///home/chbutler/oc-mirror-archive \
+#     docker://quay.apac-tech-lab.net/ocp-mirror --v2
+
+apiVersion: mirror.openshift.io/v2alpha1
+kind: ImageSetConfiguration
+mirror:
+  # OCP release — single version
+  platform:
+    channels:
+      - name: fast-4.21
+        minVersion: "4.21.24"
+        maxVersion: "4.21.24"
+        type: ocp
+
+  # Red Hat operators (from values-baremetal.yaml subscriptions)
+  operators:
+    - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.21
+      packages:
+        # CoCo core
+        - name: sandboxed-containers-operator
+          channels:
+            - name: stable
+              minVersion: "1.13.0"
+              maxVersion: "1.13.0"
+        - name: trustee-operator
+          channels:
+            - name: stable
+              minVersion: "1.2.0"
+              maxVersion: "1.2.0"
+        # GitOps (ArgoCD)
+        - name: openshift-gitops-operator
+          channels:
+            - name: latest
+        # External Secrets
+        - name: openshift-external-secrets-operator
+          channels:
+            - name: stable-v1
+        # Cert Manager
+        - name: openshift-cert-manager-operator
+          channels:
+            - name: stable-v1
+        # ACM — use default channel (release-2.17 in 4.21 catalog)
+        # filtering to non-default channel causes oc-mirror error
+        - name: advanced-cluster-management
+        # MCE — required dependency of ACM, must be mirrored explicitly
+        # ACM creates an MCE subscription; OLM needs the package in the catalog
+        - name: multicluster-engine
+        # Storage (LVM)
+        - name: lvms-operator
+          channels:
+            - name: stable-4.21
+        # CNV (KubeVirt)
+        - name: kubevirt-hyperconverged
+          channels:
+            - name: stable
+        # Node Feature Discovery
+        - name: nfd
+          channels:
+            - name: stable
+
+  # Certified operators
+    - catalog: registry.redhat.io/redhat/certified-operator-index:v4.21
+      packages:
+        - name: gpu-operator-certified
+          channels:
+            - name: v26.3
+              minVersion: "26.3.0"
+              maxVersion: "26.3.0"
+        # Intel device plugins (for TDX/SGX) — certified, NOT redhat catalog
+        - name: intel-device-plugins-operator
+          channels:
+            - name: stable
+        # Intel TDX DCAP operator — QGS lifecycle management (D-07)
+        - name: intel-tdx-dcap-operator
+          channels:
+            - name: alpha
+
+  # Community operators (patterns-operator is NOT in redhat-operator-index for 4.21)
+    - catalog: registry.redhat.io/redhat/community-operator-index:v4.21
+      packages:
+        - name: patterns-operator
+
+  # NOTE: Kyverno is deployed via Helm chart (kyverno.github.io/kyverno), NOT via
+  # OLM operator from the community catalog. Its images are in additionalImages below.
+
+  # Additional container images (application workloads)
+  additionalImages:
+    # Workload images from hello-openshift chart
+    - name: registry.redhat.io/ubi9/httpd-24:latest
+    # Vault — the VP Helm chart uses the Red Hat certified UBI image,
+    # NOT the Docker Hub image. Mirror from registry.connect.redhat.com.
+    - name: registry.connect.redhat.com/hashicorp/vault:1.21.4-ubi
+    # Kyverno (Helm chart, not OLM — images from ghcr.io)
+    - name: ghcr.io/kyverno/kyverno:v1.13.4
+    - name: ghcr.io/kyverno/kyvernopre:v1.13.4
+    - name: ghcr.io/kyverno/background-controller:v1.13.4
+    - name: ghcr.io/kyverno/cleanup-controller:v1.13.4
+    - name: ghcr.io/kyverno/reports-controller:v1.13.4
+
+    # --- VP Helm OCI charts (additionalImages — oc-mirror v2 handles these) ---
+    # Core VP charts (versions are latest matching glob in values-baremetal.yaml)
+    - name: quay.io/validatedpatterns/clustergroup:0.9.56
+    - name: quay.io/validatedpatterns/acm:0.1.27
+    - name: quay.io/validatedpatterns/hashicorp-vault:0.1.8
+    - name: quay.io/validatedpatterns/openshift-external-secrets:0.0.4
+    # External VP charts (also published to quay.io/validatedpatterns/)
+    - name: quay.io/validatedpatterns/trustee:0.9.0
+    - name: quay.io/validatedpatterns/sandboxed-containers:0.2.1
+    - name: quay.io/validatedpatterns/sandboxed-policies:0.2.0
+
+    # --- Kyverno Helm chart OCI artifact (chartVersion: 3.7.* in values-baremetal) ---
+    - name: ghcr.io/kyverno/charts/kyverno:3.7.2
+
+    # --- VP framework utility images ---
+    # IMPORTANT: The VP framework uses :v1 tags for imperative-container (not :latest).
+    # Mirror BOTH :latest and :v1 to cover all usage paths.
+    - name: quay.io/validatedpatterns/utility-container:latest
+    - name: quay.io/validatedpatterns/imperative-container:latest
+    - name: quay.io/validatedpatterns/imperative-container:v1
+
+    # --- VP patterns-operator UI catalog ---
+    # Required by the patterns-operator console plugin (not mirrored by oc-mirror).
+    - name: quay.io/validatedpatterns/pattern-ui-catalog:stable-v1
+
+    # --- UBI minimal (ArgoCD repo-server init container) ---
+    # ArgoCD init containers pull by :latest tag; IDMS only handles digests.
+    # Mirror the tag explicitly so ITMS can redirect.
+    - name: registry.redhat.io/ubi9/ubi-minimal:latest
+
+    # --- oc-debug support-tools (D-05: enables oc debug node/ on disconnected) ---
+    - name: registry.redhat.io/rhel9/support-tools:latest
+
+    # --- VP infrastructure images ---
+    - name: registry.redhat.io/openshift4/ose-cli:latest
+
+    # --- Intel DCAP chart images ---
+    # Digest-pinned (exact refs from intel-dcap chart templates).
+    # NOTE: osc-pccs-rhel9:latest and osc-tdx-qgs-rhel9:latest tags do not exist
+    # on registry.redhat.io — only digest-pinned refs work.
+    - name: registry.redhat.io/openshift-sandboxed-containers/osc-pccs@sha256:edf57087220516115512dc6687b96045cbc69472a3d6d381619f66beae032ad6
+    - name: registry.redhat.io/openshift-sandboxed-containers/osc-tdx-qgs@sha256:308d66da94d3116cc16530a4a2cd60a995458f331e1cb3487c6ea3fdae60545b
+    - name: registry.connect.redhat.com/intel/intel-sgx-plugin@sha256:4ac8769c4f0a82b3ea04cf1532f15e9935c71fe390ff5a9dc3ee57f970a65f0b
+
+    # --- Workload images ---
+    - name: registry.access.redhat.com/ubi9/ubi:latest
+    - name: registry.access.redhat.com/ubi9/ubi:9.7-1764578509

ansible/init-data-gzipper.yaml

@@ -35,6 +35,48 @@
         msg: "KBS TLS certificate not found in secret 'kbs-tls-self-signed' in namespace 'imperative'"
       when: kbs_secret_result.resources | length == 0
 
+    - name: Read registry CA cert from Kubernetes
+      kubernetes.core.k8s_info:
+        kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}"
+        api_version: v1
+        kind: Secret
+        name: registry-ca
+        namespace: imperative
+      register: ca_secret_result
+
+    - name: Extract and split CA certificates
+      ansible.builtin.shell: |
+        set -o pipefail
+        python3 -c "
+        import base64, sys
+        raw = base64.b64decode(sys.argv[1]).decode('utf-8')
+        certs = []
+        current = []
+        for line in raw.splitlines():
+            if line.strip() == '-----BEGIN CERTIFICATE-----':
+                current = [line]
+            elif line.strip() == '-----END CERTIFICATE-----':
+                current.append(line)
+                certs.append(chr(10).join(current))
+                current = []
+            elif current:
+                current.append(line)
+        print('---CERT_BOUNDARY---'.join(certs), end='')
+        " "{{ ca_secret_result.resources[0].data.pem }}"
+      register: ca_certs_raw
+      changed_when: false
+      when: ca_secret_result.resources | default([]) | length > 0
+
+    - name: Build CA certificate list
+      ansible.builtin.set_fact:
+        registry_ca_certs: "{{ ca_certs_raw.stdout.split('---CERT_BOUNDARY---') | select | list }}"
+      when: ca_certs_raw is defined and ca_certs_raw.rc is defined and ca_certs_raw.rc == 0
+
+    - name: Set empty CA cert list when secret not found
+      ansible.builtin.set_fact:
+        registry_ca_certs: []
+      when: ca_secret_result.resources | default([]) | length == 0
+
     - name: Define temp file paths
       ansible.builtin.set_fact:
         rendered_path: "{{ tmpdir.path }}/rendered.toml"

ansible/initdata-debug.toml.tpl

@@ -29,6 +29,10 @@ image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_f
 # CDH does NOT use this URI — registry auth is handled via imagePullSecrets
 # on the workload's service account instead. Kept here for baremetal support.
 authenticated_registry_credentials_uri = 'kbs:///default/credential/regcred'
+{% if registry_ca_certs | default([]) | length > 0 %}
+extra_root_certificates = [{% for cert in registry_ca_certs %}"""
+{{ cert }}"""{%- if not loop.last %}, {% endif %}{% endfor %}]
+{% endif %}
 '''
 
 "policy.rego" = '''

ansible/initdata-default.toml.tpl

@@ -29,6 +29,10 @@ image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_f
 # CDH does NOT use this URI — registry auth is handled via imagePullSecrets
 # on the workload's service account instead. Kept here for baremetal support.
 authenticated_registry_credentials_uri = 'kbs:///default/credential/regcred'
+{% if registry_ca_certs | default([]) | length > 0 %}
+extra_root_certificates = [{% for cert in registry_ca_certs %}"""
+{{ cert }}"""{%- if not loop.last %}, {% endif %}{% endfor %}]
+{% endif %}
 '''
 
 "policy.rego" = '''