Make the pack guide and template accurate for third-party pack authors

.github/workflows/lint.yaml

@@ -17,8 +17,12 @@ jobs:
 
       - name: Install kubeconform
         run: |
-          curl -sSL https://github.com/yannh/kubeconform/releases/latest/download/kubeconform-linux-amd64.tar.gz | \
-            sudo tar -xz -C /usr/local/bin kubeconform
+          KUBECONFORM_VERSION=v0.8.0
+          KUBECONFORM_SHA256=9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883
+          curl -sSLo /tmp/kubeconform.tar.gz \
+            "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz"
+          echo "${KUBECONFORM_SHA256}  /tmp/kubeconform.tar.gz" | sha256sum -c -
+          sudo tar -xzf /tmp/kubeconform.tar.gz -C /usr/local/bin kubeconform
 
       # Vanilla YAML example
       - name: Validate vanilla-yaml manifests
@@ -65,8 +69,8 @@ jobs:
         run: helm template test examples/auth-fastapi/chart/ --set nebariapp.enabled=true --set nebariapp.hostname=test.example.com
 
       # Wrap existing chart (Podinfo) Helm example
-      - name: Update podinfo dependency
-        run: helm dependency update examples/wrap-existing-chart/chart/
+      - name: Build wrap-existing-chart dependencies
+        run: helm dependency build examples/wrap-existing-chart/chart/
 
       - name: Lint wrap-existing-chart
         run: helm lint examples/wrap-existing-chart/chart/
@@ -76,3 +80,42 @@ jobs:
 
       - name: Template wrap-existing-chart (NebariApp enabled)
         run: helm template test examples/wrap-existing-chart/chart/ --set nebariapp.enabled=true --set nebariapp.hostname=test.example.com
+
+      - name: Template auth-fastapi (NetworkPolicy enabled)
+        run: helm template test examples/auth-fastapi/chart/ --set nebariapp.enabled=true --set nebariapp.hostname=test.example.com --set networkPolicy.enabled=true | kubeconform -strict -ignore-missing-schemas
+
+      # Without spec.routing the operator creates no HTTPRoute or TLS, yet the
+      # NebariApp still reports Ready. Fail fast if an example chart drops it.
+      - name: Every chart renders a NebariApp with routing
+        run: |
+          for c in basic-nginx auth-fastapi wrap-existing-chart; do
+            routes=$(helm template test examples/$c/chart/ --set nebariapp.enabled=true --set nebariapp.hostname=test.example.com \
+              | yq 'select(.kind == "NebariApp") | .spec.routing.routes | length')
+            if [ "${routes:-0}" -lt 1 ]; then echo "$c: NebariApp has no spec.routing.routes"; exit 1; fi
+            echo "$c: $routes route(s)"
+          done
+
+      # NIC locks the "default" AppProject to deny-all; packs must use nebari-apps.
+      - name: ArgoCD examples use the nebari-apps project
+        run: |
+          if grep -rn --include='*.md' -E '^\s*project:\s*default\b' README.md docs examples; then
+            echo "ArgoCD Application examples must use 'project: nebari-apps'"; exit 1
+          fi
+
+      - name: Operator version references match OPERATOR_REF
+        run: dev/check-operator-refs.sh
+
+  app-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Test auth-fastapi token verification
+        working-directory: examples/auth-fastapi
+        run: |
+          pip install -r app/requirements.txt -r tests/requirements.txt
+          pytest tests

.github/workflows/test-integration.yaml

@@ -14,9 +14,16 @@ jobs:
   nebariapp-integration:
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        # v0.1.1 is the release the template documents; v0.1.0-alpha.20 is what
+        # NIC v0.14.0 deploys, so packs are also proven on today's clusters.
+        operator-ref: [v0.1.1, v0.1.0-alpha.20]
+    name: nebariapp-integration (${{ matrix.operator-ref }})
     env:
       CLUSTER_NAME: pack-integration
-      OPERATOR_REF: v0.1.0-alpha.19
+      OPERATOR_REF: ${{ matrix.operator-ref }}
     steps:
       - name: Checkout software-pack-template
         uses: actions/checkout@v5
@@ -84,7 +91,7 @@ jobs:
         run: nebari-operator/dev/scripts/services/keycloak/setup.sh
 
       # ============================================================
-      # Install nebari-operator
+      # Install nebari-operator, configured the way NIC configures it
       # ============================================================
       - name: Install nebari-operator
         run: |
@@ -95,6 +102,9 @@ jobs:
           kubectl wait --for=condition=available deployment/nebari-operator-controller-manager \
             -n nebari-operator-system --timeout=120s
 
+      - name: Configure nebari-operator and expose Keycloak
+        run: dev/configure-operator.sh
+
       - name: Label default namespace for operator
         run: kubectl label namespace default nebari.dev/managed=true --overwrite
 
@@ -109,53 +119,48 @@ jobs:
             ghcr.io/nebari-dev/software-pack-template/auth-fastapi-example:latest \
             --name pack-integration
 
+      # Each "Verify" step waits for RoutingReady, TLSReady (and AuthReady with
+      # --auth), checks the generated HTTPRoute/SecurityPolicy exist, and sends a
+      # request through the Gateway. Ready=True alone is not enough: a NebariApp
+      # without spec.routing is Ready and unreachable.
+
       # ============================================================
       # Test: vanilla-yaml
       # ============================================================
       - name: "Deploy vanilla-yaml"
         run: |
-          # Apply deployment + service
           kubectl apply -f examples/vanilla-yaml/deployment.yaml \
                         -f examples/vanilla-yaml/service.yaml
-
-          # Apply nebariapp with hostname patched for nebari.local domain
           sed 's/my-pack.nebari.example.com/vanilla.nebari.local/' \
             examples/vanilla-yaml/nebariapp.yaml | kubectl apply -f -
 
-      - name: "Verify vanilla-yaml NebariApp"
+      - name: "Verify vanilla-yaml"
         run: |
           kubectl wait --for=condition=ready pod -l app=my-pack --timeout=120s
-          kubectl wait --for=condition=Ready nebariapp/my-pack --timeout=120s
-          kubectl get httproute -l app.kubernetes.io/managed-by=nebari-operator
-          echo "vanilla-yaml: NebariApp Ready"
+          dev/verify-nebariapp.sh my-pack vanilla.nebari.local
 
       - name: "Cleanup vanilla-yaml"
         if: always()
         run: |
           kubectl delete nebariapp my-pack --ignore-not-found --wait --timeout=60s
           kubectl delete -f examples/vanilla-yaml/deployment.yaml \
-                         -f examples/vanilla-yaml/service.yaml --ignore-not-found --wait
+                         -f examples/vanilla-yaml/service.yaml --ignore-not-found --wait \
+                         --cascade=foreground --timeout=60s
 
       # ============================================================
       # Test: kustomize-nginx (production overlay - auth enabled)
       # ============================================================
       - name: "Deploy kustomize-nginx (production)"
-        run: |
-          kubectl apply -k examples/kustomize-nginx/overlays/production/
+        run: kubectl apply -k examples/kustomize-nginx/overlays/production/
 
-      - name: "Verify kustomize-nginx NebariApp"
+      - name: "Verify kustomize-nginx"
         run: |
           kubectl wait --for=condition=ready pod -l app=my-pack --timeout=120s
-          kubectl wait --for=condition=Ready nebariapp/my-pack --timeout=120s
-          kubectl get httproute -l app.kubernetes.io/managed-by=nebari-operator
-          # Production overlay has auth enabled - verify SecurityPolicy
-          kubectl get securitypolicy
-          echo "kustomize-nginx (production): NebariApp Ready with auth"
+          dev/verify-nebariapp.sh my-pack my-pack.nebari.example.com --auth
 
       - name: "Cleanup kustomize-nginx"
         if: always()
-        run: |
-          kubectl delete -k examples/kustomize-nginx/overlays/production/ --ignore-not-found --wait --timeout=60s
+        run: kubectl delete -k examples/kustomize-nginx/overlays/production/ --ignore-not-found --wait --cascade=foreground --timeout=60s
 
       # ============================================================
       # Test: basic-nginx (Helm, no auth)
@@ -168,19 +173,15 @@ jobs:
             --set nebariapp.hostname=basic.nebari.local \
             --wait --timeout 2m
 
-      - name: "Verify basic-nginx NebariApp"
-        run: |
-          kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=test-basic --timeout=120s
-          kubectl wait --for=condition=Ready nebariapp/test-basic-my-pack --timeout=120s
-          kubectl get httproute -l app.kubernetes.io/managed-by=nebari-operator
-          echo "basic-nginx: NebariApp Ready"
+      - name: "Verify basic-nginx"
+        run: dev/verify-nebariapp.sh test-basic-my-pack basic.nebari.local
 
       - name: "Cleanup basic-nginx"
         if: always()
         run: helm uninstall test-basic --wait || true
 
       # ============================================================
-      # Test: auth-fastapi (Helm, auth enabled)
+      # Test: auth-fastapi (Helm, auth enabled, full login)
       # ============================================================
       - name: "Deploy auth-fastapi"
         run: |
@@ -189,16 +190,41 @@ jobs:
             --set nebariapp.enabled=true \
             --set nebariapp.hostname=fastapi.nebari.local \
             --set nebariapp.auth.enabled=true \
-            --wait --timeout 2m
+            --set oidc.jwksURL=http://keycloak-keycloakx-http.keycloak.svc.cluster.local/auth/realms/nebari/protocol/openid-connect/certs \
+            --set networkPolicy.enabled=true \
+            --wait --timeout 3m
 
-      - name: "Verify auth-fastapi NebariApp"
+      - name: "Verify auth-fastapi"
         run: |
-          kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=test-fastapi --timeout=120s
-          kubectl wait --for=condition=Ready nebariapp/test-fastapi-my-pack --timeout=120s
-          kubectl get httproute -l app.kubernetes.io/managed-by=nebari-operator
-          # Auth enabled - verify SecurityPolicy
-          kubectl get securitypolicy
-          echo "auth-fastapi: NebariApp Ready with auth"
+          dev/verify-nebariapp.sh test-fastapi-my-pack fastapi.nebari.local --auth
+          # Log in through Keycloak and check the app shows a verified identity
+          dev/login-test.sh fastapi.nebari.local
+
+      - name: "Verify auth-fastapi rejects forged tokens off the gateway"
+        run: |
+          # An unsigned token sent straight to the Service must not be trusted.
+          # With the NetworkPolicy on, the request should not connect at all;
+          # temporarily remove it to check the app's own verification as well.
+          FORGED='eyJhbGciOiJub25lIn0.eyJwcmVmZXJyZWRfdXNlcm5hbWUiOiJtYWxsb3J5In0.'
+          probe() {
+            kubectl run forged-probe --rm -i --restart=Never --quiet --image=curlimages/curl:8.11.1 -- \
+              sh -c "curl -s -m 5 -H 'Cookie: IdToken-a1b2c3d4=$FORGED' http://test-fastapi-my-pack.default.svc.cluster.local/; echo curl_exit=\$?"
+          }
+          out=$(probe || true)
+          echo "$out" | grep -q 'curl_exit=28' || { echo "NetworkPolicy did not block direct access"; echo "$out"; exit 1; }
+          kubectl delete networkpolicy test-fastapi-my-pack --wait
+          # The CNI can take a few seconds to drop the rule after the delete.
+          for attempt in 1 2 3 4 5 6; do
+            out=$(probe || true)
+            echo "$out" | grep -q 'curl_exit=28' || break
+            echo "probe $attempt still blocked, retrying"; sleep 5
+          done
+          if echo "$out" | grep -q 'curl_exit=28'; then
+            echo "NetworkPolicy still blocks the Service after it was deleted"; echo "$out"; exit 1
+          fi
+          echo "$out" | grep -q 'could not be verified' || { echo "forged token was not rejected"; echo "$out"; exit 1; }
+          if echo "$out" | grep -q mallory; then echo "forged identity was displayed"; exit 1; fi
+          echo "forged token rejected"
 
       - name: "Cleanup auth-fastapi"
         if: always()
@@ -209,20 +235,18 @@ jobs:
       # ============================================================
       - name: "Deploy wrap-existing-chart (podinfo)"
         run: |
-          helm dependency update examples/wrap-existing-chart/chart/
+          helm dependency build examples/wrap-existing-chart/chart/
           helm install test-wrap examples/wrap-existing-chart/chart/ \
             --set nebariapp.enabled=true \
             --set nebariapp.hostname=podinfo.nebari.local \
             --wait --timeout 3m
 
-      - name: "Verify wrap-existing-chart NebariApp"
+      - name: "Verify wrap-existing-chart"
         run: |
           # Podinfo's chart labels its pods with app.kubernetes.io/name=<release>-podinfo
           # and does not set app.kubernetes.io/instance, so wait on the Deployment instead.
           kubectl wait --for=condition=available deployment/test-wrap-podinfo --timeout=120s
-          kubectl wait --for=condition=Ready nebariapp/test-wrap-my-pack --timeout=120s
-          kubectl get httproute -l app.kubernetes.io/managed-by=nebari-operator
-          echo "wrap-existing-chart: NebariApp Ready"
+          dev/verify-nebariapp.sh test-wrap-my-pack podinfo.nebari.local
 
       - name: "Cleanup wrap-existing-chart"
         if: always()
@@ -244,12 +268,15 @@ jobs:
           kubectl get httproute -A || true
           echo ""
           echo "=== SecurityPolicies ==="
-          kubectl get securitypolicy -A || true
+          kubectl get securitypolicy -A -o yaml | grep -A6 'conditions:' || true
           echo ""
           echo "=== Operator logs ==="
           kubectl logs -n nebari-operator-system deployment/nebari-operator-controller-manager \
             --tail=100 || true
           echo ""
+          echo "=== Envoy Gateway logs ==="
+          kubectl logs -n envoy-gateway-system deployment/envoy-gateway --tail=50 || true
+          echo ""
           echo "=== Pods ==="
           kubectl get pods -A || true
           echo ""

.github/workflows/test.yaml

@@ -116,8 +116,8 @@ jobs:
         run: helm uninstall test-fastapi -n test-fastapi --wait
 
       # Test podinfo wrapper Helm example
-      - name: Update podinfo dependency
-        run: helm dependency update examples/wrap-existing-chart/chart/
+      - name: Build wrap-existing-chart dependencies
+        run: helm dependency build examples/wrap-existing-chart/chart/
 
       - name: Install wrap-existing-chart
         run: |