fix: remove encryptedPort from Modal

config/webhook/kustomization.yaml

@@ -5,11 +5,14 @@ resources:
 # but never Creates it, so without this the manager crash-loops. See secret.yaml.
 - secret.yaml
 
-# Narrow the generated Pod webhook to opt-in Pods outside system namespaces.
+# Narrow the generated Pod webhooks to opt-in Pods outside system namespaces. The patches
+# live in a subdirectory because envtest installs every webhook-kind file at THIS level
+# (non-recursively), and a patch read as an object breaks its setup.
 patches:
-- path: selector_patch.yaml
+- path: patches/mutating_pod_selector.yaml
   target:
     kind: MutatingWebhookConfiguration
+- path: patches/validating_pod_selector.yaml
 
 configurations:
 - kustomizeconfig.yaml

config/webhook/manifests.yaml

@@ -36,7 +36,7 @@ webhooks:
       name: webhook-service
       namespace: system
       path: /validate-nebula-inftyai-com-v1alpha1-nodepool
-  failurePolicy: Fail
+  failurePolicy: Ignore
   name: vnodepool-v1alpha1.nebula.inftyai.com
   rules:
   - apiGroups:
@@ -49,3 +49,23 @@ webhooks:
     resources:
     - nodepools
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate--v1-pod
+  failurePolicy: Fail
+  name: vpod-v1.nebula.inftyai.com
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None

config/webhook/patches/validating_pod_selector.yaml

@@ -0,0 +1,22 @@
+# The same narrowing as mutating_pod_selector.yaml, for the Pod VALIDATING webhook: it is
+# just as failurePolicy=Fail and on the Pod-CREATE path. Merged by webhook name rather than
+# by index, because this configuration also holds the NodePool webhook, which must stay
+# unselected.
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- name: vpod-v1.nebula.inftyai.com
+  objectSelector:
+    matchLabels:
+      nebula.inftyai.com/enabled: "true"
+  namespaceSelector:
+    matchExpressions:
+      - key: kubernetes.io/metadata.name
+        operator: NotIn
+        values:
+          - kube-system
+          - kube-node-lease
+          - kube-public
+          - nebula-system

internal/webhook/v1/nodepool_webhook.go

@@ -36,7 +36,9 @@ func SetupNodePoolWebhookWithManager(mgr ctrl.Manager) error {
 		Complete()
 }
 
-// +kubebuilder:webhook:path=/validate-nebula-inftyai-com-v1alpha1-nodepool,mutating=false,failurePolicy=fail,sideEffects=None,groups=nebula.inftyai.com,resources=nodepools,verbs=create;update,versions=v1alpha1,name=vnodepool-v1alpha1.nebula.inftyai.com,admissionReviewVersions=v1
+// failurePolicy=ignore: the validator only warns, so a webhook outage must not block
+// NodePool writes. The Pod validator enforces runtime limits and stays Fail.
+// +kubebuilder:webhook:path=/validate-nebula-inftyai-com-v1alpha1-nodepool,mutating=false,failurePolicy=ignore,sideEffects=None,groups=nebula.inftyai.com,resources=nodepools,verbs=create;update,versions=v1alpha1,name=vnodepool-v1alpha1.nebula.inftyai.com,admissionReviewVersions=v1
 
 // NodePoolCustomValidator validates NodePools on create and update. A setting a listed
 // provider cannot serve (Spot on Modal, a restricted egress on RunPod) is NOT rejected:

internal/webhook/v1/pod_webhook.go

@@ -25,6 +25,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1"
 )
@@ -35,6 +36,7 @@ var podlog = logf.Log.WithName("pod-webhook")
 func SetupPodWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).For(&corev1.Pod{}).
 		WithDefaulter(&PodCustomDefaulter{}).
+		WithValidator(&PodCustomValidator{}).
 		Complete()
 }
 
@@ -95,7 +97,7 @@ func (d *PodCustomDefaulter) Default(_ context.Context, obj runtime.Object) erro
 		})
 	}
 
-	if hasGate(pod, nebulav1alpha1.ProviderSelectionGate) {
+	if hasNebulaGate(pod) {
 		return nil // already gated; nothing more to do
 	}
 
@@ -107,10 +109,91 @@ func (d *PodCustomDefaulter) Default(_ context.Context, obj runtime.Object) erro
 	return nil
 }
 
-// hasGate reports whether the Pod already carries the named scheduling gate.
-func hasGate(pod *corev1.Pod, name string) bool {
+// +kubebuilder:webhook:path=/validate--v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod-v1.nebula.inftyai.com,admissionReviewVersions=v1
+
+// PodCustomValidator rejects an opted-in Pod the providers cannot run as written: they
+// launch one container and no init containers, and expose at most one port (Modal's
+// connect URL routes to one; see modal.firstPort). Admitting more would drop the rest
+// silently.
+type PodCustomValidator struct{}
+
+var _ webhook.CustomValidator = &PodCustomValidator{}
+
+// ValidateCreate implements webhook.CustomValidator.
+func (v *PodCustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return nil, fmt.Errorf("expected a Pod object but got %T", obj)
+	}
+	return nil, validatePod(pod)
+}
+
+// ValidateUpdate implements webhook.CustomValidator. The shape is immutable but the opt-in
+// label is not, so only label transitions are judged: opting in is refused outright, and
+// opting out must not strand Nebula's gate. A Pod staying opted in is left alone: one
+// admitted before this check existed would otherwise have every later write rejected,
+// placement's gate release among them, and stay gated forever.
+func (v *PodCustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) {
+	oldPod, ok := oldObj.(*corev1.Pod)
+	if !ok {
+		return nil, fmt.Errorf("expected a Pod object but got %T", oldObj)
+	}
+	newPod, ok := newObj.(*corev1.Pod)
+	if !ok {
+		return nil, fmt.Errorf("expected a Pod object but got %T", newObj)
+	}
+	if optedIn(oldPod) {
+		// Placement ignores a Pod without the label, so it would never release the gate.
+		if !optedIn(newPod) && hasNebulaGate(newPod) {
+			return nil, fmt.Errorf("removing label %s would leave the Pod gated forever: also remove "+
+				"scheduling gate %q in the same update, or keep the label",
+				nebulav1alpha1.EnabledLabel, nebulav1alpha1.ProviderSelectionGate)
+		}
+		return nil, nil
+	}
+	// Opting in is CREATE-only: the defaulter's gate and toleration cannot be added later,
+	// and checking for hand-copied ones would drift whenever the defaulter adds another.
+	if optedIn(newPod) {
+		return nil, fmt.Errorf("opt in at creation: label %s takes effect only on a new Pod",
+			nebulav1alpha1.EnabledLabel)
+	}
+	return nil, nil
+}
+
+// ValidateDelete implements webhook.CustomValidator. Not registered for deletes.
+func (v *PodCustomValidator) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+func optedIn(pod *corev1.Pod) bool {
+	return pod.Labels[nebulav1alpha1.EnabledLabel] == nebulav1alpha1.EnabledValue
+}
+
+// validatePod enforces PodCustomValidator's rules on an opted-in Pod placement would see.
+func validatePod(pod *corev1.Pod) error {
+	if !optedIn(pod) {
+		return nil
+	}
+	if pod.Spec.NodeName != "" {
+		return nil // pre-bound: never placed, so never run by a provider (see needsPlacement)
+	}
+	if n := len(pod.Spec.Containers); n != 1 {
+		return fmt.Errorf("nebula runs exactly one container per Pod, got %d", n)
+	}
+	// Native sidecars are init containers too, so this rejects them as well.
+	if n := len(pod.Spec.InitContainers); n > 0 {
+		return fmt.Errorf("nebula does not run init containers, got %d", n)
+	}
+	if c := pod.Spec.Containers[0]; len(c.Ports) > 1 {
+		return fmt.Errorf("nebula exposes at most one port per Pod, container %q declares %d", c.Name, len(c.Ports))
+	}
+	return nil
+}
+
+// hasNebulaGate reports whether the Pod carries Nebula's provider-selection scheduling gate.
+func hasNebulaGate(pod *corev1.Pod) bool {
 	for _, g := range pod.Spec.SchedulingGates {
-		if g.Name == name {
+		if g.Name == nebulav1alpha1.ProviderSelectionGate {
 			return true
 		}
 	}

internal/webhook/v1/pod_webhook_test.go

@@ -18,6 +18,7 @@ package v1
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
@@ -27,7 +28,7 @@ import (
 )
 
 func gated(pod *corev1.Pod) bool {
-	return hasGate(pod, nebulav1alpha1.ProviderSelectionGate)
+	return hasNebulaGate(pod)
 }
 
 func podWith(labels map[string]string, nodeName string, gates ...string) *corev1.Pod {
@@ -172,3 +173,118 @@ func TestDefault_RejectsNonPod(t *testing.T) {
 		t.Fatal("expected an error for a non-Pod object")
 	}
 }
+
+func TestValidateCreate_OneContainerOnePort(t *testing.T) {
+	optedIn := map[string]string{nebulav1alpha1.EnabledLabel: "true"}
+	container := func(name string, ports ...int32) corev1.Container {
+		c := corev1.Container{Name: name}
+		for _, p := range ports {
+			c.Ports = append(c.Ports, corev1.ContainerPort{ContainerPort: p})
+		}
+		return c
+	}
+	cases := []struct {
+		name       string
+		labels     map[string]string
+		containers []corev1.Container
+		inits      []corev1.Container
+		wantErr    string
+	}{
+		{"one container, one port", optedIn, []corev1.Container{container("main", 8080)}, nil, ""},
+		{"one container, no port", optedIn, []corev1.Container{container("main")}, nil, ""},
+		{"two ports", optedIn, []corev1.Container{container("main", 8080, 9090)}, nil, "at most one port"},
+		{"two containers", optedIn, []corev1.Container{container("main", 8080), container("sidecar")}, nil, "exactly one container"},
+		{"no container", optedIn, nil, nil, "exactly one container"},
+		{"init container", optedIn, []corev1.Container{container("main")}, []corev1.Container{container("setup")}, "init containers"},
+		// Pods outside Nebula are never Nebula's to judge, whatever the selector says.
+		{"not opted in", nil, []corev1.Container{container("a", 1, 2), container("b")}, []corev1.Container{container("i")}, ""},
+	}
+	v := &PodCustomValidator{}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			pod := podWith(tc.labels, "")
+			pod.Spec.Containers = tc.containers
+			pod.Spec.InitContainers = tc.inits
+			_, err := v.ValidateCreate(context.Background(), pod)
+			switch {
+			case tc.wantErr == "" && err != nil:
+				t.Fatalf("ValidateCreate err = %v, want nil", err)
+			case tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)):
+				t.Fatalf("ValidateCreate err = %v, want it to contain %q", err, tc.wantErr)
+			}
+		})
+	}
+}
+
+func TestValidateCreate_RejectsNonPod(t *testing.T) {
+	if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), &corev1.Service{}); err == nil {
+		t.Fatal("expected an error for a non-Pod object")
+	}
+}
+
+// A pre-bound Pod skips placement (see needsPlacement), so no provider ever runs it and its
+// shape is not Nebula's to judge, even when it carries the opt-in label.
+func TestValidateCreate_SkipsPreBoundPod(t *testing.T) {
+	pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "node-1")
+	pod.Spec.Containers = []corev1.Container{
+		{Name: "main", Ports: []corev1.ContainerPort{{ContainerPort: 8080}, {ContainerPort: 9090}}},
+		{Name: "sidecar"},
+	}
+	pod.Spec.InitContainers = []corev1.Container{{Name: "setup"}}
+	if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), pod); err != nil {
+		t.Fatalf("ValidateCreate err = %v, want nil for a pre-bound Pod", err)
+	}
+}
+
+func TestValidateUpdate_JudgesLabelTransitions(t *testing.T) {
+	optedIn := map[string]string{nebulav1alpha1.EnabledLabel: "true"}
+	sidecarPod := func(labels map[string]string) *corev1.Pod {
+		pod := podWith(labels, "", nebulav1alpha1.ProviderSelectionGate)
+		pod.Spec.Containers = []corev1.Container{{Name: "main"}, {Name: "sidecar"}}
+		return pod
+	}
+	// validPod is valid in shape but was created without the label, so no defaulter ran on it.
+	validPod := func(labels map[string]string, nodeName string, gates ...string) *corev1.Pod {
+		pod := podWith(labels, nodeName, gates...)
+		pod.Spec.Containers = []corev1.Container{{Name: "main"}}
+		return pod
+	}
+	released := func(pod *corev1.Pod) *corev1.Pod {
+		pod.Spec.SchedulingGates = nil
+		return pod
+	}
+	cases := []struct {
+		name     string
+		old, new *corev1.Pod
+		wantErr  string // substring; empty means admitted
+	}{
+		// Created unlabelled, so the defaulter never ran: whatever it hand-copied, the Pod
+		// may lack the gate or the toleration placement needs.
+		{"relabelled with a bad shape", sidecarPod(nil), sidecarPod(optedIn), "opt in at creation"},
+		{"relabelled without the gate", validPod(nil, ""), validPod(optedIn, ""), "opt in at creation"},
+		{"relabelled with the gate but no toleration",
+			validPod(nil, "", nebulav1alpha1.ProviderSelectionGate),
+			validPod(optedIn, "", nebulav1alpha1.ProviderSelectionGate), "opt in at creation"},
+		{"relabelled while bound", validPod(nil, "node-1"), validPod(optedIn, "node-1"), "opt in at creation"},
+		// An opted-in Pod admitted before this check existed: rejecting its later writes
+		// would leave it gated forever.
+		{"already opted in", sidecarPod(optedIn), sidecarPod(optedIn), ""},
+		{"still not opted in", sidecarPod(nil), sidecarPod(map[string]string{"other": "x"}), ""},
+		// Placement ignores an unlabelled Pod, so nothing would ever release the gate it keeps.
+		{"opting out while gated", sidecarPod(optedIn), sidecarPod(nil), "gated forever"},
+		{"opting out and releasing the gate", sidecarPod(optedIn), released(sidecarPod(nil)), ""},
+		{"opting out after placement", released(sidecarPod(optedIn)), released(sidecarPod(nil)), ""},
+	}
+	v := &PodCustomValidator{}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := v.ValidateUpdate(context.Background(), tc.old, tc.new)
+			switch {
+			case tc.wantErr == "" && err != nil:
+				t.Fatalf("ValidateUpdate err = %v, want nil", err)
+			case tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)):
+				t.Fatalf("ValidateUpdate err = %v, want it to contain %q", err, tc.wantErr)
+			}
+		})
+	}
+}

pkg/provider/modal/client.go

@@ -180,7 +180,6 @@ func (c *sdkClient) CreateSandbox(ctx context.Context, spec SandboxSpec) (string
 		MemoryMiB:      spec.MemoryMiB,
 		CPULimit:       spec.CPULimit,
 		MemoryLimitMiB: spec.MemoryLimitMiB,
-		EncryptedPorts: spec.Ports,
 		// Nil leaves Modal's SchedulerPlacement unset entirely (the SDK only builds one
 		// when Regions is non-empty), which is the unconstrained, un-multiplied case.
 		Regions: spec.Regions,
@@ -331,8 +330,6 @@ func (c *sdkClient) registrySecret(ctx context.Context, kv map[string]string) (*
 // provider off rather than be scoped to one request.
 func (c *sdkClient) mintCredential(ctx context.Context, sb *modal.Sandbox, port int) (Credential, error) {
 	creds, err := sb.CreateConnectToken(ctx, &modal.SandboxCreateConnectTokenParams{
-		// Derived from the exposed set rather than carried separately, so the routed
-		// port cannot name one the sandbox was never told to accept traffic on.
 		Port: port,
 	})
 	if err != nil {
@@ -793,9 +790,7 @@ func (c *sdkClient) observe(ctx context.Context, sb *modal.Sandbox) (Sandbox, er
 	// minted at create and already persisted on the Pod's endpoint annotation, so
 	// re-deriving it per tick would be a round trip for a value the API server holds.
 	//
-	// The alternative — a tunnel URL — is worse than nothing: a tunnel is PUBLIC to
-	// whoever learns it, so substituting one for an authenticated URL silently downgrades
-	// access. A sandbox whose mint failed reports no address at all, and that stays a FACT
+	// A sandbox whose mint failed reports no address at all, and that stays a FACT
 	// rather than an error, since observe's errors fail the entire List.
 	return out, nil
 }