fix(ci): enable OTLP export only when a collector is requested

.agents/skills/helm-dev-environment/SKILL.md

@@ -266,8 +266,9 @@ task running while using those endpoints.
 ### Viewing local traces
 
 The gateway exports OTLP/gRPC to
-`http://openshell-collector.observability.svc.cluster.local:4317` through the
-default Skaffold values. Forward OTLP/gRPC and the trace UI to the host:
+`http://openshell-collector.observability.svc.cluster.local:4317` through
+`ci/values-local-tracing.yaml`, which only `skaffold.yaml` layers. The Kubernetes
+E2E wrapper omits it, so E2E clusters export no traces. Forward OTLP/gRPC and the trace UI to the host:
 
 ```bash
 mise run helm:k3s:forward
@@ -278,9 +279,10 @@ Kubernetes compute-driver spans under their distinct service names, along with
 Agent Sandbox controller reconciliation spans linked through the Sandbox
 trace-context annotation. The same command exposes OTLP/gRPC on
 `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on
-`http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and
-`gateway:vm` tasks detect the collector listener at startup and enable trace
-export, OCSF log export, and the agent trace relay only while it is reachable.
+`http://127.0.0.1:8090`. The local `gateway`, `gateway:docker`,
+`gateway:podman`, and `gateway:vm` tasks enable trace export, OCSF log export,
+and the agent trace relay only when `OPENSHELL_DEV_OTLP_ENDPOINT` is set, for
+example `OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317`.
 Spans an agent exports from inside a sandbox arrive through the supervisor's
 relay with `openshell.telemetry.source=agent` and share `openshell.sandbox.id`
 with that sandbox's supervisor spans.
@@ -642,6 +644,7 @@ for dependencies still declared in `Chart.yaml`.
 | `deploy/helm/openshell/skaffold.yaml` | Skaffold config — images, Helm releases, values overlays |
 | `deploy/helm/openshell/values.yaml` | Default Helm values |
 | `deploy/helm/openshell/ci/values-skaffold.yaml` | Dev overrides (image pull policy, TLS disabled for local Skaffold) |
+| `deploy/helm/openshell/ci/values-local-tracing.yaml` | OTLP export to the `helm:k3s:create` collector (Skaffold only) |
 | `deploy/helm/openshell/ci/values-cert-manager.yaml` | cert-manager PKI overlay (opt-in; disables pkiInitJob) |
 | `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay |
 | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) |

crates/openshell-driver-vm/README.md

@@ -70,7 +70,7 @@ By default `mise run gateway:vm`:
 - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`.
 - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`.
 - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`.
-- Enables OTLP trace export, OCSF log export, and the agent trace relay to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures.
+- Enables OTLP trace export, OCSF log export, and the agent trace relay only when `OPENSHELL_DEV_OTLP_ENDPOINT` names a collector, for example `http://127.0.0.1:4317`. Otherwise, it omits the OTLP configuration.
 
 For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges:
 

deploy/helm/openshell/ci/values-local-tracing.yaml

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Export gateway traces to the collector that `mise run helm:k3s:create`
+# installs. Layer only onto clusters that run that collector.
+gatewayConfig:
+  openshell.gateway.otlp:
+    endpoint: http://openshell-collector.observability.svc.cluster.local:4317
+  openshell.gateway.otlp.agent:
+    enabled: true

deploy/helm/openshell/ci/values-skaffold.yaml

@@ -5,16 +5,10 @@
 gatewayConfig:
   openshell.drivers.kubernetes:
     image_pull_policy: if_not_present
-  openshell.gateway.otlp:
-    endpoint: http://openshell-collector.observability.svc.cluster.local:4317
-  openshell.gateway.otlp.agent:
-    enabled: true
   openshell.gateway.auth:
     allow_unauthenticated_users: true
 
 server:
-  otlp:
-    endpoint: http://openshell-collector.observability.svc.cluster.local:4317
   # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob).
   disableTls: true
   auth:

deploy/helm/openshell/skaffold.yaml

@@ -124,6 +124,7 @@ deploy:
         valuesFiles:
           - values.yaml
           - ci/values-skaffold.yaml
+          - ci/values-local-tracing.yaml
           # Add ci/values-cert-manager.yaml here (and uncomment the cert-manager
           # release above) to switch TLS generation to cert-manager.
           #- ci/values-cert-manager.yaml

skills/debug-openshell-cluster/SKILL.md

@@ -1582,7 +1582,7 @@ credential failures.
 | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs |
 | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable endpoint or one that does not speak the table's `protocol`, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint` and `ocsf_protocol`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). |
 | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, or shipper falling behind retention | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). |
-| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` |
+| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with `OPENSHELL_DEV_OTLP_ENDPOINT` set but no collector is listening, for example because `helm:k3s:forward` stopped | Start the collector forwarding again, or restart the gateway task without `OPENSHELL_DEV_OTLP_ENDPOINT`; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` |
 | Sandbox log shows `OTLP agent trace relay inactive: not enabled by the gateway` at startup, then `agent OTLP export attempted but the agent trace relay is not enabled` when the agent exports | The gateway does not enable `[openshell.gateway.otlp.agent]`, so the supervisor refuses the workload's OTLP relay address | Set `enabled = true` in `[openshell.gateway.otlp.agent]` (Helm: `gatewayConfig."openshell.gateway.otlp.agent".enabled`) with an endpoint from that table or `[openshell.gateway.otlp]`, and recreate the sandbox; `openshell logs <sandbox> --source sandbox` then shows `OTLP agent trace relay enabled` |
 | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs |
 | Docker or Podman sandbox never registers | Wrong gateway endpoint, unavailable host networking, or supervisor startup failure | Gateway logs and supervisor container logs |

tasks/scripts/gateway-docker.sh

@@ -18,6 +18,7 @@
 #   OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:docker
 #   OPENSHELL_SUPERVISOR_IMAGE=ghcr.io/... mise run gateway:docker
 #   OPENSHELL_SANDBOX_RUNTIME_IMAGE=ghcr.io/... mise run gateway:docker
+#   OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317 mise run gateway:docker
 #
 # After the gateway is running, point the CLI at it with either:
 #   openshell --gateway docker-dev <command>
@@ -83,27 +84,6 @@ ensure_docker_runtime_image() {
   fi
 }
 
-append_local_otlp_config_if_available() {
-  local config_path=$1
-  if ! port_is_in_use 4317; then
-    echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled."
-    return
-  fi
-
-  cat >>"${config_path}" <<'EOF'
-
-[openshell.gateway.otlp]
-endpoint = "http://127.0.0.1:4317"
-
-[openshell.gateway.otlp.ocsf]
-enabled = true
-
-[openshell.gateway.otlp.agent]
-enabled = true
-EOF
-  echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317."
-}
-
 register_gateway_metadata() {
   local name=$1
   local endpoint=$2
@@ -231,7 +211,7 @@ if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then
   printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}"
 fi
 
-append_local_otlp_config_if_available "${CONFIG_PATH}"
+append_dev_otlp_config "${CONFIG_PATH}"
 
 GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
 register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"

tasks/scripts/gateway-podman.sh

@@ -19,6 +19,7 @@
 #   OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:podman
 #   OPENSHELL_SUPERVISOR_IMAGE=ghcr.io/... mise run gateway:podman
 #   OPENSHELL_SANDBOX_RUNTIME_IMAGE=ghcr.io/... mise run gateway:podman
+#   OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317 mise run gateway:podman
 
 set -euo pipefail
 
@@ -112,27 +113,6 @@ port_is_in_use() {
   (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1
 }
 
-append_local_otlp_config_if_available() {
-  local config_path=$1
-  if ! port_is_in_use 4317; then
-    echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled."
-    return
-  fi
-
-  cat >>"${config_path}" <<'EOF'
-
-[openshell.gateway.otlp]
-endpoint = "http://127.0.0.1:4317"
-
-[openshell.gateway.otlp.ocsf]
-enabled = true
-
-[openshell.gateway.otlp.agent]
-enabled = true
-EOF
-  echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317."
-}
-
 register_gateway_metadata() {
   local name=$1
   local endpoint=$2
@@ -273,7 +253,7 @@ if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then
   printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}"
 fi
 
-append_local_otlp_config_if_available "${CONFIG_PATH}"
+append_dev_otlp_config "${CONFIG_PATH}"
 
 GATEWAY_ENDPOINT="http://${CLI_ENDPOINT_HOST}:${PORT}"
 register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"

tasks/scripts/gateway-toml.sh

@@ -17,3 +17,27 @@ toml_escape() {
   value=${value//$'\t'/\\t}
   printf '%s' "${value}"
 }
+
+# Append OTLP export to a gateway config when OPENSHELL_DEV_OTLP_ENDPOINT is
+# set. Export stays off otherwise so no collector is assumed.
+append_dev_otlp_config() {
+  local config_path=$1
+  local endpoint=${OPENSHELL_DEV_OTLP_ENDPOINT:-}
+  if [[ -z "${endpoint}" ]]; then
+    echo "OTLP export disabled; set OPENSHELL_DEV_OTLP_ENDPOINT to enable it."
+    return
+  fi
+
+  cat >>"${config_path}" <<TOML
+
+[openshell.gateway.otlp]
+endpoint = "$(toml_escape "${endpoint}")"
+
+[openshell.gateway.otlp.ocsf]
+enabled = true
+
+[openshell.gateway.otlp.agent]
+enabled = true
+TOML
+  echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for ${endpoint}."
+}

tasks/scripts/gateway-vm.sh

@@ -22,6 +22,7 @@
 #   OPENSHELL_VM_GATEWAY_NAME=my-vm-gateway mise run gateway:vm
 #   OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:vm
 #   OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:vm
+#   OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317 mise run gateway:vm
 #   mise run gateway:vm -- --gpu
 #
 # This script also writes ~/.config/openshell/active_gateway so the
@@ -90,27 +91,6 @@ port_is_in_use() {
   (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1
 }
 
-append_local_otlp_config_if_available() {
-  local config_path=$1
-  if ! port_is_in_use 4317; then
-    echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled."
-    return
-  fi
-
-  cat >>"${config_path}" <<'EOF'
-
-[openshell.gateway.otlp]
-endpoint = "http://127.0.0.1:4317"
-
-[openshell.gateway.otlp.ocsf]
-enabled = true
-
-[openshell.gateway.otlp.agent]
-enabled = true
-EOF
-  echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317."
-}
-
 invoking_user() {
   if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then
     printf '%s\n' "${SUDO_USER}"
@@ -397,7 +377,7 @@ if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then
   printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}"
 fi
 
-append_local_otlp_config_if_available "${CONFIG_PATH}"
+append_dev_otlp_config "${CONFIG_PATH}"
 
 GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
 register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}"

tasks/scripts/gateway.sh

@@ -18,6 +18,8 @@ set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 # shellcheck source=tasks/scripts/gateway-pull-policy.sh
 source "${ROOT}/tasks/scripts/gateway-pull-policy.sh"
+# shellcheck source=tasks/scripts/gateway-toml.sh
+source "${ROOT}/tasks/scripts/gateway-toml.sh"
 GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}"
 
 usage() {
@@ -41,6 +43,9 @@ Environment:
                           or ::1 for Podman Machine on macOS.
   OPENSHELL_SERVER_PORT   Gateway port. Defaults to 8080 for Kubernetes,
                           18080 for Podman/Docker, and 18081 for VM.
+  OPENSHELL_DEV_OTLP_ENDPOINT
+                          OTLP collector for trace export, OCSF log export,
+                          and the agent trace relay. Unset disables export.
 Docker, Podman, and VM runs delegate to their gateway:<driver> setup scripts.
 EOF
 }
@@ -262,15 +267,6 @@ name = "${GATEWAY_NAME}"
 compute_driver = "${DRIVER}"
 disable_tls = true
 
-[openshell.gateway.otlp]
-endpoint = "http://127.0.0.1:4317"
-
-[openshell.gateway.otlp.ocsf]
-enabled = true
-
-[openshell.gateway.otlp.agent]
-enabled = true
-
 [openshell.gateway.auth]
 allow_unauthenticated_users = true
 
@@ -293,6 +289,8 @@ if [[ -n "${GRPC_ENDPOINT}" ]]; then
   printf 'grpc_endpoint = "%s"\n' "${GRPC_ENDPOINT}" >>"${CONFIG_PATH}"
 fi
 
+append_dev_otlp_config "${CONFIG_PATH}"
+
 GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
 register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"