diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index e866ec6927..56061385c3 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -266,8 +266,9 @@ task running while using those endpoints. ### Viewing local traces The gateway exports OTLP/gRPC to -`http://openshell-collector.observability.svc.cluster.local:4317` through the -default Skaffold values. Forward OTLP/gRPC and the trace UI to the host: +`http://openshell-collector.observability.svc.cluster.local:4317` through +`ci/values-local-tracing.yaml`, which only `skaffold.yaml` layers. The Kubernetes +E2E wrapper omits it, so E2E clusters export no traces. Forward OTLP/gRPC and the trace UI to the host: ```bash mise run helm:k3s:forward @@ -278,9 +279,10 @@ Kubernetes compute-driver spans under their distinct service names, along with Agent Sandbox controller reconciliation spans linked through the Sandbox trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on -`http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and -`gateway:vm` tasks detect the collector listener at startup and enable trace -export, OCSF log export, and the agent trace relay only while it is reachable. +`http://127.0.0.1:8090`. The local `gateway`, `gateway:docker`, +`gateway:podman`, and `gateway:vm` tasks enable trace export, OCSF log export, +and the agent trace relay only when `OPENSHELL_DEV_OTLP_ENDPOINT` is set, for +example `OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317`. Spans an agent exports from inside a sandbox arrive through the supervisor's relay with `openshell.telemetry.source=agent` and share `openshell.sandbox.id` with that sandbox's supervisor spans. @@ -642,6 +644,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/skaffold.yaml` | Skaffold config — images, Helm releases, values overlays | | `deploy/helm/openshell/values.yaml` | Default Helm values | | `deploy/helm/openshell/ci/values-skaffold.yaml` | Dev overrides (image pull policy, TLS disabled for local Skaffold) | +| `deploy/helm/openshell/ci/values-local-tracing.yaml` | OTLP export to the `helm:k3s:create` collector (Skaffold only) | | `deploy/helm/openshell/ci/values-cert-manager.yaml` | cert-manager PKI overlay (opt-in; disables pkiInitJob) | | `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay | | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a0371bacfb..cb9433d08e 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -70,7 +70,7 @@ By default `mise run gateway:vm`: - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. -- Enables OTLP trace export, OCSF log export, and the agent trace relay to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. +- Enables OTLP trace export, OCSF log export, and the agent trace relay only when `OPENSHELL_DEV_OTLP_ENDPOINT` names a collector, for example `http://127.0.0.1:4317`. Otherwise, it omits the OTLP configuration. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: diff --git a/deploy/helm/openshell/ci/values-local-tracing.yaml b/deploy/helm/openshell/ci/values-local-tracing.yaml new file mode 100644 index 0000000000..68d1f23500 --- /dev/null +++ b/deploy/helm/openshell/ci/values-local-tracing.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Export gateway traces to the collector that `mise run helm:k3s:create` +# installs. Layer only onto clusters that run that collector. +gatewayConfig: + openshell.gateway.otlp: + endpoint: http://openshell-collector.observability.svc.cluster.local:4317 + openshell.gateway.otlp.agent: + enabled: true diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 1ff2bb2fc4..76e0167e79 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -5,16 +5,10 @@ gatewayConfig: openshell.drivers.kubernetes: image_pull_policy: if_not_present - openshell.gateway.otlp: - endpoint: http://openshell-collector.observability.svc.cluster.local:4317 - openshell.gateway.otlp.agent: - enabled: true openshell.gateway.auth: allow_unauthenticated_users: true server: - otlp: - endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). disableTls: true auth: diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index 3ad4d3b8bf..f505c479fa 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -124,6 +124,7 @@ deploy: valuesFiles: - values.yaml - ci/values-skaffold.yaml + - ci/values-local-tracing.yaml # Add ci/values-cert-manager.yaml here (and uncomment the cert-manager # release above) to switch TLS generation to cert-manager. #- ci/values-cert-manager.yaml diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ec8c9dee6f..26d77f8ffd 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1582,7 +1582,7 @@ credential failures. | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable endpoint or one that does not speak the table's `protocol`, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint` and `ocsf_protocol`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, or shipper falling behind retention | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | -| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | +| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with `OPENSHELL_DEV_OTLP_ENDPOINT` set but no collector is listening, for example because `helm:k3s:forward` stopped | Start the collector forwarding again, or restart the gateway task without `OPENSHELL_DEV_OTLP_ENDPOINT`; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Sandbox log shows `OTLP agent trace relay inactive: not enabled by the gateway` at startup, then `agent OTLP export attempted but the agent trace relay is not enabled` when the agent exports | The gateway does not enable `[openshell.gateway.otlp.agent]`, so the supervisor refuses the workload's OTLP relay address | Set `enabled = true` in `[openshell.gateway.otlp.agent]` (Helm: `gatewayConfig."openshell.gateway.otlp.agent".enabled`) with an endpoint from that table or `[openshell.gateway.otlp]`, and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Docker or Podman sandbox never registers | Wrong gateway endpoint, unavailable host networking, or supervisor startup failure | Gateway logs and supervisor container logs | diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index c7e71abfed..5a951eb28d 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -18,6 +18,7 @@ # OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:docker # OPENSHELL_SUPERVISOR_IMAGE=ghcr.io/... mise run gateway:docker # OPENSHELL_SANDBOX_RUNTIME_IMAGE=ghcr.io/... mise run gateway:docker +# OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317 mise run gateway:docker # # After the gateway is running, point the CLI at it with either: # openshell --gateway docker-dev @@ -83,27 +84,6 @@ ensure_docker_runtime_image() { fi } -append_local_otlp_config_if_available() { - local config_path=$1 - if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." - return - fi - - cat >>"${config_path}" <<'EOF' - -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - -[openshell.gateway.otlp.ocsf] -enabled = true - -[openshell.gateway.otlp.agent] -enabled = true -EOF - echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." -} - register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -231,7 +211,7 @@ if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" fi -append_local_otlp_config_if_available "${CONFIG_PATH}" +append_dev_otlp_config "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index c032cdad80..bb837e464a 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -19,6 +19,7 @@ # OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:podman # OPENSHELL_SUPERVISOR_IMAGE=ghcr.io/... mise run gateway:podman # OPENSHELL_SANDBOX_RUNTIME_IMAGE=ghcr.io/... mise run gateway:podman +# OPENSHELL_DEV_OTLP_ENDPOINT=http://127.0.0.1:4317 mise run gateway:podman set -euo pipefail @@ -112,27 +113,6 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } -append_local_otlp_config_if_available() { - local config_path=$1 - if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." - return - fi - - cat >>"${config_path}" <<'EOF' - -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - -[openshell.gateway.otlp.ocsf] -enabled = true - -[openshell.gateway.otlp.agent] -enabled = true -EOF - echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." -} - register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -273,7 +253,7 @@ if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}" fi -append_local_otlp_config_if_available "${CONFIG_PATH}" +append_dev_otlp_config "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://${CLI_ENDPOINT_HOST}:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-toml.sh b/tasks/scripts/gateway-toml.sh index f9b1e1916b..ce53b639a9 100644 --- a/tasks/scripts/gateway-toml.sh +++ b/tasks/scripts/gateway-toml.sh @@ -17,3 +17,27 @@ toml_escape() { value=${value//$'\t'/\\t} printf '%s' "${value}" } + +# Append OTLP export to a gateway config when OPENSHELL_DEV_OTLP_ENDPOINT is +# set. Export stays off otherwise so no collector is assumed. +append_dev_otlp_config() { + local config_path=$1 + local endpoint=${OPENSHELL_DEV_OTLP_ENDPOINT:-} + if [[ -z "${endpoint}" ]]; then + echo "OTLP export disabled; set OPENSHELL_DEV_OTLP_ENDPOINT to enable it." + return + fi + + cat >>"${config_path}" </dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } -append_local_otlp_config_if_available() { - local config_path=$1 - if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." - return - fi - - cat >>"${config_path}" <<'EOF' - -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - -[openshell.gateway.otlp.ocsf] -enabled = true - -[openshell.gateway.otlp.agent] -enabled = true -EOF - echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." -} - invoking_user() { if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then printf '%s\n' "${SUDO_USER}" @@ -397,7 +377,7 @@ if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}" fi -append_local_otlp_config_if_available "${CONFIG_PATH}" +append_dev_otlp_config "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 3b04ee2684..1fc9d9e9f8 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -18,6 +18,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # shellcheck source=tasks/scripts/gateway-pull-policy.sh source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" usage() { @@ -41,6 +43,9 @@ Environment: or ::1 for Podman Machine on macOS. OPENSHELL_SERVER_PORT Gateway port. Defaults to 8080 for Kubernetes, 18080 for Podman/Docker, and 18081 for VM. + OPENSHELL_DEV_OTLP_ENDPOINT + OTLP collector for trace export, OCSF log export, + and the agent trace relay. Unset disables export. Docker, Podman, and VM runs delegate to their gateway: setup scripts. EOF } @@ -262,15 +267,6 @@ name = "${GATEWAY_NAME}" compute_driver = "${DRIVER}" disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - -[openshell.gateway.otlp.ocsf] -enabled = true - -[openshell.gateway.otlp.agent] -enabled = true - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -293,6 +289,8 @@ if [[ -n "${GRPC_ENDPOINT}" ]]; then printf 'grpc_endpoint = "%s"\n' "${GRPC_ENDPOINT}" >>"${CONFIG_PATH}" fi +append_dev_otlp_config "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"