From ad1893048b1853a539e0c53deea5aac6f64bdc38 Mon Sep 17 00:00:00 2001 From: Kris Hicks Date: Tue, 25 Aug 2026 10:59:46 -0700 Subject: [PATCH 1/2] fix(helm): refresh kubeconfig for existing k3d clusters Docker can recreate the k3d load balancer on a new API port. Start existing clusters and prefer fresh k3d entries so create does not retain a stale endpoint. Signed-off-by: Kris Hicks --- .agents/skills/helm-dev-environment/SKILL.md | 2 ++ tasks/scripts/helm-k3s-local.sh | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index ebd9a01595..6c0777217f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -26,6 +26,8 @@ mise run helm:k3s:create ``` Creates a k3d cluster and merges its kubeconfig into the worktree-local `kubeconfig` file. +When the named cluster already exists, the task starts any stopped containers and refreshes +same-named kubeconfig entries so a recreated load balancer's current API port takes effect. Also applies the upstream agent-sandbox CRDs/controller (pinned via `AGENT_SANDBOX_VERSION` in `tasks/scripts/helm-k3s-local.sh`, fetched from `github.com/kubernetes-sigs/agent-sandbox` releases), enables its OTLP tracing on v0.5 and later, installs an OTLP trace diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 39fed2f4b4..dc8adb9bdf 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -142,12 +142,15 @@ k3d_cluster_exists() { merge_kubeconfig() { require_kubectl - local tmp k3d_cfg merged_dir + local tmp merged_dir tmp="$(mktemp)" k3d kubeconfig get "${CLUSTER_NAME}" >"${tmp}" if [[ -s "${KUBECONFIG_TARGET}" ]]; then - KUBECONFIG="${KUBECONFIG_TARGET}:${tmp}" kubectl config view --flatten >"${tmp}.out" + # Put the freshly generated k3d config first so its cluster, context, and + # user entries replace stale entries with the same names. The API server's + # random host port can change when Docker recreates the load balancer. + KUBECONFIG="${tmp}:${KUBECONFIG_TARGET}" kubectl config view --flatten >"${tmp}.out" mv "${tmp}.out" "${KUBECONFIG_TARGET}" else merged_dir="$(dirname "${KUBECONFIG_TARGET}")" @@ -394,7 +397,8 @@ EOF local lb_port_map="${HOST_LB_PORT}:80@loadbalancer" if k3d_cluster_exists; then - echo "k3d cluster '${CLUSTER_NAME}' already exists; merging kubeconfig." + echo "k3d cluster '${CLUSTER_NAME}' already exists; ensuring it is running." + k3d cluster start "${CLUSTER_NAME}" else echo "Creating k3d cluster '${CLUSTER_NAME}'..." k3d cluster create "${CLUSTER_NAME}" \ From 193cdd6f894cf69d33da2450801f30d2eefb8918 Mon Sep 17 00:00:00 2001 From: Kris Hicks Date: Wed, 26 Aug 2026 10:53:28 -0700 Subject: [PATCH 2/2] fix(dev): conditionally enable local OTLP export Probe port 4317 before adding OTLP configuration for the VM, Docker, and Podman gateway tasks. Document the startup behavior and troubleshooting for local collector availability. Signed-off-by: Kris Hicks --- .../skills/debug-openshell-cluster/SKILL.md | 1 + .agents/skills/helm-dev-environment/SKILL.md | 5 +++-- crates/openshell-driver-docker/README.md | 4 ++++ crates/openshell-driver-podman/README.md | 4 ++++ crates/openshell-driver-vm/README.md | 1 + tasks/scripts/gateway-docker.sh | 20 ++++++++++++++++--- tasks/scripts/gateway-podman.sh | 20 ++++++++++++++++--- tasks/scripts/gateway-vm.sh | 20 ++++++++++++++++--- 8 files changed, 64 insertions(+), 11 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 80d5057b17..90b9a38409 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -600,6 +600,7 @@ openshell logs | Symptom | Likely cause | Check | |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | +| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 6c0777217f..c60733e954 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -120,8 +120,9 @@ Kubernetes compute-driver spans under their distinct service names, along with Agent Sandbox controller reconciliation spans linked through the Sandbox trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on -`http://127.0.0.1:8090`; the local `gateway`, `gateway:docker`, and `gateway:vm` -tasks export to the collector endpoint automatically. +`http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and +`gateway:vm` tasks detect the collector listener at startup and enable trace +export only while it is reachable. **HA test deploy** (two gateway replicas + external PostgreSQL Secret): uncomment `#- ci/values-high-availability.yaml` in `deploy/helm/openshell/skaffold.yaml`, diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 9b9657e94d..bbd7e69b88 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -8,6 +8,10 @@ spans export to the same OTLP/gRPC collector with the service name context and emits the compute-driver RPC boundary that a standalone driver would expose. +`mise run gateway:docker` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + The standalone `openshell-driver-docker` binary accepts `OPENSHELL_OTLP_ENDPOINT`. When set, it exports Docker driver spans to that collector, continues W3C trace context from gateway RPC metadata, and flushes diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index b0436634ee..d55d20ec93 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -13,6 +13,10 @@ spans export to the same OTLP/gRPC collector with the service name uses the same compute-driver RPC span names in its in-process and standalone forms. +`mise run gateway:podman` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + Before creating the container, the driver inspects the final sandbox image and captures its immutable image ID and raw OCI `Config.User`. Container creation uses that image ID with pulling disabled, preventing a mutable tag from changing diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..87d48712bc 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -44,6 +44,7 @@ By default `mise run gateway:vm`: - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. +- Enables OTLP trace export to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 6e7b85634a..7bf7a68892 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -65,6 +65,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -202,9 +217,6 @@ version = 1 compute_drivers = ["docker"] disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -223,6 +235,8 @@ grpc_endpoint = "${GRPC_ENDPOINT}" supervisor_bin = "${SUPERVISOR_BIN}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index 8b6d98ca0c..48ac717595 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -129,6 +129,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -207,9 +222,6 @@ compute_drivers = ["podman"] default_image = "${SANDBOX_IMAGE}" disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -266,6 +278,8 @@ if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}" fi +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://${CLI_ENDPOINT_HOST}:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index c4aac270da..288ba289d2 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -83,6 +83,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + invoking_user() { if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then printf '%s\n' "${SUDO_USER}" @@ -327,9 +342,6 @@ version = 1 compute_drivers = ["vm"] disable_tls = ${DISABLE_TLS} -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -348,6 +360,8 @@ driver_dir = "${DRIVER_DIR}" state_dir = "${VM_DRIVER_STATE_DIR}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" save_active_gateway "${GATEWAY_NAME}"