diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 80d5057b17..90b9a38409 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -600,6 +600,7 @@ openshell logs | Symptom | Likely cause | Check | |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | +| `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Gateway exits while resolving compute-driver listener requirements | Callback alias topology is unsupported, the Podman network cannot be inspected, or the selected address is not private/authorized | Gateway startup error, `podman info --debug`, Podman network inspection, host IPv4 default route | | Admin, health, reflection, or HTTP request is denied on an additional Docker/Podman callback-only listener | Additional callback listeners intentionally expose only sandbox-callable gRPC methods | Retry through the gateway's primary endpoint; inspect the listener-purpose startup log if the address was unexpected | diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index ebd9a01595..c60733e954 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -26,6 +26,8 @@ mise run helm:k3s:create ``` Creates a k3d cluster and merges its kubeconfig into the worktree-local `kubeconfig` file. +When the named cluster already exists, the task starts any stopped containers and refreshes +same-named kubeconfig entries so a recreated load balancer's current API port takes effect. Also applies the upstream agent-sandbox CRDs/controller (pinned via `AGENT_SANDBOX_VERSION` in `tasks/scripts/helm-k3s-local.sh`, fetched from `github.com/kubernetes-sigs/agent-sandbox` releases), enables its OTLP tracing on v0.5 and later, installs an OTLP trace @@ -118,8 +120,9 @@ Kubernetes compute-driver spans under their distinct service names, along with Agent Sandbox controller reconciliation spans linked through the Sandbox trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on -`http://127.0.0.1:8090`; the local `gateway`, `gateway:docker`, and `gateway:vm` -tasks export to the collector endpoint automatically. +`http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and +`gateway:vm` tasks detect the collector listener at startup and enable trace +export only while it is reachable. **HA test deploy** (two gateway replicas + external PostgreSQL Secret): uncomment `#- ci/values-high-availability.yaml` in `deploy/helm/openshell/skaffold.yaml`, diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 9b9657e94d..bbd7e69b88 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -8,6 +8,10 @@ spans export to the same OTLP/gRPC collector with the service name context and emits the compute-driver RPC boundary that a standalone driver would expose. +`mise run gateway:docker` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + The standalone `openshell-driver-docker` binary accepts `OPENSHELL_OTLP_ENDPOINT`. When set, it exports Docker driver spans to that collector, continues W3C trace context from gateway RPC metadata, and flushes diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index b0436634ee..d55d20ec93 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -13,6 +13,10 @@ spans export to the same OTLP/gRPC collector with the service name uses the same compute-driver RPC span names in its in-process and standalone forms. +`mise run gateway:podman` enables this export only when a local collector is +listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration +so the development gateway does not repeatedly report export failures. + Before creating the container, the driver inspects the final sandbox image and captures its immutable image ID and raw OCI `Config.User`. Container creation uses that image ID with pulling disabled, preventing a mutable tag from changing diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..87d48712bc 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -44,6 +44,7 @@ By default `mise run gateway:vm`: - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. +- Enables OTLP trace export to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 6e7b85634a..7bf7a68892 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -65,6 +65,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -202,9 +217,6 @@ version = 1 compute_drivers = ["docker"] disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -223,6 +235,8 @@ grpc_endpoint = "${GRPC_ENDPOINT}" supervisor_bin = "${SUPERVISOR_BIN}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index 8b6d98ca0c..48ac717595 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -129,6 +129,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + register_gateway_metadata() { local name=$1 local endpoint=$2 @@ -207,9 +222,6 @@ compute_drivers = ["podman"] default_image = "${SANDBOX_IMAGE}" disable_tls = true -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -266,6 +278,8 @@ if [[ -n "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE+x}" ]]; then printf 'proxy_ca_bundle = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_CA_BUNDLE}")" >>"${CONFIG_PATH}" fi +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://${CLI_ENDPOINT_HOST}:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index c4aac270da..288ba289d2 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -83,6 +83,21 @@ port_is_in_use() { (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 } +append_local_otlp_config_if_available() { + local config_path=$1 + if ! port_is_in_use 4317; then + echo "OTLP collector not detected on 127.0.0.1:4317; trace export disabled." + return + fi + + cat >>"${config_path}" <<'EOF' + +[openshell.gateway.otlp] +endpoint = "http://127.0.0.1:4317" +EOF + echo "OTLP trace export enabled for http://127.0.0.1:4317." +} + invoking_user() { if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then printf '%s\n' "${SUDO_USER}" @@ -327,9 +342,6 @@ version = 1 compute_drivers = ["vm"] disable_tls = ${DISABLE_TLS} -[openshell.gateway.otlp] -endpoint = "http://127.0.0.1:4317" - [openshell.gateway.auth] allow_unauthenticated_users = true @@ -348,6 +360,8 @@ driver_dir = "${DRIVER_DIR}" state_dir = "${VM_DRIVER_STATE_DIR}" EOF +append_local_otlp_config_if_available "${CONFIG_PATH}" + GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" save_active_gateway "${GATEWAY_NAME}" diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 39fed2f4b4..dc8adb9bdf 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -142,12 +142,15 @@ k3d_cluster_exists() { merge_kubeconfig() { require_kubectl - local tmp k3d_cfg merged_dir + local tmp merged_dir tmp="$(mktemp)" k3d kubeconfig get "${CLUSTER_NAME}" >"${tmp}" if [[ -s "${KUBECONFIG_TARGET}" ]]; then - KUBECONFIG="${KUBECONFIG_TARGET}:${tmp}" kubectl config view --flatten >"${tmp}.out" + # Put the freshly generated k3d config first so its cluster, context, and + # user entries replace stale entries with the same names. The API server's + # random host port can change when Docker recreates the load balancer. + KUBECONFIG="${tmp}:${KUBECONFIG_TARGET}" kubectl config view --flatten >"${tmp}.out" mv "${tmp}.out" "${KUBECONFIG_TARGET}" else merged_dir="$(dirname "${KUBECONFIG_TARGET}")" @@ -394,7 +397,8 @@ EOF local lb_port_map="${HOST_LB_PORT}:80@loadbalancer" if k3d_cluster_exists; then - echo "k3d cluster '${CLUSTER_NAME}' already exists; merging kubeconfig." + echo "k3d cluster '${CLUSTER_NAME}' already exists; ensuring it is running." + k3d cluster start "${CLUSTER_NAME}" else echo "Creating k3d cluster '${CLUSTER_NAME}'..." k3d cluster create "${CLUSTER_NAME}" \