ci(e2e): add fedora rootful podman lane

.agents/skills/test-release-canary/SKILL.md

@@ -13,7 +13,7 @@ The Release Canary (`.github/workflows/release-canary.yml`) smoke-tests the arti
 |---|---|---|
 | `macos` | `macos-latest-xlarge` | `install.sh` resolves the Homebrew formula, brew installs the cask, and `openshell status` reaches the brew-services–backed local gateway with the VM driver. |
 | `ubuntu` | `ubuntu-latest` | `install.sh` installs the Debian package, the post-install systemd user service starts, and `openshell status` reaches the local gateway with the Docker driver. |
-| `fedora` | `fedora:latest` container | `install.sh` installs the RPM packages, the local gateway starts under Podman, and `openshell status` succeeds. |
+| `fedora` | `linux-amd64-cpu8` + Fedora Nix VM | `install.sh` installs the RPM packages, the root-owned local gateway starts with rootful Podman, and `openshell status` succeeds. |
 | `kubernetes` | `ubuntu-latest` + kind | `helm install oci://ghcr.io/nvidia/openshell/helm-chart --version 0.0.0-dev` succeeds in a kind cluster, the gateway pod becomes Ready, port-forward exposes 8080, and the released CLI registers the in-cluster gateway and runs `openshell status` against it. |
 
 All canary jobs disable anonymous OpenShell telemetry. Host package jobs inject
@@ -116,6 +116,41 @@ Swap `0.0.0-dev` for `0.0.0-dev.<sha>` to pin to a specific dev build. Tear down
 
 Loopback registration auto-derives the gateway name to `openshell` if `--name` is omitted, which collides with the `install.sh`-installed local gateway — always pass `--name kind` (or another distinct name) when registering in addition to a local install.
 
+## Local Fedora reproduction
+
+The `fedora` job uses the repository's Nix test-guest harness instead of
+running Fedora inside Docker. It can be reproduced on a Linux host with Nix,
+KVM, and the repository checkout:
+
+```shell
+export INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/$(git rev-parse HEAD)/install.sh"
+nix run .#test-guest -- \
+  --distro fedora \
+  --with podman-rootful \
+  -- \
+  sudo env \
+    SUDO_USER=root \
+    HOME=/root \
+    XDG_RUNTIME_DIR=/run/user/0 \
+    DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \
+    OPENSHELL_TELEMETRY_ENABLED=false \
+    INSTALL_SH_URL="$INSTALL_SH_URL" \
+    bash -s <<'EOF'
+set -euo pipefail
+mkdir -p "${XDG_RUNTIME_DIR}"
+chmod 700 "${XDG_RUNTIME_DIR}"
+systemctl start user-runtime-dir@0.service || true
+systemctl start user@0.service
+systemctl --user daemon-reload
+mkdir -p "${HOME}/.config/openshell"
+printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_PODMAN_SOCKET=/run/podman/podman.sock\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \
+  "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env"
+podman --url unix:///run/podman/podman.sock info
+curl -LsSf "${INSTALL_SH_URL}" | sh
+openshell status
+EOF
+```
+
 ## Diagnosing failures
 
 | Symptom | Likely cause | Where to look |

.github/actions/setup-e2e-sandbox/action.yml

@@ -0,0 +1,29 @@
+name: Setup E2E Sandbox
+description: Download an architecture-matched prebuilt OpenShell sandbox binary for E2E tests
+
+inputs:
+  artifact-prefix:
+    description: Artifact name prefix; linux-<arch> is appended automatically
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Download prebuilt sandbox
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ${{ format('{0}-linux-{1}', inputs.artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }}
+        path: .e2e/prebuilt-sandbox
+
+    - name: Configure prebuilt sandbox
+      shell: bash
+      run: |
+        set -euo pipefail
+        sandbox="$GITHUB_WORKSPACE/.e2e/prebuilt-sandbox/openshell-sandbox"
+        if [[ ! -f "$sandbox" ]]; then
+          echo "downloaded artifact is missing $sandbox" >&2
+          exit 1
+        fi
+        chmod +x "$sandbox"
+        "$sandbox" --version
+        echo "OPENSHELL_SANDBOX_BIN=$sandbox" >> "$GITHUB_ENV"

.github/workflows/branch-e2e.yml

@@ -136,6 +136,8 @@ jobs:
       runner: linux-arm64-cpu8
       cli-artifact-prefix: rust-binary-cli
       gateway-artifact-prefix: rust-binary-gateway
+      # The supervisor build uploads the openshell-sandbox binary artifact.
+      sandbox-artifact-prefix: rust-binary-supervisor
       vm-driver-artifact-name: driver-vm-linux-amd64
 
   gpu-e2e:

.github/workflows/e2e-test.yml

@@ -27,6 +27,11 @@ on:
         required: false
         type: string
         default: ""
+      sandbox-artifact-prefix:
+        description: "Optional prebuilt sandbox artifact prefix (artifact suffix is linux-<arch>)"
+        required: false
+        type: string
+        default: ""
       vm-driver-artifact-name:
         description: "Optional prebuilt VM driver artifact name"
         required: false
@@ -129,6 +134,154 @@ jobs:
           OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: ${{ format('openshell-mcp-conformance-client:{0}', inputs.image-tag) }}
         run: ${{ matrix.cmd }}
 
+  e2e-podman-rootless:
+    name: E2E (rust-podman-rootless, Ubuntu 26.04 Nix VM)
+    # Run rootless Podman inside a Nix-managed Ubuntu guest so Podman, pasta,
+    # and user-namespace setup are provisioned by versioned repository tooling
+    # rather than mutable hosted-runner packages.
+    runs-on: ubuntu-26.04
+    timeout-minutes: 60
+    env:
+      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['checkout-ref'] || github.sha }}
+          persist-credentials: false
+
+      - name: Use prebuilt OpenShell CLI
+        uses: ./.github/actions/setup-e2e-cli
+        with:
+          artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell gateway
+        uses: ./.github/actions/setup-e2e-gateway
+        with:
+          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell sandbox
+        uses: ./.github/actions/setup-e2e-sandbox
+        with:
+          artifact-prefix: ${{ inputs.sandbox-artifact-prefix }}
+
+      - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
+        with:
+          name: openshell
+
+      - name: Install mise
+        run: |
+          curl https://mise.run | MISE_VERSION=v2026.4.25 sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
+
+      - name: Install tools
+        run: mise install --locked
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            clang \
+            cmake \
+            libclang-dev \
+            libssl-dev \
+            libz3-dev \
+            openssh-client \
+            pkg-config
+
+      - name: Run rootless Podman E2E
+        run: |
+          set -euo pipefail
+
+          mise x -- e2e/run.sh \
+            --vm ubuntu-26-04 \
+            --with podman-rootless \
+            --host-cli-bin "$OPENSHELL_BIN" \
+            --gateway-bin "$OPENSHELL_GATEWAY_BIN" \
+            --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \
+            --gateway-config e2e/configs/gateway/podman.toml \
+            --features e2e-podman
+
+  e2e-podman-fedora-rootful:
+    name: E2E (rust-podman-rootful, Fedora Nix VM)
+    # Run rootful Podman inside a Nix-managed Fedora guest. This exercises the
+    # RPM-family rootful bridge/listener topology without nesting Podman inside
+    # a Docker job container.
+    runs-on: ubuntu-26.04
+    timeout-minutes: 60
+    env:
+      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs['checkout-ref'] || github.sha }}
+          persist-credentials: false
+
+      - name: Use prebuilt OpenShell CLI
+        uses: ./.github/actions/setup-e2e-cli
+        with:
+          artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell gateway
+        uses: ./.github/actions/setup-e2e-gateway
+        with:
+          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}
+
+      - name: Use prebuilt OpenShell sandbox
+        uses: ./.github/actions/setup-e2e-sandbox
+        with:
+          artifact-prefix: ${{ inputs.sandbox-artifact-prefix }}
+
+      - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
+        with:
+          name: openshell
+
+      - name: Install mise
+        run: |
+          curl https://mise.run | MISE_VERSION=v2026.4.25 sh
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
+
+      - name: Install tools
+        run: mise install --locked
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential \
+            clang \
+            cmake \
+            libclang-dev \
+            libssl-dev \
+            libz3-dev \
+            openssh-client \
+            pkg-config
+
+      - name: Run Fedora rootful Podman E2E
+        run: |
+          set -euo pipefail
+
+          mise x -- e2e/run.sh \
+            --vm fedora \
+            --with podman-rootful \
+            --guest-gateway-user root \
+            --host-cli-bin "$OPENSHELL_BIN" \
+            --gateway-bin "$OPENSHELL_GATEWAY_BIN" \
+            --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \
+            --gateway-config e2e/configs/gateway/podman-rootful.toml \
+            --features e2e-podman \
+            --suite sandbox_lifecycle
+
   e2e-vm:
     name: E2E (rust-vm-${{ matrix.suite }})
     # libkrun needs KVM, so this job must run directly on a GitHub-hosted

.github/workflows/release-canary.yml

@@ -61,53 +61,43 @@ jobs:
     name: Fedora RPM
     if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     runs-on: linux-amd64-cpu8
-    timeout-minutes: 20
+    timeout-minutes: 30
     env:
-      FEDORA_CANARY_CONTAINER: openshell-fedora-canary-${{ github.run_id }}-${{ github.run_attempt }}
+      INSTALL_SH_URL: https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh
     steps:
-      - name: Start Fedora systemd container and root user manager
-        run: |
-          set -euo pipefail
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
 
-          docker run --detach \
-            --name "${FEDORA_CANARY_CONTAINER}" \
-            --privileged \
-            --cgroupns=host \
-            --tmpfs /run \
-            --tmpfs /tmp \
-            --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \
-            fedora:latest \
-            bash -lc 'dnf install -y curl dbus-daemon podman systemd && exec /usr/sbin/init'
-
-          for _ in $(seq 1 120); do
-            if docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then
-              break
-            fi
-            if [ "$(docker inspect -f '{{.State.Running}}' "${FEDORA_CANARY_CONTAINER}")" != "true" ]; then
-              echo "::error::Fedora systemd container exited before systemd became reachable"
-              docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true
-              exit 1
-            fi
-            sleep 1
-          done
+      - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
 
-          if ! docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then
-            echo "::error::Fedora systemd container did not become reachable within 120s"
-            docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true
-            exit 1
-          fi
+      - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
+        with:
+          name: openshell
+
+      - name: Install RPM in Fedora rootful Podman VM and check status
+        run: |
+          set -euo pipefail
 
-          docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \
+          nix run .#test-guest -- \
+            --distro fedora \
+            --with podman-rootful \
+            -- \
+            sudo env \
+            SUDO_USER=root \
             HOME=/root \
             XDG_RUNTIME_DIR=/run/user/0 \
             DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \
+            OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \
+            INSTALL_SH_URL="$INSTALL_SH_URL" \
             bash -s <<'EOF'
           set -euo pipefail
-          # install.sh manages the RPM gateway as a systemd user unit. This
-          # container is booted with systemd as PID 1, but it still has no
-          # login session. Start root's user manager explicitly so the
-          # installer can test service restart and gateway registration
-          # instead of its "restart later" fallback.
+
+          # install.sh manages the RPM gateway as a systemd user unit. Start
+          # root's user manager explicitly so the canary exercises the rootful
+          # service path on a real Fedora VM.
           mkdir -p "${XDG_RUNTIME_DIR}"
           chmod 700 "${XDG_RUNTIME_DIR}"
           systemctl start user-runtime-dir@0.service || true
@@ -125,33 +115,15 @@ jobs:
             systemctl --user status --no-pager >&2 || true
             exit 1
           fi
-          EOF
-
-      - name: Install and check status
-        run: |
-          set -euo pipefail
 
-          docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \
-            HOME=/root \
-            XDG_RUNTIME_DIR=/run/user/0 \
-            DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \
-            OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \
-            INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh" \
-            bash -s <<'EOF'
-          set -euo pipefail
           mkdir -p "${HOME}/.config/openshell"
-          printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \
+          printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_PODMAN_SOCKET=/run/podman/podman.sock\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \
             "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env"
-          podman info
+          podman --url unix:///run/podman/podman.sock info
           curl -LsSf "${INSTALL_SH_URL}" | sh
           openshell status
           EOF
 
-      - name: Stop Fedora systemd container
-        if: always()
-        run: |
-          docker rm -f "${FEDORA_CANARY_CONTAINER}" >/dev/null 2>&1 || true
-
   ubuntu-snap:
     name: Ubuntu Snap
     if: ${{ github.event.workflow_run.conclusion == 'success' }}