test(features): run feature tests against environment services, starting with Keycloak

.github/workflows/e2e-suites.yml

@@ -48,6 +48,11 @@ jobs:
             family: conformance
             package: sandbox
             runner: ubuntu-24.04
+          - environment: ubuntu-docker-rootful-keycloak
+            installer: deb
+            family: features
+            package: keycloak
+            runner: ubuntu-24.04
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     steps:
@@ -77,7 +82,7 @@ jobs:
           # actions/cache doesn't save after an exact key hit, so a fixed key
           # would rebuild the changed layers on every run. Key on the playbooks
           # and fall back to the newest entry for the same runner.
-          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }}
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix', 'scripts/keycloak-realm.json') }}
           restore-keys: |
             tmachine-${{ runner.os }}-${{ runner.arch }}-
 

TESTING.md

@@ -204,6 +204,24 @@ an explicit matrix and runs the runner once per row. Branch E2E calls it when
 the `test:e2e` label is set. It is advisory for now. To run a suite in another
 environment on CI, add a row.
 
+**Environments with services.** A suite that needs something besides the
+gateway, such as an OIDC issuer, runs in a tmachine environment that provides
+it. The environment's setup starts the service (with a restart policy, since
+setup is cached as a disk) and writes how to reach it as `OPENSHELL_TEST_*`
+lines in `/etc/openshell-test/env.d/<service>.env`. The runner passes those
+variables to the tests and rejects names without that prefix. A suite-local
+fixture reads them with plain environment variables, so you can run the suite
+against your own deployment by setting them yourself. See the
+`ubuntu-docker-rootful-keycloak` environment in `tests/tmachine/config.nix`,
+`tests/tmachine/ansible/playbooks/keycloak.yaml` and
+`tests/suites/features/keycloak`:
+
+```shell
+nix run .#openshell-test-runner -- \
+  --environment ubuntu-docker-rootful-keycloak --installer deb \
+  --family features --package keycloak
+```
+
 The older conformance, feature and driver lanes in `.github/workflows/` still
 run through the tmachine testsuites in `tests/tmachine/config.nix`. They move
 to the runner as their suites move to testkit.

tasks/scripts/check_cargo_lockfile_alignment.py

@@ -38,6 +38,7 @@
 ALIGNED_WORKSPACES = (
     "tests/testkit",
     "tests/suites/conformance",
+    "tests/suites/features",
 )
 
 

tests/runner/src/execute.rs

@@ -7,7 +7,9 @@
 //!
 //! Tests find the gateway the way the CLI does, so the runner points them at
 //! the one the installer registered with `OPENSHELL_GATEWAY` and at the
-//! installed CLI with `OPENSHELL_BIN`.
+//! installed CLI with `OPENSHELL_BIN`. Anything else the environment provides,
+//! such as Keycloak, the tests learn from the variables it publishes (see
+//! [`crate::test_env`]).
 //!
 //! Adapted from the executor in Evan Lezar's NVIDIA/OpenShell#4629.
 
@@ -21,6 +23,7 @@ use tmachine::config::Config;
 
 use crate::bundle::{Bundle, PROFILE, READY_BINARY};
 use crate::report::{Status, decide_status};
+use crate::test_env;
 
 const GUEST_DIR: &str = "/var/tmp/openshell-tests";
 /// How long the gateway has to become healthy after the environment is up.
@@ -126,13 +129,15 @@ async fn provision(config: &Config, environment: &str, installer: &str) -> Resul
 }
 
 /// The environment variables that point the tests at the system under test:
-/// the registered gateway, if known, and the installed CLI.
-pub fn guest_env(gateway: Option<&str>, openshell_bin: &str) -> Vec<String> {
+/// the registered gateway, if known, the installed CLI, and what the
+/// environment publishes for tests (see [`test_env`]).
+pub fn guest_env(gateway: Option<&str>, openshell_bin: &str, test_env: &[String]) -> Vec<String> {
     let mut vars = Vec::new();
     if let Some(gateway) = gateway {
         vars.push(format!("OPENSHELL_GATEWAY={gateway}"));
     }
     vars.push(format!("OPENSHELL_BIN={openshell_bin}"));
+    vars.extend(test_env.iter().cloned());
     vars
 }
 
@@ -166,7 +171,10 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
         !openshell_bin.is_empty(),
         "the installer did not put openshell on the guest's PATH"
     );
-    let env = guest_env(run.gateway, openshell_bin);
+    let test_env = test_env::read(session)
+        .await
+        .context("read the variables the environment publishes for tests")?;
+    let env = guest_env(run.gateway, openshell_bin, &test_env);
 
     let guest = GUEST_DIR;
     checked(
@@ -279,15 +287,26 @@ mod tests {
     #[test]
     fn points_tests_at_the_gateway_and_the_cli() {
         assert_eq!(
-            guest_env(Some("openshell"), "/usr/bin/openshell"),
+            guest_env(Some("openshell"), "/usr/bin/openshell", &[]),
             [
                 "OPENSHELL_GATEWAY=openshell",
                 "OPENSHELL_BIN=/usr/bin/openshell",
             ]
         );
         assert_eq!(
-            guest_env(None, "/usr/bin/openshell"),
+            guest_env(None, "/usr/bin/openshell", &[]),
             ["OPENSHELL_BIN=/usr/bin/openshell"]
         );
+        assert_eq!(
+            guest_env(
+                None,
+                "/usr/bin/openshell",
+                &["OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell".into()]
+            ),
+            [
+                "OPENSHELL_BIN=/usr/bin/openshell",
+                "OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell",
+            ]
+        );
     }
 }

tests/runner/src/lib.rs

@@ -8,3 +8,4 @@
 pub mod bundle;
 pub mod execute;
 pub mod report;
+pub mod test_env;

tests/runner/src/test_env.rs

@@ -0,0 +1,123 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Variables an environment publishes for its tests.
+//!
+//! An environment that runs something besides `OpenShell`, such as Keycloak,
+//! says how tests reach it in `/etc/openshell-test/env.d/*.env`: one
+//! `OPENSHELL_TEST_*=value` per line. The tmachine environment's setup
+//! writes the file; the runner passes its variables to the tests, and suite
+//! fixtures read them. Tests don't depend on tmachine: anyone running a suite
+//! against their own deployment sets the same variables.
+
+use anyhow::{Context, Result, bail, ensure};
+use tmachine::Session;
+
+/// Where environments publish the variables tests read.
+pub const ENV_DIR: &str = "/etc/openshell-test/env.d";
+/// The prefix every published variable must have, so an environment can't
+/// override how tests find the gateway or the CLI.
+pub const ENV_PREFIX: &str = "OPENSHELL_TEST_";
+
+/// Read the variables the booted environment publishes, as `KEY=VALUE`
+/// strings. An environment that publishes none has no `ENV_DIR`.
+pub async fn read(session: &Session) -> Result<Vec<String>> {
+    let script = format!(
+        "for f in {ENV_DIR}/*.env; do [ -e \"$f\" ] || continue; printf '\\0%s\\n' \"$f\"; cat \"$f\"; done"
+    );
+    let output = session.exec(&["sh", "-c", &script]).await?;
+    ensure!(
+        output.status.success(),
+        "read {ENV_DIR}: {}",
+        String::from_utf8_lossy(&output.stderr).trim()
+    );
+    let contents =
+        String::from_utf8(output.stdout).with_context(|| format!("{ENV_DIR} is not UTF-8"))?;
+    parse_files(&contents)
+}
+
+/// Parse the files `read` prints: each is a NUL, its path and a newline,
+/// then its contents.
+fn parse_files(output: &str) -> Result<Vec<String>> {
+    let mut vars = Vec::new();
+    for file in output.split('\0').filter(|file| !file.is_empty()) {
+        let (path, body) = file.split_once('\n').unwrap_or((file, ""));
+        vars.extend(parse_env(path, body)?);
+    }
+    Ok(vars)
+}
+
+/// Parse a published env file: `KEY=VALUE` lines, blank lines and `#`
+/// comments. Values are taken literally; there is no quoting. Errors name
+/// the key, never the value, since values can be credentials.
+pub fn parse_env(path: &str, contents: &str) -> Result<Vec<String>> {
+    let mut vars = Vec::new();
+    for (number, line) in contents.lines().enumerate() {
+        let line = line.trim_end();
+        if line.trim().is_empty() || line.trim_start().starts_with('#') {
+            continue;
+        }
+        let Some((key, _value)) = line.split_once('=') else {
+            bail!("{path}:{}: expected KEY=VALUE", number + 1);
+        };
+        ensure!(
+            key.starts_with(ENV_PREFIX)
+                && key.len() > ENV_PREFIX.len()
+                && key
+                    .bytes()
+                    .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
+            "{path}:{}: {key:?} is not an {ENV_PREFIX}* variable name",
+            number + 1
+        );
+        vars.push(line.to_string());
+    }
+    Ok(vars)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_published_variables() {
+        let vars = parse_env(
+            "keycloak.env",
+            "# Published by keycloak.yaml\n\nOPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell\nOPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b\n",
+        )
+        .unwrap();
+        assert_eq!(
+            vars,
+            [
+                "OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell",
+                "OPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b",
+            ]
+        );
+    }
+
+    #[test]
+    fn reads_every_published_file() {
+        assert_eq!(parse_files("").unwrap(), Vec::<String>::new());
+        let vars = parse_files(
+            "\0/etc/openshell-test/env.d/a.env\nOPENSHELL_TEST_A=1\n\0/etc/openshell-test/env.d/b.env\nOPENSHELL_TEST_B=2",
+        )
+        .unwrap();
+        assert_eq!(vars, ["OPENSHELL_TEST_A=1", "OPENSHELL_TEST_B=2"]);
+
+        let err = parse_files("\0/etc/openshell-test/env.d/b.env\nPATH=/x\n").unwrap_err();
+        assert!(err.to_string().contains("b.env:1"), "{err}");
+    }
+
+    #[test]
+    fn refuses_variables_outside_the_prefix() {
+        let err = parse_env("x.env", "OPENSHELL_GATEWAY=other\n").unwrap_err();
+        assert!(err.to_string().contains("OPENSHELL_GATEWAY"), "{err}");
+    }
+
+    #[test]
+    fn errors_do_not_echo_values() {
+        let err = parse_env("x.env", "openshell_test_secret=hunter2\n").unwrap_err();
+        assert!(!err.to_string().contains("hunter2"), "{err}");
+        let err = parse_env("x.env", "hunter2\n").unwrap_err();
+        assert!(!err.to_string().contains("hunter2"), "{err}");
+    }
+}