smoke: add TestCuttingEdgeCluster (rhel10/rocky10/ubuntu26)

.github/workflows/smoke-tests.yaml

@@ -83,6 +83,35 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: make smoke-legacy
 
+  # Bleeding-edge platforms: rhel10, rocky10 and ubuntu26 as both managers and
+  # workers. Gated like every other suite in this file -- a push to main, or a
+  # smoke-test / smoke-cutting-edge label on a PR.
+  smoke-cutting-edge:
+    runs-on: arc-runner-set-mirantis-public
+    if: |
+      github.event_name == 'push' ||
+      contains(github.event.pull_request.labels.*.name, 'smoke-test') ||
+      contains(github.event.pull_request.labels.*.name, 'smoke-cutting-edge')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4
+        with:
+          terraform_wrapper: false
+      - name: Run cutting-edge smoke test
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: make smoke-cutting-edge
+
   smoke-windows:
     runs-on: arc-runner-set-mirantis-public
     if: |

Makefile

@@ -60,6 +60,13 @@ functional-test:
 integration-test:
 	go test -v ./test/integration/... -timeout 20m
 
+# TestCuttingEdgeCluster: rhel10, rocky10 and ubuntu26 as both managers and
+# workers, on the newest MCR/MKE pairing. Run manually, or in CI on a push to
+# main or the smoke-cutting-edge label.
+.PHONY: smoke-cutting-edge
+smoke-cutting-edge:
+	go test -count=1 -v ./test/smoke/... -run TestCuttingEdgeCluster -timeout 50m
+
 .PHONY: smoke-modern
 smoke-modern:
 	go test -count=1 -v ./test/smoke/... -run TestModernCluster -timeout 50m

pkg/configurer/linux.go

@@ -273,7 +273,14 @@ func (c LinuxConfigurer) EnableMCR(h Host, _ commonconfig.MCRConfig) error {
 		return fmt.Errorf("init manager could not enable docker-ee, %w", err)
 	}
 	if err := c.StartService(h, "docker"); err != nil {
-		return fmt.Errorf("init manager could not start docker-ee, %w", err)
+		// rig's StartService discards systemctl's own stderr, so the actual
+		// failure reason is otherwise invisible -- seen in practice on
+		// RHEL10/Rocky10 (PR #642), where this returned only a generic
+		// "exit status 1" with no way to tell why. journalctl has the real
+		// reason; best-effort include it (ignore its own error, since a
+		// failure fetching diagnostics should not mask the original one).
+		diag, _ := h.Sudo().ExecOutput("journalctl -xeu docker --no-pager -n 40")
+		return fmt.Errorf("init manager could not start docker-ee, %w\n%s", err, diag)
 	}
 
 	return nil

test/platforms.go

@@ -46,6 +46,36 @@ func (p Platform) GetMSR() map[string]interface{} {
 	}
 }
 
+// el10UserData is cloud-init user data for RHEL 10 / Rocky 10.
+//
+// It must start with a shebang: the provision-aws module passes user_data to
+// the launch template verbatim (base64-encoded, not wrapped), and cloud-init
+// only executes a payload it recognises.
+//
+// EL10 minimal and cloud images do not carry kernel-modules-extra, which
+// provides xt_addrtype. iptables 1.8.11-15.el10 stopped recommending that
+// package unconditionally, and docker-ee does not Require it, so MCR's bridge
+// driver fails to append "-m addrtype --dst-type LOCAL" to nat-PREROUTING and
+// dockerd exits before it ever serves the socket. The nftables firewall
+// backend is not an alternative here: Docker 29 refuses to enable it in Swarm
+// mode, which MKE requires. See docker/docker-ce-packaging#1340.
+//
+// The package is pinned to the running kernel on purpose; an unpinned install
+// can land modules for a newer, not-yet-booted kernel, leaving modprobe
+// failing until reboot.
+//
+// No firewall rules here, unlike the EL9 and earlier entries below: firewalld
+// is not installed on the EL10 AMIs this suite resolves, so firewall-cmd is
+// not on PATH. Node-to-node traffic is already carried by the security group's
+// self rule in examples/terraform/aws-simple/launchpad.tf.
+//
+// Written on one line with literal \n escapes, and with no double quotes:
+// terratest renders Vars into `-var 'nodegroups={...}'` without escaping, so a
+// Go string carrying real newlines reaches terraform as a quoted HCL string
+// split over several lines, which is a parse error. HCL unescapes the \n on
+// the way in, so the instance still receives a normal multi-line script.
+const el10UserData = `#!/bin/bash\nset -eux\ndnf install -y kernel-modules-extra-$(uname -r)\nmodprobe xt_addrtype br_netfilter ip_tables\nprintf 'xt_addrtype\nbr_netfilter\nip_tables\n' >/etc/modules-load.d/docker.conf\n`
+
 var Platforms = map[string]Platform{
 	"Ubuntu20": {
 		Name:       "ubuntu_20.04",
@@ -117,6 +147,20 @@ var Platforms = map[string]Platform{
 		Public:     true,
 		UserData:   "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload",
 	},
+	"Rocky10": {
+		Name:       "rocky_10",
+		Count:      1,
+		VolumeSize: "100",
+		Public:     true,
+		UserData:   el10UserData,
+	},
+	"Rhel10": {
+		Name:       "rhel_10",
+		Count:      1,
+		VolumeSize: "100",
+		Public:     true,
+		UserData:   el10UserData,
+	},
 	"Windows2019": {
 		Name:       "windows_2019",
 		Count:      1,
@@ -138,6 +182,13 @@ var Platforms = map[string]Platform{
 		Public:     true,
 		UserData:   "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp",
 	},
+	"Ubuntu26": {
+		Name:       "ubuntu_26.04",
+		Count:      1,
+		VolumeSize: "100",
+		Public:     true,
+		UserData:   "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp",
+	},
 	"Windows2025": {
 		Name:       "windows_2025",
 		Count:      1,
@@ -152,11 +203,4 @@ var Platforms = map[string]Platform{
 		Public:     true,
 		UserData:   "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp",
 	},
-	"Ubuntu26": {
-		Name:       "ubuntu_26.04",
-		Count:      1,
-		VolumeSize: "100",
-		Public:     true,
-		UserData:   "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp",
-	},
 }

test/smoke/smoke_test.go

@@ -245,6 +245,42 @@ func TestModernCluster(t *testing.T) {
 	})
 }
 
+// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu26 managers and
+// workers, using the latest MCR and MKE versions. The rhel_10/rocky_10
+// platform keys are available in terraform-mirantis-provision-aws >= v0.1.7
+// (pinned to 0.1.8 in examples/terraform/aws-simple).
+//
+// MCR package availability, checked 2026-11-03:
+//   - rhel/10 (also used by rocky10): stable-29.2 through stable-29.8.0
+//   - ubuntu resolute: stable-29.8.0 and test-29.8.0 only, hence the channel
+//   - sles/16: no directory under repos.mirantis.com/sles/ (12, 12.3, 15
+//     only), so a sles16 worker cannot install MCR and is not in the matrix
+//
+// Separately tracked: the RHEL10/Rocky10 MCR-daemon-start gap (missing xt_*
+// kernel modules break iptables, and MCR's nftables backend is incompatible
+// with swarm mode).
+func TestCuttingEdgeCluster(t *testing.T) {
+	runSmokeTest(t, smokeConfig{
+		// AWS LB/target-group names are capped at 32 chars: the stack name is
+		// "smoke-{Name}-{5-char-random}" and Terraform appends suffixes like
+		// "-mke-kube" (9 chars), capping len(Name) at 11. Hence "cuttingedge"
+		// rather than "cutting-edge" (12 chars).
+		Name:            "cuttingedge",
+		MCRChannel:      "stable-29.8.0",
+		MKEVersion:      "3.9.7",
+		MSRVersion:      "3.1.18",
+		SSHKeyAlgorithm: "ed25519",
+		Nodegroups: map[string]interface{}{
+			"MngrRhel10":   test.Platforms["Rhel10"].GetManager(),
+			"MngrRocky10":  test.Platforms["Rocky10"].GetManager(),
+			"MngrUbuntu26": test.Platforms["Ubuntu26"].GetManager(),
+			"WrkRhel10":    test.Platforms["Rhel10"].GetWorker(),
+			"WrkRocky10":   test.Platforms["Rocky10"].GetWorker(),
+			"WrkUbuntu26":  test.Platforms["Ubuntu26"].GetWorker(),
+		},
+	})
+}
+
 // TestLegacyCluster exercises rhel8/rocky8/ubuntu22 managers and workers
 // with MCR stable-25.0 and MKE 3.8.8. sles12 was tried as a worker here but
 // launchpad's Validate Hosts phase fails on it (hostname --all-ip-addresses
@@ -303,21 +339,3 @@ func TestFIPSCluster(t *testing.T) {
 		},
 	})
 }
-
-// TestCuttingEdgeCluster exercises a single ubuntu_26.04 (Resolute) manager
-// against repos-internal.mirantis.com's test-29 channel, the only place
-// Ubuntu 26.04 MCR packages exist as of 2026-09-29 (repos.mirantis.com has
-// none yet). Validates PRODENG-3593's ResoluteConfigurer end-to-end.
-func TestCuttingEdgeCluster(t *testing.T) {
-	runSmokeTest(t, smokeConfig{
-		Name:            "edge",
-		MCRChannel:      "test-29",
-		MCRRepoURL:      "https://repos-internal.mirantis.com",
-		MKEVersion:      "3.9.2",
-		MSRVersion:      "3.1.18",
-		SSHKeyAlgorithm: "ed25519",
-		Nodegroups: map[string]interface{}{
-			"MngrUbuntu26": test.Platforms["Ubuntu26"].GetManager(),
-		},
-	})
-}