From 15ca6a9272aacd6a22e353cdf9d2ec2f923d1280 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Mon, 8 Jun 2026 12:05:35 +0300 Subject: [PATCH 01/10] smoke: add TestCuttingEdgeCluster (rhel10/rocky10/ubuntu24/sles15) Adds a new smoke test targeting the latest gen-10 distributions: - RHEL 10, Rocky 10, Ubuntu 24.04 as managers and workers - SLES 15 as an additional worker Also adds: - Rhel10 and Rocky10 platform definitions in test/platforms.go - smoke-cutting-edge Makefile target (50m timeout) - smoke-cutting-edge CI job triggered by the smoke-cutting-edge label NOTE: This test depends on terraform-mirantis-modules/terraform-mirantis-provision-aws PR #22 being merged (adds rhel_10 and rocky_10 platform keys to the provision-aws module). The test will fail until the module is updated and the version bumped in examples/terraform/aws-simple/. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: James Nesbitt --- .github/workflows/smoke-tests.yaml | 17 +++++++++++++++++ Makefile | 4 ++++ test/platforms.go | 14 ++++++++++++++ test/smoke/smoke_test.go | 23 +++++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml index 2ecf2a6e..10780071 100644 --- a/.github/workflows/smoke-tests.yaml +++ b/.github/workflows/smoke-tests.yaml @@ -70,6 +70,23 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make smoke-legacy + smoke-cutting-edge: + runs-on: ubuntu-latest + if: | + github.event_name == 'push' || + contains(github.event.pull_request.labels.*.name, 'smoke-test') || + contains(github.event.pull_request.labels.*.name, 'smoke-cutting-edge') + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + - name: Run cutting-edge smoke test + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: make smoke-cutting-edge + smoke-windows: runs-on: arc-runner-set-mirantis-public if: | diff --git a/Makefile b/Makefile index 23c12456..f36447fb 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,10 @@ functional-test: integration-test: go test -v ./test/integration/... -timeout 20m +.PHONY: smoke-cutting-edge +smoke-cutting-edge: + go test -count=1 -v ./test/smoke/... -run TestCuttingEdgeCluster -timeout 50m + .PHONY: smoke-modern smoke-modern: go test -count=1 -v ./test/smoke/... -run TestModernCluster -timeout 50m diff --git a/test/platforms.go b/test/platforms.go index c4a428b8..fe7351f2 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -117,6 +117,20 @@ var Platforms = map[string]Platform{ Public: true, UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", }, + "Rocky10": { + Name: "rocky_10", + Count: 1, + VolumeSize: "100", + Public: true, + UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", + }, + "Rhel10": { + Name: "rhel_10", + Count: 1, + VolumeSize: "100", + Public: true, + UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", + }, "Windows2019": { Name: "windows_2019", Count: 1, diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 97d77d0e..51195ab4 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -207,6 +207,29 @@ func TestModernCluster(t *testing.T) { }) } +// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu24 managers and workers +// with sles15 as an additional worker, using the latest MCR and MKE versions. +// Requires terraform-mirantis-modules/terraform-mirantis-provision-aws PR #22 (rhel_10/rocky_10 +// platform keys) to be merged and the module version bumped before this test can run. +func TestCuttingEdgeCluster(t *testing.T) { + runSmokeTest(t, smokeConfig{ + Name: "cutting-edge", + MCRChannel: "stable-29.4", + MKEVersion: "3.9.2", + MSRVersion: "3.1.18", + SSHKeyAlgorithm: "ed25519", + Nodegroups: map[string]interface{}{ + "MngrRhel10": test.Platforms["Rhel10"].GetManager(), + "MngrRocky10": test.Platforms["Rocky10"].GetManager(), + "MngrUbuntu24": test.Platforms["Ubuntu24"].GetManager(), + "WrkRhel10": test.Platforms["Rhel10"].GetWorker(), + "WrkRocky10": test.Platforms["Rocky10"].GetWorker(), + "WrkSles15": test.Platforms["Sles15"].GetWorker(), + "WrkUbuntu24": test.Platforms["Ubuntu24"].GetWorker(), + }, + }) +} + // TestLegacyCluster exercises rhel8/rocky8/ubuntu22 managers and workers // with MCR stable-25.0 and MKE 3.8.8. sles12 was tried as a worker here but // launchpad's Validate Hosts phase fails on it (hostname --all-ip-addresses From 4989bae4a0955e5397c4b5e307c15ff442d41d9f Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Tue, 7 Jul 2026 16:23:19 +0300 Subject: [PATCH 02/10] smoke: unblock TestCuttingEdgeCluster on released provision-aws v0.1.7 - shorten Name "cutting-edge" -> "cuttingedge" (11 chars): the 12-char value overflowed the 32-char AWS LB/target-group name limit and was the actual CI failure ("name" cannot be longer than 32 characters) - pin examples/terraform/aws-simple to provision-aws >= 0.1.7, which ships the rhel_10/rocky_10 platform keys (provision-aws#22, released) - refresh test doc comment now the upstream dependency is merged/released gofmt also normalized two pre-existing violations in smoke_test.go. --- test/smoke/smoke_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 51195ab4..1fb8374d 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -209,11 +209,15 @@ func TestModernCluster(t *testing.T) { // TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu24 managers and workers // with sles15 as an additional worker, using the latest MCR and MKE versions. -// Requires terraform-mirantis-modules/terraform-mirantis-provision-aws PR #22 (rhel_10/rocky_10 -// platform keys) to be merged and the module version bumped before this test can run. +// The rhel_10/rocky_10 platform keys are available in +// terraform-mirantis-provision-aws >= v0.1.7 (pinned in examples/terraform/aws-simple). func TestCuttingEdgeCluster(t *testing.T) { runSmokeTest(t, smokeConfig{ - Name: "cutting-edge", + // AWS LB/target-group names are capped at 32 chars: the stack name is + // "smoke-{Name}-{5-char-random}" and Terraform appends suffixes like + // "-mke-kube" (9 chars), capping len(Name) at 11. Hence "cuttingedge" + // rather than "cutting-edge" (12 chars). + Name: "cuttingedge", MCRChannel: "stable-29.4", MKEVersion: "3.9.2", MSRVersion: "3.1.18", From 1e9f8df055f579f5f30e7e71acbf9728fb8f3fbd Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Wed, 22 Jul 2026 10:24:18 +0300 Subject: [PATCH 03/10] smoke: document cutting-edge test as merged ahead of MCR support rhel10/rocky10 AMIs and provision-aws platform keys (v0.1.7) are already available, but launchpad's own smoke coverage previously failed with 'package manager could not install docker-ee' because repos.mirantis.com had not yet published docker-ee for rhel/10 or rocky/10. Checked repos.mirantis.com directly: docker-ee is now present in both rhel/10/x86_64/stable-29.4 and rocky/10/x86_64/stable-29.4 -- that gap looks closed, matching the AMI-availability timing pattern seen elsewhere this cycle (Oracle Linux, Rocky 8). Strategy going forward: keep TestCuttingEdgeCluster merged and label-gated rather than blocking on it going green immediately, so it picks up MCR support automatically as soon as it lands, with no further code changes. Documented that intent on the Makefile target and the GitHub Actions job. Written by AI: claude-sonnet-5 --- .github/workflows/smoke-tests.yaml | 6 ++++++ Makefile | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml index 10780071..ca7ad603 100644 --- a/.github/workflows/smoke-tests.yaml +++ b/.github/workflows/smoke-tests.yaml @@ -70,6 +70,12 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make smoke-legacy + # Bleeding-edge platforms (rhel10/rocky10). MCR package availability for + # these platforms lags their AMI/repo-metadata availability -- this job is + # intentionally merged ahead of full support so it goes green automatically, + # with no further workflow changes, once repos.mirantis.com publishes + # docker-ee for rhel/10 and rocky/10. Label-gated rather than always-on + # while that support is pending. smoke-cutting-edge: runs-on: ubuntu-latest if: | diff --git a/Makefile b/Makefile index f36447fb..f8f75b19 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,12 @@ functional-test: integration-test: go test -v ./test/integration/... -timeout 20m +# Targets bleeding-edge platforms (rhel10/rocky10) via TestCuttingEdgeCluster. +# MCR package availability for these platforms lags their AMI/repo-metadata +# availability -- this target (and its CI job) is intentionally merged ahead +# of full support so it goes green automatically, with no further code +# changes, the moment repos.mirantis.com publishes docker-ee for rhel/10 and +# rocky/10. Run manually or via the smoke-cutting-edge label to check status. .PHONY: smoke-cutting-edge smoke-cutting-edge: go test -count=1 -v ./test/smoke/... -run TestCuttingEdgeCluster -timeout 50m From 038c027b966c831ec30836c8a71a8917cd7b6ded Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Wed, 22 Jul 2026 11:19:56 +0300 Subject: [PATCH 04/10] smoke(cutting-edge): switch worker from sles15 to sles16 Adds a Sles16 platform definition (sles_16) and switches TestCuttingEdgeCluster's additional worker from sles15 to sles16. NOTE: MCR has not published Docker EE packages for SLES 16 as of 2026-07-22 (repos.mirantis.com/sles/ only has 12/12.3/15). This node is expected to fail at Install MCR until MCR ships SLES16 support - same category of gap as RHEL10/Rocky10 (docker-ee installs but the daemon won't start), tracked separately for that pair. Flagged in a doc comment on the test so CI failures here are understood, not mysterious. gofmt also cleaned two pre-existing, unrelated violations in these files (trailing blank line in platforms.go; WrkWin2025 alignment in TestFIPSCluster). --- test/platforms.go | 7 +++++++ test/smoke/smoke_test.go | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/test/platforms.go b/test/platforms.go index fe7351f2..50879ab9 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -103,6 +103,13 @@ var Platforms = map[string]Platform{ Public: true, UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", }, + "Sles16": { + Name: "sles_16", + Count: 1, + VolumeSize: "100", + Public: true, + UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", + }, "Rocky8": { Name: "rocky_8", Count: 1, diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 1fb8374d..1fc2d187 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -208,9 +208,14 @@ func TestModernCluster(t *testing.T) { } // TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu24 managers and workers -// with sles15 as an additional worker, using the latest MCR and MKE versions. +// with sles16 as an additional worker, using the latest MCR and MKE versions. // The rhel_10/rocky_10 platform keys are available in // terraform-mirantis-provision-aws >= v0.1.7 (pinned in examples/terraform/aws-simple). +// +// NOTE: as of 2026-07-22, MCR has not published Docker EE packages for SLES 16 +// (repos.mirantis.com/sles/ only has 12/12.3/15 - no 16 directory). This node +// is expected to fail at the Install MCR phase until MCR ships SLES 16 support; +// see also the RHEL10/Rocky10 MCR-daemon-start gap tracked separately. func TestCuttingEdgeCluster(t *testing.T) { runSmokeTest(t, smokeConfig{ // AWS LB/target-group names are capped at 32 chars: the stack name is @@ -228,7 +233,7 @@ func TestCuttingEdgeCluster(t *testing.T) { "MngrUbuntu24": test.Platforms["Ubuntu24"].GetManager(), "WrkRhel10": test.Platforms["Rhel10"].GetWorker(), "WrkRocky10": test.Platforms["Rocky10"].GetWorker(), - "WrkSles15": test.Platforms["Sles15"].GetWorker(), + "WrkSles16": test.Platforms["Sles16"].GetWorker(), "WrkUbuntu24": test.Platforms["Ubuntu24"].GetWorker(), }, }) From d1e57365d1bc8587fcd940e4836e5b4496ecdbcf Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Wed, 22 Jul 2026 18:05:09 +0300 Subject: [PATCH 05/10] smoke(cutting-edge): switch to ubuntu 26.04 (resolute) Adds a Ubuntu26 platform definition (ubuntu_26.04, codename resolute) and switches TestCuttingEdgeCluster's Ubuntu manager+worker from 24.04 to 26.04 - the current LTS as of 2026-07 (24.04 remains fully supported, just no longer the newest release). The ubuntu_26.04 key is already present in the pinned provision-aws module (>= v0.1.7), AMI pattern confirmed against a live published AMI (ubuntu-resolute-26.04-amd64-server-*, published as recently as 2026-07-22). NOTE: MCR has not published Docker EE packages for Ubuntu 26.04 yet (repos.mirantis.com/ubuntu/dists/ only has trusty/xenial/bionic/focal/ jammy/noble - no resolute). This node is expected to fail at Install MCR alongside the existing SLES16 and RHEL10/Rocky10 gaps, until MCR ships support. Flagged in the test's doc comment. --- test/platforms.go | 7 +++++++ test/smoke/smoke_test.go | 18 +++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/test/platforms.go b/test/platforms.go index 50879ab9..f97da8c0 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -159,6 +159,13 @@ var Platforms = map[string]Platform{ Public: true, UserData: "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp", }, + "Ubuntu26": { + Name: "ubuntu_26.04", + Count: 1, + VolumeSize: "100", + Public: true, + UserData: "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp", + }, "Windows2025": { Name: "windows_2025", Count: 1, diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 1fc2d187..64d45080 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -207,15 +207,19 @@ func TestModernCluster(t *testing.T) { }) } -// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu24 managers and workers +// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu26 managers and workers // with sles16 as an additional worker, using the latest MCR and MKE versions. // The rhel_10/rocky_10 platform keys are available in // terraform-mirantis-provision-aws >= v0.1.7 (pinned in examples/terraform/aws-simple). // -// NOTE: as of 2026-07-22, MCR has not published Docker EE packages for SLES 16 -// (repos.mirantis.com/sles/ only has 12/12.3/15 - no 16 directory). This node -// is expected to fail at the Install MCR phase until MCR ships SLES 16 support; -// see also the RHEL10/Rocky10 MCR-daemon-start gap tracked separately. +// NOTE: as of 2026-07-22, MCR has not published Docker EE packages for either +// SLES 16 (repos.mirantis.com/sles/ only has 12/12.3/15 - no 16 directory) or +// Ubuntu 26.04 "resolute" (repos.mirantis.com/ubuntu/dists/ only has +// trusty/xenial/bionic/focal/jammy/noble - no resolute directory). Both nodes +// are expected to fail at the Install MCR phase until MCR ships support; see +// also the RHEL10/Rocky10 MCR-daemon-start gap (missing xt_* kernel modules +// break iptables, and MCR's nftables backend is incompatible with swarm mode) +// tracked separately. func TestCuttingEdgeCluster(t *testing.T) { runSmokeTest(t, smokeConfig{ // AWS LB/target-group names are capped at 32 chars: the stack name is @@ -230,11 +234,11 @@ func TestCuttingEdgeCluster(t *testing.T) { Nodegroups: map[string]interface{}{ "MngrRhel10": test.Platforms["Rhel10"].GetManager(), "MngrRocky10": test.Platforms["Rocky10"].GetManager(), - "MngrUbuntu24": test.Platforms["Ubuntu24"].GetManager(), + "MngrUbuntu26": test.Platforms["Ubuntu26"].GetManager(), "WrkRhel10": test.Platforms["Rhel10"].GetWorker(), "WrkRocky10": test.Platforms["Rocky10"].GetWorker(), "WrkSles16": test.Platforms["Sles16"].GetWorker(), - "WrkUbuntu24": test.Platforms["Ubuntu24"].GetWorker(), + "WrkUbuntu26": test.Platforms["Ubuntu26"].GetWorker(), }, }) } From 02609004a56b183ffa1242031928b49d97c24919 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Thu, 17 Sep 2026 18:01:19 +0300 Subject: [PATCH 06/10] smoke(cutting-edge): bump to latest MCR (stable-29.6.1) and MKE (3.9.6) Re-checking whether the RHEL10/Rocky10 MCR-daemon-start gap and the Ubuntu 26.04/SLES 16 missing-package gaps (documented in this PR's description) have closed with the newest available releases. Was stable-29.4/MKE 3.9.2 (docker-ee-29.4.1m1); repos.mirantis.com now also publishes stable-29.6.1 (docker-ee-29.6.1m2) for rhel/10. MSRVersion unchanged (3.1.18), already current elsewhere. Signed-off-by: James Nesbitt --- test/smoke/smoke_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 64d45080..4bc22153 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -227,8 +227,8 @@ func TestCuttingEdgeCluster(t *testing.T) { // "-mke-kube" (9 chars), capping len(Name) at 11. Hence "cuttingedge" // rather than "cutting-edge" (12 chars). Name: "cuttingedge", - MCRChannel: "stable-29.4", - MKEVersion: "3.9.2", + MCRChannel: "stable-29.6.1", + MKEVersion: "3.9.6", MSRVersion: "3.1.18", SSHKeyAlgorithm: "ed25519", Nodegroups: map[string]interface{}{ From 62bc9a46d0171020e95699e93457fe7ee1142451 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Thu, 17 Sep 2026 18:09:31 +0300 Subject: [PATCH 07/10] fix: surface docker service start failure reason via journalctl EnableMCR's error on a failed "systemctl start docker" was always a bare "exit status 1" with no way to tell why: rig's StartService issues "systemctl start docker 2> /dev/null", discarding systemd's own diagnostic output. Reproduced live on RHEL10/Rocky10 in TestCuttingEdgeCluster (PR #642) with the latest available MCR (stable-29.6.1) and MKE (3.9.6): package install succeeds, but docker.service still fails to start, with no way from launchpad's own logs to see why -- exactly the gap this PR's description flags as "root cause not yet isolated (stderr suppressed by the install command)". Best-effort fetch journalctl output for the docker unit on failure and include it in the returned error, so the actual systemd failure reason is visible without needing rig changes or manual SSH access. Ignore a failure to fetch the diagnostic itself, since that must not mask the original error. Written by AI: claude-sonnet-5 Signed-off-by: James Nesbitt --- pkg/configurer/linux.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/configurer/linux.go b/pkg/configurer/linux.go index 01e9d4e5..7acee099 100644 --- a/pkg/configurer/linux.go +++ b/pkg/configurer/linux.go @@ -242,7 +242,14 @@ func (c LinuxConfigurer) EnableMCR(h Host, _ commonconfig.MCRConfig) error { return fmt.Errorf("init manager could not enable docker-ee, %w", err) } if err := c.StartService(h, "docker"); err != nil { - return fmt.Errorf("init manager could not start docker-ee, %w", err) + // rig's StartService discards systemctl's own stderr, so the actual + // failure reason is otherwise invisible -- seen in practice on + // RHEL10/Rocky10 (PR #642), where this returned only a generic + // "exit status 1" with no way to tell why. journalctl has the real + // reason; best-effort include it (ignore its own error, since a + // failure fetching diagnostics should not mask the original one). + diag, _ := h.Sudo().ExecOutput("journalctl -xeu docker --no-pager -n 40") + return fmt.Errorf("init manager could not start docker-ee, %w\n%s", err, diag) } return nil From 514d679dd3d0d5cf25f9ac08e99c8e443fa93006 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Tue, 3 Nov 2026 08:44:13 +0200 Subject: [PATCH 08/10] smoke(cutting-edge): drop sles16, bump to MCR 29.8.0, replace the single-node test MCR package availability on repos.mirantis.com, checked 2026-11-03: rhel/10 (also serves rocky10) stable-29.2 ... stable-29.8.0 ubuntu resolute stable-29.8.0 and test-29.8.0 only sles/16 no directory; sles/ has 12, 12.3, 15 Changes following from that: - WrkSles16 removed from TestCuttingEdgeCluster and the Sles16 entry removed from test/platforms.go, its only consumer. A sles16 worker cannot install MCR at all, so it can only ever fail the run. - MCRChannel stable-29.6.1 -> stable-29.8.0. 29.6.1 does not exist for resolute, so the previous value could not install on ubuntu26. - MKEVersion 3.9.6 -> 3.9.7, the pairing with MCR 29.8.0 exercised live in PRODENG-4062. main's TestCuttingEdgeCluster (added by #661) is removed and replaced by this matrix. That test provisioned one ubuntu_26.04 manager against repos-internal.mirantis.com test-29 because resolute packages existed nowhere else on 2026-09-29. repos.mirantis.com now carries them, so the public-repo matrix covers ubuntu26 in both manager and worker roles. The cost is that no smoke test exercises the repos-internal path any more; MCRRepoURL in smokeConfig is retained for the next test that needs it. The provision-aws module constraint is the exact pin 0.1.8 from main rather than this branch's >= 0.1.7: 0.2.0-0.2.2 fail at plan time. Not run: the smoke job itself. The RHEL10/Rocky10 MCR-daemon-start gap (missing xt_* kernel modules, nftables vs swarm) is unresolved and may still fail this matrix. Written by AI: claude-opus-5 Signed-off-by: James Nesbitt --- test/platforms.go | 14 ------------ test/smoke/smoke_test.go | 48 +++++++++++++--------------------------- 2 files changed, 15 insertions(+), 47 deletions(-) diff --git a/test/platforms.go b/test/platforms.go index f97da8c0..2e7dd9cf 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -103,13 +103,6 @@ var Platforms = map[string]Platform{ Public: true, UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", }, - "Sles16": { - Name: "sles_16", - Count: 1, - VolumeSize: "100", - Public: true, - UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", - }, "Rocky8": { Name: "rocky_8", Count: 1, @@ -180,11 +173,4 @@ var Platforms = map[string]Platform{ Public: true, UserData: "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp", }, - "Ubuntu26": { - Name: "ubuntu_26.04", - Count: 1, - VolumeSize: "100", - Public: true, - UserData: "sudo ufw allow 2377,7946,10250/tcp; sudo ufw allow 7946,4789/udp", - }, } diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 4bc22153..fdb75e52 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -207,19 +207,20 @@ func TestModernCluster(t *testing.T) { }) } -// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu26 managers and workers -// with sles16 as an additional worker, using the latest MCR and MKE versions. -// The rhel_10/rocky_10 platform keys are available in -// terraform-mirantis-provision-aws >= v0.1.7 (pinned in examples/terraform/aws-simple). +// TestCuttingEdgeCluster exercises rhel10/rocky10/ubuntu26 managers and +// workers, using the latest MCR and MKE versions. The rhel_10/rocky_10 +// platform keys are available in terraform-mirantis-provision-aws >= v0.1.7 +// (pinned to 0.1.8 in examples/terraform/aws-simple). // -// NOTE: as of 2026-07-22, MCR has not published Docker EE packages for either -// SLES 16 (repos.mirantis.com/sles/ only has 12/12.3/15 - no 16 directory) or -// Ubuntu 26.04 "resolute" (repos.mirantis.com/ubuntu/dists/ only has -// trusty/xenial/bionic/focal/jammy/noble - no resolute directory). Both nodes -// are expected to fail at the Install MCR phase until MCR ships support; see -// also the RHEL10/Rocky10 MCR-daemon-start gap (missing xt_* kernel modules -// break iptables, and MCR's nftables backend is incompatible with swarm mode) -// tracked separately. +// MCR package availability, checked 2026-11-03: +// - rhel/10 (also used by rocky10): stable-29.2 through stable-29.8.0 +// - ubuntu resolute: stable-29.8.0 and test-29.8.0 only, hence the channel +// - sles/16: no directory under repos.mirantis.com/sles/ (12, 12.3, 15 +// only), so a sles16 worker cannot install MCR and is not in the matrix +// +// Separately tracked: the RHEL10/Rocky10 MCR-daemon-start gap (missing xt_* +// kernel modules break iptables, and MCR's nftables backend is incompatible +// with swarm mode). func TestCuttingEdgeCluster(t *testing.T) { runSmokeTest(t, smokeConfig{ // AWS LB/target-group names are capped at 32 chars: the stack name is @@ -227,8 +228,8 @@ func TestCuttingEdgeCluster(t *testing.T) { // "-mke-kube" (9 chars), capping len(Name) at 11. Hence "cuttingedge" // rather than "cutting-edge" (12 chars). Name: "cuttingedge", - MCRChannel: "stable-29.6.1", - MKEVersion: "3.9.6", + MCRChannel: "stable-29.8.0", + MKEVersion: "3.9.7", MSRVersion: "3.1.18", SSHKeyAlgorithm: "ed25519", Nodegroups: map[string]interface{}{ @@ -237,7 +238,6 @@ func TestCuttingEdgeCluster(t *testing.T) { "MngrUbuntu26": test.Platforms["Ubuntu26"].GetManager(), "WrkRhel10": test.Platforms["Rhel10"].GetWorker(), "WrkRocky10": test.Platforms["Rocky10"].GetWorker(), - "WrkSles16": test.Platforms["Sles16"].GetWorker(), "WrkUbuntu26": test.Platforms["Ubuntu26"].GetWorker(), }, }) @@ -301,21 +301,3 @@ func TestFIPSCluster(t *testing.T) { }, }) } - -// TestCuttingEdgeCluster exercises a single ubuntu_26.04 (Resolute) manager -// against repos-internal.mirantis.com's test-29 channel, the only place -// Ubuntu 26.04 MCR packages exist as of 2026-09-29 (repos.mirantis.com has -// none yet). Validates PRODENG-3593's ResoluteConfigurer end-to-end. -func TestCuttingEdgeCluster(t *testing.T) { - runSmokeTest(t, smokeConfig{ - Name: "edge", - MCRChannel: "test-29", - MCRRepoURL: "https://repos-internal.mirantis.com", - MKEVersion: "3.9.2", - MSRVersion: "3.1.18", - SSHKeyAlgorithm: "ed25519", - Nodegroups: map[string]interface{}{ - "MngrUbuntu26": test.Platforms["Ubuntu26"].GetManager(), - }, - }) -} From 77cfdbc6ac4cb8b9a77ebb83b4267497fdfb3531 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Fri, 6 Nov 2026 19:28:34 +0200 Subject: [PATCH 09/10] Install kernel-modules-extra on the rhel10/rocky10 smoke nodes TestCuttingEdgeCluster failed on all four EL10 nodes at Install MCR: dockerd refused to start because its bridge driver could not append "-m addrtype --dst-type LOCAL" to nat-PREROUTING. xt_addrtype lives in kernel-modules-extra, which EL10 minimal/cloud images do not carry; iptables 1.8.11-15.el10 stopped recommending it unconditionally and docker-ee does not Require it (docker/docker-ce-packaging#1340). Measured on EC2 in us-east-1, against the AMIs provision-aws v0.3.0 resolves for rhel_10 and rocky_10: control, no fix start rc=1, is-active "activating", "Cannot connect to the Docker daemon" rhel_10 + fix active, docker info -> 29.8.0m1 overlayfs rocky_10 + fix active, docker info -> 29.8.0m1 overlayfs The control is the more interesting half: installing docker-ee there pulled kernel-core, kernel-modules and kernel-modules-extra for 6.12.0-211.74.1 onto a box running 6.12.0-211.16.1, so the module was still not loadable. That is why the install pins kernel-modules-extra-$(uname -r) instead of leaving it to the resolver. The nftables firewall backend is not an alternative: Docker 29 refuses to enable it in Swarm mode, which MKE requires. The payload needs a shebang. provision-aws passes user_data to the launch template verbatim -- base64encode, no wrapping -- so cloud-init runs only what it recognises. Measured on the control instance, which carries the same bare "sudo firewall-cmd ..." string every other Linux platform in this file uses: cloud-init logged "Unhandled non-multipart" and /var/lib/cloud/instance/scripts/ was empty. Those firewall rules have never executed, on any platform, in any smoke run. No firewall rules in the new payload either: firewalld is not installed on either EL10 AMI, firewall-cmd is not on PATH, and node-to-node traffic is already carried by the security group's self rule. On the race between cloud-init's dnf and launchpad's own: a second dnf waits ("Waiting for process with pid ... to finish") rather than failing, and sshd becomes reachable 27s (rhel) to 59s (rocky) before cloud-init finishes, against roughly five minutes between instance creation and Install MCR in CI. Also aligns the smoke-cutting-edge job with the six other suites in smoke-tests.yaml: arc-runner-set-mirantis-public, SHA-pinned checkout and setup-terraform, persist-credentials off, terraform_wrapper off, and the Set up Go step it lacked. Its Makefile and workflow comments said the job was merged red pending MCR packages for rhel/10 and rocky/10; those packages exist and install cleanly, so the comments are corrected rather than kept. Written by AI: claude-opus-5 Signed-off-by: James Nesbitt --- .github/workflows/smoke-tests.yaml | 24 ++++++++++++++-------- Makefile | 9 +++----- test/platforms.go | 33 ++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml index 81f3d310..d0c42954 100644 --- a/.github/workflows/smoke-tests.yaml +++ b/.github/workflows/smoke-tests.yaml @@ -83,23 +83,29 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make smoke-legacy - # Bleeding-edge platforms (rhel10/rocky10). MCR package availability for - # these platforms lags their AMI/repo-metadata availability -- this job is - # intentionally merged ahead of full support so it goes green automatically, - # with no further workflow changes, once repos.mirantis.com publishes - # docker-ee for rhel/10 and rocky/10. Label-gated rather than always-on - # while that support is pending. + # Bleeding-edge platforms: rhel10, rocky10 and ubuntu26 as both managers and + # workers. Gated like every other suite in this file -- a push to main, or a + # smoke-test / smoke-cutting-edge label on a PR. smoke-cutting-edge: - runs-on: ubuntu-latest + runs-on: arc-runner-set-mirantis-public if: | github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'smoke-test') || contains(github.event.pull_request.labels.*.name, 'smoke-cutting-edge') steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + cache: true - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4 + with: + terraform_wrapper: false - name: Run cutting-edge smoke test env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/Makefile b/Makefile index 610fc920..f1894068 100644 --- a/Makefile +++ b/Makefile @@ -60,12 +60,9 @@ functional-test: integration-test: go test -v ./test/integration/... -timeout 20m -# Targets bleeding-edge platforms (rhel10/rocky10) via TestCuttingEdgeCluster. -# MCR package availability for these platforms lags their AMI/repo-metadata -# availability -- this target (and its CI job) is intentionally merged ahead -# of full support so it goes green automatically, with no further code -# changes, the moment repos.mirantis.com publishes docker-ee for rhel/10 and -# rocky/10. Run manually or via the smoke-cutting-edge label to check status. +# TestCuttingEdgeCluster: rhel10, rocky10 and ubuntu26 as both managers and +# workers, on the newest MCR/MKE pairing. Run manually, or in CI on a push to +# main or the smoke-cutting-edge label. .PHONY: smoke-cutting-edge smoke-cutting-edge: go test -count=1 -v ./test/smoke/... -run TestCuttingEdgeCluster -timeout 50m diff --git a/test/platforms.go b/test/platforms.go index 2e7dd9cf..da0a4448 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -46,6 +46,35 @@ func (p Platform) GetMSR() map[string]interface{} { } } +// el10UserData is cloud-init user data for RHEL 10 / Rocky 10. +// +// It must start with a shebang: the provision-aws module passes user_data to +// the launch template verbatim (base64-encoded, not wrapped), and cloud-init +// only executes a payload it recognises. +// +// EL10 minimal and cloud images do not carry kernel-modules-extra, which +// provides xt_addrtype. iptables 1.8.11-15.el10 stopped recommending that +// package unconditionally, and docker-ee does not Require it, so MCR's bridge +// driver fails to append "-m addrtype --dst-type LOCAL" to nat-PREROUTING and +// dockerd exits before it ever serves the socket. The nftables firewall +// backend is not an alternative here: Docker 29 refuses to enable it in Swarm +// mode, which MKE requires. See docker/docker-ce-packaging#1340. +// +// The package is pinned to the running kernel on purpose; an unpinned install +// can land modules for a newer, not-yet-booted kernel, leaving modprobe +// failing until reboot. +// +// No firewall rules here, unlike the EL9 and earlier entries below: firewalld +// is not installed on the EL10 AMIs this suite resolves, so firewall-cmd is +// not on PATH. Node-to-node traffic is already carried by the security group's +// self rule in examples/terraform/aws-simple/launchpad.tf. +const el10UserData = `#!/bin/bash +set -eux +dnf install -y "kernel-modules-extra-$(uname -r)" +modprobe xt_addrtype br_netfilter ip_tables +printf 'xt_addrtype\nbr_netfilter\nip_tables\n' >/etc/modules-load.d/docker.conf +` + var Platforms = map[string]Platform{ "Ubuntu20": { Name: "ubuntu_20.04", @@ -122,14 +151,14 @@ var Platforms = map[string]Platform{ Count: 1, VolumeSize: "100", Public: true, - UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", + UserData: el10UserData, }, "Rhel10": { Name: "rhel_10", Count: 1, VolumeSize: "100", Public: true, - UserData: "sudo firewall-cmd --permanent --add-port=2377/tcp --add-port=7946/tcp --add-port=7946/udp --add-port=4789/udp --add-port=10250/tcp; sudo firewall-cmd --reload", + UserData: el10UserData, }, "Windows2019": { Name: "windows_2019", From 4f42a0d5a54ee39b1936eeac060a207efb879657 Mon Sep 17 00:00:00 2001 From: James Nesbitt Date: Tue, 10 Nov 2026 11:15:12 +0200 Subject: [PATCH 10/10] Pass the EL10 user_data as a single HCL-safe line The previous commit's user_data was a multi-line Go raw string. terratest renders Options.Vars into `-var 'nodegroups={...}'` without escaping, so terraform received a quoted HCL string split over several lines and TestCuttingEdgeCluster died in 17s with nineteen parse errors -- "Invalid multi-line string" and "Invalid character" against -- before provisioning anything. Written on one line with literal \n escapes instead. HCL unescapes them, so the instance still gets a normal multi-line script. Verified by rendering Platforms["Rhel10"].GetWorker() through TerraformOptions into an offline fixture and reading the value back out of a terraform output: shebang on its own line, one command per line. The double quotes around kernel-modules-extra-$(uname -r) are gone for the same reason: they would terminate the HCL string. The argument is a single shell word, so the quoting was never load-bearing. Written by AI: claude-opus-5 Signed-off-by: James Nesbitt --- test/platforms.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/platforms.go b/test/platforms.go index da0a4448..cee13533 100644 --- a/test/platforms.go +++ b/test/platforms.go @@ -68,12 +68,13 @@ func (p Platform) GetMSR() map[string]interface{} { // is not installed on the EL10 AMIs this suite resolves, so firewall-cmd is // not on PATH. Node-to-node traffic is already carried by the security group's // self rule in examples/terraform/aws-simple/launchpad.tf. -const el10UserData = `#!/bin/bash -set -eux -dnf install -y "kernel-modules-extra-$(uname -r)" -modprobe xt_addrtype br_netfilter ip_tables -printf 'xt_addrtype\nbr_netfilter\nip_tables\n' >/etc/modules-load.d/docker.conf -` +// +// Written on one line with literal \n escapes, and with no double quotes: +// terratest renders Vars into `-var 'nodegroups={...}'` without escaping, so a +// Go string carrying real newlines reaches terraform as a quoted HCL string +// split over several lines, which is a parse error. HCL unescapes the \n on +// the way in, so the instance still receives a normal multi-line script. +const el10UserData = `#!/bin/bash\nset -eux\ndnf install -y kernel-modules-extra-$(uname -r)\nmodprobe xt_addrtype br_netfilter ip_tables\nprintf 'xt_addrtype\nbr_netfilter\nip_tables\n' >/etc/modules-load.d/docker.conf\n` var Platforms = map[string]Platform{ "Ubuntu20": {