Skip to content

[release-5.0] OCPBUGS-114738: E2E: Fix ovs dynamic pinning, kubelet and mustgather tests #1621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
openshift-cherrypick-robot wants to merge 4 commits into
base: release-5.0
Choose a base branch
from
+84 −92
Open

[release-5.0] OCPBUGS-114738: E2E: Fix ovs dynamic pinning, kubelet and mustgather tests #1621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2732046
E2E: ovs: Use systemctl reboot instead of modifying profile
mrniranjan Jul 18, 2026
52e9b1b
E2E: annotations. modify existing tests to preserve existing annotations
mrniranjan Jul 18, 2026
1b1d8d0
E2E: mustgather: Fetch performance profile instead of hardcoding
mrniranjan Jul 18, 2026
b51cfaa
E2E: kubelet Experimental Annotation fixes
mrniranjan Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions test/e2e/performanceprofile/functests/6_mustgather_testing/mustgather.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strings"

machineconfigv1 "github.com/openshift/api/machineconfiguration/v1"
"github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components"
testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift"
Expand Down Expand Up @@ -61,16 +62,18 @@ var _ = Describe("[rfe_id: 50649] Performance Addon Operator Must Gather", Label
if profile == nil {
Skip("No Performance Profile found")
}
//replace peformance.yaml for profile.Name when data is generated in the node
profileName := profile.Name
kubeletConfigName := components.GetComponentName(profileName, components.ComponentNamePrefix)
tunedName := components.GetComponentName(profileName, components.ProfileNamePerformance)
ClusterSpecificFiles := []string{
"cluster-scoped-resources/performance.openshift.io/performanceprofiles/performance.yaml",
"cluster-scoped-resources/machineconfiguration.openshift.io/kubeletconfigs/performance-performance.yaml",
"namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/openshift-node-performance-performance.yaml",
fmt.Sprintf("cluster-scoped-resources/performance.openshift.io/performanceprofiles/%s.yaml", profileName),
fmt.Sprintf("cluster-scoped-resources/machineconfiguration.openshift.io/kubeletconfigs/%s.yaml", kubeletConfigName),
fmt.Sprintf("namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/%s.yaml", tunedName),
}
// On a hypershift env, the tuned file name has an indentifier in the end
if hypershiftutils.IsHypershiftCluster() {
ClusterSpecificFiles = []string{
"namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/openshift-node-performance-performance-*.yaml",
fmt.Sprintf("namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/%s-*.yaml", tunedName),
}
}
By(fmt.Sprintf("Checking Folder: %q\n", mgContentFolder))
Expand Down
63 changes: 10 additions & 53 deletions test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package __performance_kubelet_node_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"path/filepath"
"strconv"
Expand Down Expand Up @@ -33,9 +32,7 @@ import (
testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/poolname"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/systemd"
)

Expand All @@ -58,8 +55,7 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab
isolatedCPUSet cpuset.CPUSet
workerRTNode *corev1.Node
workerRTNodes []corev1.Node
profile, initialProfile *performancev2.PerformanceProfile
poolName string
profile *performancev2.PerformanceProfile
ovsSliceCgroup string
ctx context.Context = context.Background()
ovsSystemdServices []string
Expand Down Expand Up @@ -89,8 +85,6 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab
profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())

poolName = poolname.GetByProfile(ctx, profile)

isCgroupV2, err = cgroup.IsVersion2(ctx, testclient.DataPlaneClient)
Expect(err).ToNot(HaveOccurred())

Expand Down Expand Up @@ -160,56 +154,19 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab

})

Context("[Performance Profile Modified]", Label(string(label.Tier1)), func() {
BeforeEach(func() {
initialProfile = profile.DeepCopy()
})
Context("[Node Reboot]", Label(string(label.Tier1)), func() {
It("[test_id:64099] Activation file doesn't get deleted", func() {
policy := "best-effort"
// Need to make some changes to pp , causing system reboot
// and check if activation files is modified or deleted
profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred(), "Unable to fetch latest performance profile")
currentPolicy := profile.Spec.NUMA.TopologyPolicy
if *currentPolicy == "best-effort" {
policy = "restricted"
}
profile.Spec.NUMA = &performancev2.NUMA{
TopologyPolicy: &policy,
}
By("Updating the performance profile")
profiles.UpdateWithRetry(profile)

By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName))
profilesupdate.WaitForTuningUpdating(ctx, profile)

By(fmt.Sprintf("Waiting when %s finishes updates", poolName))
profilesupdate.WaitForTuningUpdated(ctx, profile)
By(fmt.Sprintf("Rebooting the worker node %q", workerRTNode.Name))
_, _ = nodes.ExecCommand(ctx, workerRTNode, []string{"sh", "-c", "chroot /rootfs systemctl reboot"})
nodes.WaitForNotReadyOrFail("Reboot", workerRTNode.Name, 10*time.Minute, 30*time.Second)
nodes.WaitForReadyOrFail("Reboot", workerRTNode.Name, 10*time.Minute, 30*time.Second)

By("Checking Activation file")
cmd := []string{"ls", activation_file}
for _, node := range workerRTNodes {
output, err := nodes.ExecCommand(context.TODO(), &node, cmd)
Expect(err).ToNot(HaveOccurred(), "file %s doesn't exist ", activation_file)
out := testutils.ToString(output)
Expect(out).To(Equal(activation_file))
}
})
AfterEach(func() {
By("Reverting the Profile")
profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())
currentSpec, _ := json.Marshal(profile.Spec)
spec, _ := json.Marshal(initialProfile.Spec)
if !bytes.Equal(currentSpec, spec) {
profiles.UpdateWithRetry(initialProfile)

By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName))
profilesupdate.WaitForTuningUpdating(ctx, profile)

By(fmt.Sprintf("Waiting when %s finishes updates", poolName))
profilesupdate.WaitForTuningUpdated(ctx, profile)
}
output, err := nodes.ExecCommand(context.TODO(), workerRTNode, cmd)
Expect(err).ToNot(HaveOccurred(), "file %s doesn't exist", activation_file)
out := testutils.ToString(output)
Expect(out).To(Equal(activation_file))
})
})
})
Expand Down
100 changes: 66 additions & 34 deletions test/e2e/performanceprofile/functests/7_performance_kubelet_node/kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package __performance_kubelet_node_test

import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
Expand All @@ -24,6 +23,7 @@ import (
testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils"
testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/infrastructure"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes"
"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/poolname"
Expand All @@ -41,14 +41,13 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
)

testutils.CustomBeforeAll(func() {
// TODO: The code here is broken as it masks workerRTNodes defined above
// and the value of workerRTNodes is never used. Keep the linter happy for now
// and remove the ineffectual assignment of workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) below.
workerRTNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels)
var err error
workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())

_, err = nodes.MatchingOptionalSelector(workerRTNodes)
workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes)
Expect(err).ToNot(HaveOccurred())
Expect(workerRTNodes).ToNot(BeEmpty(), "no RT worker nodes found")

profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())
Expand All @@ -58,6 +57,12 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord

})
Context("Additional kubelet arguments", Label(string(label.Tier2)), func() {
BeforeEach(func() {
var err error
profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())
})

It("[test_id:45488]Test performance profile annotation for changing multiple kubelet settings", func() {
sysctls := "{\"allowedUnsafeSysctls\":[\"net.core.somaxconn\",\"kernel.msg*\"],\"systemReserved\":{\"memory\":\"300Mi\"},\"kubeReserved\":{\"memory\":\"768Mi\"},\"imageMinimumGCAge\":\"3m\"}"
profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, sysctls)
Expand All @@ -72,24 +77,25 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
profilesupdate.WaitForTuningUpdated(ctx, profile)

for _, node := range workerRTNodes {
kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node)
kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node)
Expect(err).ToNot(HaveOccurred())
sysctlsValue := kubeletConfig.AllowedUnsafeSysctls
Expect(sysctlsValue).Should(ContainElements("net.core.somaxconn", "kernel.msg*"))
Expect(kubeletConfig.KubeReserved["memory"]).To(Equal("768Mi"))
Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).To(Equal(180))
Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).To(BeNumerically("==", 180))
}
kubeletArguments := []string{"/bin/bash", "-c", "ps -ef | grep kubelet | grep config"}

autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"}
for _, node := range workerRTNodes {
out, err := nodes.ExecCommand(context.TODO(), &node, kubeletArguments)
out, err := nodes.ExecCommand(ctx, &node, autoSizingCmd)
Expect(err).ToNot(HaveOccurred())
stdout := testutils.ToString(out)
Expect(strings.Contains(stdout, "300Mi")).To(BeTrue())
Expect(stdout).To(ContainSubstring("300Mi"))
}
})
Context("When setting cpu manager related parameters", func() {
It("[test_id:45493]Should not override performance-addon-operator values", func() {
paoValues := "{\"cpuManagerPolicy\":\"static\",\"cpuManagerReconcilePeriod\":\"5s\"}"
paoValues := "{\"cpuManagerPolicy\":\"none\",\"cpuManagerReconcilePeriod\":\"10s\"}"
profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, paoValues)

By("updating Performance profile")
Expand All @@ -102,10 +108,10 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
profilesupdate.WaitForTuningUpdated(ctx, profile)

for _, node := range workerRTNodes {
kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node)
kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node)
Expect(err).ToNot(HaveOccurred())
Expect(kubeletConfig.CPUManagerPolicy).Should(Equal("static"))
Expect(kubeletConfig.CPUManagerReconcilePeriod.Seconds()).To(Equal(5))
Expect(kubeletConfig.CPUManagerReconcilePeriod.Seconds()).To(BeNumerically("==", 5))
}
})
})
Expand Down Expand Up @@ -133,7 +139,7 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
Name: components.GetComponentName(profile.Name, components.ComponentNamePrefix),
Namespace: metav1.NamespaceNone,
}
err := testclient.ControlPlaneClient.Get(context.TODO(), configKey, &kubeletConfig)
err := testclient.ControlPlaneClient.Get(ctx, configKey, &kubeletConfig)
if err != nil {
klog.Warningf("Failed to get the KubeletConfig %q", configKey.Name)
}
Expand All @@ -144,10 +150,19 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
Expect(kubeletConfigString).To(ContainSubstring(`"kubeReserved":{"memory":"768Mi"}`))
Expect(kubeletConfigString).To(ContainSubstring(`"systemReserved":{"memory":"300Mi"}`))

for _, node := range workerRTNodes {
kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node)
// Re-fetch nodes to get current allocatable and capacity after
// the tuning update, since workerRTNodes was populated before the
// annotation was applied and its Status values are stale.
updatedNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())
updatedNodes, err = nodes.MatchingOptionalSelector(updatedNodes)
Expect(err).ToNot(HaveOccurred())
Expect(updatedNodes).ToNot(BeEmpty(), "no RT worker nodes found after update")

for _, node := range updatedNodes {
kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node)
Expect(err).ToNot(HaveOccurred())
totalCapactity := node.Status.Capacity.Memory().MilliValue()
totalCapacity := node.Status.Capacity.Memory().MilliValue()
evictionMemory := kubeletConfig.EvictionHard["memory.available"]
kubeReserved := kubeletConfig.KubeReserved["memory"]
evictionMemoryInt, err := strconv.ParseInt(strings.TrimSuffix(evictionMemory, "Mi"), 10, 64)
Expand All @@ -158,12 +173,32 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
kubeReservedMemoryResource := resource.NewQuantity(kubeReservedMemoryInt*1024*1024, resource.BinarySI)
evictionMemoryResource := resource.NewQuantity(evictionMemoryInt*1024*1024, resource.BinarySI)
totalKubeMemory := systemReservedResource.MilliValue() + kubeReservedMemoryResource.MilliValue() + evictionMemoryResource.MilliValue()
calculatedAllocatable := totalCapactity - totalKubeMemory

// Pre-allocated hugepages are subtracted from allocatable memory by the
// kubelet but are still included in node capacity. The standard formula
// Allocatable = Capacity - kubeReserved - systemReserved - evictionHard
// does not account for this, so we must subtract hugepages to match the
// actual allocatable reported by the node.
var totalHugepages int64
for resourceName, quantity := range node.Status.Capacity {
if strings.HasPrefix(string(resourceName), corev1.ResourceHugePagesPrefix) {
totalHugepages += quantity.MilliValue()
}
}

calculatedAllocatable := totalCapacity - totalKubeMemory - totalHugepages
currentAllocatable := node.Status.Allocatable.Memory().MilliValue()
Expect(calculatedAllocatable).To(Equal(currentAllocatable))
}
})

It("[test_id:45495] Test setting PAO managed parameters", func() {
isArm, err := infrastructure.IsARM(ctx, &workerRTNodes[0])
Expect(err).ToNot(HaveOccurred())
if isArm {
Skip("Changing topologyManagerPolicy is not supported on ARM architecture")
}

var paoParameters string
if *profile.Spec.NUMA.TopologyPolicy == "single-numa-node" {
paoParameters = "{\"topologyManagerPolicy\":\"restricted\"}"
Expand All @@ -186,7 +221,7 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
profilesupdate.WaitForTuningUpdated(ctx, profile)
}
for _, node := range workerRTNodes {
kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node)
kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node)
Expect(err).ToNot(HaveOccurred())
Expect(kubeletConfig.TopologyManagerPolicy).To(Equal("single-numa-node"))
}
Expand All @@ -196,38 +231,35 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord
By("Reverting the Profile")
profiles.UpdateWithRetry(initialProfile)

kubeletArguments := []string{"/bin/bash", "-c", "ps -ef | grep kubelet | grep config"}
By(fmt.Sprintf("Waiting for %s to be updated after revert", poolName))
profilesupdate.WaitForTuningUpdated(ctx, initialProfile)

for _, node := range workerRTNodes {
kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node)
kubeletConfig, err := nodes.GetKubeletConfig(ctx, &node)
Expect(err).ToNot(HaveOccurred())
Expect(kubeletConfig.AllowedUnsafeSysctls).To(Equal(nil))
Expect(kubeletConfig.AllowedUnsafeSysctls).To(BeEmpty())
Expect(kubeletConfig.KubeReserved["memory"]).ToNot(Equal("768Mi"))
Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).ToNot(Equal(180))
}

autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"}
for _, node := range workerRTNodes {
out, err := nodes.ExecCommand(context.TODO(), &node, kubeletArguments)
out, err := nodes.ExecCommand(ctx, &node, autoSizingCmd)
Expect(err).ToNot(HaveOccurred())
stdout := testutils.ToString(out)
Expect(strings.Contains(stdout, "300Mi")).To(BeTrue())
Expect(stdout).ToNot(ContainSubstring("300Mi"))
}

})
AfterAll(func() {
By("Reverting the Profile")
profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels)
Expect(err).ToNot(HaveOccurred())
currentSpec, _ := json.Marshal(profile.Spec)
spec, _ := json.Marshal(initialProfile.Spec)
// revert only if the profile changes.
if !equality.Semantic.DeepEqual(currentSpec, spec) {
if !equality.Semantic.DeepEqual(profile.Spec, initialProfile.Spec) || !equality.Semantic.DeepEqual(profile.Annotations, initialProfile.Annotations) {
profiles.UpdateWithRetry(initialProfile)

By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName))
profilesupdate.WaitForTuningUpdating(ctx, initialProfile)

By(fmt.Sprintf("Waiting when %s finishes updates", poolName))
By(fmt.Sprintf("Waiting for %s to be updated after revert", poolName))
profilesupdate.WaitForTuningUpdated(ctx, initialProfile)

}
})

Expand Down