INTEROP: Batch test PR combining 8 interop OPP changes

ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-4.22-lpMainline-lp-interop.yaml

@@ -19,6 +19,10 @@ base_images:
     name: "2.17"
     namespace: acm-qe
     tag: multicluster-observability-operator-opp
+  ocs-ci-tests:
+    name: ocs-ci-container
+    namespace: ci
+    tag: stable
   tests-private:
     name: tests-private
     namespace: ci
@@ -36,7 +40,7 @@ images:
   items:
   - dockerfile_literal: |
       FROM this-is-ignored
-      RUN dnf install -y git python39
+      RUN dnf install -y git python39 jq
     from: cli
     optional: true
     to: cli-with-git
@@ -104,6 +108,7 @@ tests:
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
         ]
       QUAY_OPERATOR_CHANNEL: stable-3.17
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
       ZONES_COUNT: "3"
     post:
     - ref: acm-fetch-operator-versions
@@ -134,6 +139,7 @@ tests:
     - ref: acm-opp-app
     - ref: interop-opp-odf-health
     - ref: interop-opp-observability-odf
+    - ref: interop-tests-ocs-tests
     - ref: interop-tests-opp-quay-smoke
     - ref: acm-tests-observability
 - as: cr--full-stack--vsphere
@@ -172,6 +178,7 @@ tests:
       QUAY_OPERATOR_CHANNEL: stable-3.17
       RESOURCE_BIND_TIMEOUT: "180"
       SIZE_VARIANT: large
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
     post:
     - ref: acm-fetch-operator-versions
     - ref: acm-must-gather
@@ -187,6 +194,7 @@ tests:
     - chain: cucushift-installer-check-cluster-health
     - ref: interop-opp-odf-health
     - ref: interop-opp-observability-odf
+    - ref: interop-tests-ocs-tests
     - ref: acm-tests-observability
     - ref: acm-opp-app
     workflow: acm-ipi-vsphere

ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-lpMainline-lp-interop.yaml

@@ -15,6 +15,10 @@ base_images:
     name: "2.17"
     namespace: acm-qe
     tag: multicluster-observability-operator-opp
+  ocs-ci-tests:
+    name: ocs-ci-container
+    namespace: ci
+    tag: stable
   tests-private:
     name: tests-private
     namespace: ci
@@ -32,7 +36,7 @@ images:
   items:
   - dockerfile_literal: |
       FROM this-is-ignored
-      RUN dnf install -y git python39
+      RUN dnf install -y git python39 jq
     from: cli
     optional: true
     to: cli-with-git
@@ -88,6 +92,7 @@ tests:
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
         ]
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
       ZONES_COUNT: "3"
     post:
     - ref: acm-fetch-operator-versions
@@ -111,11 +116,15 @@ tests:
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
+    - ref: interop-opp-wait-mcp
+    - ref: stackrox-opp-readiness
+    - ref: stackrox-opp-smoke
     - ref: acm-tests-clc-smoke
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app
     - ref: interop-opp-odf-health
     - ref: interop-opp-observability-odf
+    - ref: interop-tests-ocs-tests
     - ref: interop-tests-opp-quay-smoke
     - ref: acm-tests-observability
 - as: cr--full-stack--vsphere
@@ -153,13 +162,19 @@ tests:
         ]
       RESOURCE_BIND_TIMEOUT: "180"
       SIZE_VARIANT: large
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
     post:
     - ref: acm-fetch-operator-versions
     - ref: acm-must-gather
     - ref: mce-must-gather
     - chain: ipi-vsphere-post
     - ref: mpiit-data-router-reporter
     - ref: firewatch-report-issues
+    pre:
+    - chain: ipi-vsphere-pre
+    - ref: interop-opp-disk-diag
+    - ref: interop-opp-kubelet-config
+    - ref: interop-opp-wait-for-api
     test:
     - ref: install-operators
     - ref: acm-mch
@@ -168,6 +183,7 @@ tests:
     - chain: cucushift-installer-check-cluster-health
     - ref: interop-opp-odf-health
     - ref: interop-opp-observability-odf
+    - ref: interop-tests-ocs-tests
     - ref: acm-tests-observability
     - ref: acm-opp-app
     workflow: acm-ipi-vsphere

ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-lpMainline-lp-interop.yaml

@@ -32,7 +32,7 @@ images:
   items:
   - dockerfile_literal: |
       FROM this-is-ignored
-      RUN dnf install -y git python39
+      RUN dnf install -y git python39 jq
     from: cli
     optional: true
     to: cli-with-git
@@ -89,6 +89,7 @@ tests:
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
         ]
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
       ZONES_COUNT: "3"
     post:
     - ref: acm-fetch-operator-versions
@@ -112,9 +113,14 @@ tests:
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
-    - ref: acm-tests-clc-create
+    - ref: interop-opp-wait-mcp
+    - ref: stackrox-opp-readiness
+    - ref: stackrox-opp-smoke
+    - ref: acm-tests-clc-smoke
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app
+    - ref: interop-opp-odf-health
+    - ref: interop-opp-observability-odf
     - ref: interop-tests-ocs-tests
     - ref: interop-tests-opp-quay-smoke
     - ref: acm-tests-observability
@@ -153,20 +159,30 @@ tests:
         [
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
         ]
+      RESOURCE_BIND_TIMEOUT: "180"
       SIZE_VARIANT: large
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
     post:
     - ref: acm-fetch-operator-versions
     - ref: acm-must-gather
     - ref: mce-must-gather
     - chain: ipi-vsphere-post
     - ref: mpiit-data-router-reporter
     - ref: firewatch-report-issues
+    pre:
+    - chain: ipi-vsphere-pre
+    - ref: interop-opp-disk-diag
+    - ref: interop-opp-kubelet-config
+    - ref: interop-opp-wait-for-api
     test:
     - ref: install-operators
     - ref: acm-mch
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
+    - ref: interop-opp-wait-mcp
+    - ref: interop-opp-odf-health
+    - ref: interop-opp-observability-odf
     - ref: interop-tests-ocs-tests
     - ref: acm-tests-observability
     - ref: acm-opp-app

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml

@@ -19,6 +19,10 @@ base_images:
     name: "2.17"
     namespace: acm-qe
     tag: multicluster-observability-operator-opp
+  ocs-ci-tests:
+    name: ocs-ci-container
+    namespace: ci
+    tag: stable
   tests-private:
     name: tests-private
     namespace: ci
@@ -36,7 +40,7 @@ images:
   items:
   - dockerfile_literal: |
       FROM this-is-ignored
-      RUN dnf install -y git python39
+      RUN dnf install -y git python39 jq
     from: cli
     optional: true
     to: cli-with-git
@@ -95,7 +99,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "3"
       COMPUTE_NODE_TYPE: m6a.2xlarge
       CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge
-      FAIL_ON_BREACH: "true"
+      FAIL_ON_BREACH: "false"
       FIPS_ENABLED: "true"
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["4.22-lp","opp-aws-lp","opp-lp","fips"]'
@@ -109,7 +113,7 @@ tests:
           {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}
         ]
       QUAY_OPERATOR_CHANNEL: stable-3.17
-      SKIP_POLICIES: policy-acs,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
+      SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status
       ZONES_COUNT: "3"
     post:
     - ref: acm-fetch-operator-versions
@@ -138,11 +142,12 @@ tests:
     - ref: interop-opp-wait-mcp
     - ref: stackrox-opp-readiness
     - ref: stackrox-opp-smoke
-    - ref: acm-tests-clc-create
+    - ref: acm-tests-clc-smoke
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app
     - ref: interop-opp-odf-health
     - ref: interop-opp-observability-odf
+    - ref: interop-tests-ocs-tests
     - ref: interop-tests-opp-quay-smoke
     - ref: interop-opp-skip-ratio-gate
 zz_generated_metadata:

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml

@@ -53,7 +53,7 @@ tests:
       COMPUTE_NODE_REPLICAS: "6"
       COMPUTE_NODE_TYPE: m6a.2xlarge
       CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge
-      FAIL_ON_BREACH: "true"
+      FAIL_ON_BREACH: "false"
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["4.22-lp","opp-aws-lp","opp-lp","opp-upgrade"]'
       FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml

@@ -44,16 +44,14 @@ tests:
   steps:
     allow_best_effort_post_steps: true
     cluster_profile: aws-cspi-qe
-    dependencies:
-      OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial
     env:
       ACM_SUBSCRIPTION_NAMESPACE: ocm
       AWS_COMPUTE_VOLUME_SIZE: "500"
       BASE_DOMAIN: cspilp.interop.ccitredhat.com
       COMPUTE_NODE_REPLICAS: "6"
       COMPUTE_NODE_TYPE: m6a.2xlarge
       CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge
-      FAIL_ON_BREACH: "true"
+      FAIL_ON_BREACH: "false"
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["5.0-lp","opp-aws-lp","opp-lp","opp-upgrade"]'
       FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com
@@ -79,7 +77,7 @@ tests:
     - ref: ipi-conf-aws
     - ref: ipi-conf-aws-rootvolume
     - ref: ipi-install-monitoringpvc
-    - chain: ipi-install
+    - chain: ipi-install-stableinitial
     - ref: interop-opp-disk-diag
     - ref: interop-opp-kubelet-config
     - ref: interop-opp-wait-for-api

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1-upgrade.yaml

@@ -44,15 +44,13 @@ tests:
   steps:
     allow_best_effort_post_steps: true
     cluster_profile: aws-cspi-qe
-    dependencies:
-      OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial
     env:
       AWS_COMPUTE_VOLUME_SIZE: "500"
       BASE_DOMAIN: cspilp.interop.ccitredhat.com
       COMPUTE_NODE_REPLICAS: "6"
       COMPUTE_NODE_TYPE: m6a.2xlarge
       CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge
-      FAIL_ON_BREACH: "true"
+      FAIL_ON_BREACH: "false"
       FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json
       FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["5.1-lp","opp-aws-lp","opp-lp","opp-upgrade"]'
       FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com
@@ -76,7 +74,7 @@ tests:
     - ref: ipi-conf-aws
     - ref: ipi-conf-aws-rootvolume
     - ref: ipi-install-monitoringpvc
-    - chain: ipi-install
+    - chain: ipi-install-stableinitial
     - ref: interop-opp-disk-diag
     - ref: interop-opp-kubelet-config
     - ref: interop-opp-wait-for-api
@@ -86,6 +84,7 @@ tests:
     - ref: interop-opp-preflight
     - ref: interop-opp-upgrade
     - ref: cucushift-upgrade-healthcheck
+    - ref: interop-opp-product-upgrade-acm
     - ref: interop-opp-smoke
     - ref: interop-opp-skip-ratio-gate
 zz_generated_metadata:

ci-operator/step-registry/acm/opp-app/acm-opp-app-commands.sh

@@ -2,6 +2,12 @@
 set -euxo pipefail
 shopt -s inherit_errexit
 
+# shellcheck disable=SC2317
+_propagate_junit () {
+    mkdir -p "${SHARED_DIR}/junit"
+    find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true
+}
+
 ################################################################################
 # Test Overview
 ################################################################################
@@ -283,7 +289,7 @@ function RunTestCase2 () {
 # Test Execution Setup
 ################################################################################
 # Set trap to generate JUnit XML on exit (regardless of success or failure)
-trap '{( GenerateJunitXml; true )}' EXIT
+trap '{( GenerateJunitXml; _propagate_junit; true )}' EXIT
 
 if [ "${MAP_TESTS:-}" = "true" ]; then
     eval "$(
@@ -295,6 +301,7 @@ if [ "${MAP_TESTS:-}" = "true" ]; then
     if type -t ExitTrap--PostProcessPrep; then
         trap '{(
             GenerateJunitXml
+            _propagate_junit
             LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \
                 ExitTrap--PostProcessPrep junit--acm-opp-app.xml
             true

ci-operator/step-registry/acm/opp-app/acm-opp-app-ref.yaml

@@ -1,5 +1,6 @@
 ref:
   as: acm-opp-app
+  best_effort: true
   from: cli-with-git
   commands: acm-opp-app-commands.sh
   timeout: 1h0m0s

ci-operator/step-registry/acm/policies/openshift-plus/acm-policies-openshift-plus-commands.sh

@@ -109,7 +109,7 @@ if (( ${#skipPoliciesArr[@]} > 0 )); then
       : "Non-skipped policies failed to become compliant:"
       oc get policies -n policies --ignore-not-found | grep -Ev "/(${skipRegex})$"
       oc get policies -n policies --ignore-not-found -o yaml
-      oc describe policies --all -n policies
+      oc describe policies -n policies
       exit 1
     fi
   else
@@ -125,7 +125,7 @@ else
     : "Policies failed to become compliant:"
     oc get policies -n policies --ignore-not-found
     oc get policies -n policies --ignore-not-found -o yaml
-    oc describe policies --all -n policies
+    oc describe policies -n policies
     exit 1
   fi
 fi

ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh

@@ -34,6 +34,16 @@ cleanup() {
     [[ -d "${CLUSTER_PATH}/auth" ]] && rm -rf "${CLUSTER_PATH}/auth"
 }
 
+_propagate_junit() {
+    local shared_junit="${SHARED_DIR}/junit"
+    mkdir -p "${shared_junit}"
+    # ocs-tests writes JUnit into ${ARTIFACT_DIR}/ocs-tests/
+    local ocs_dir="${ARTIFACT_DIR}/ocs-tests"
+    if compgen -G "${ocs_dir}"/junit*.xml > /dev/null 2>&1; then
+        cp -v "${ocs_dir}"/junit*.xml "${shared_junit}/"
+    fi
+}
+
 if [ "${MAP_TESTS}" = "true" ]; then
     # Avoid conflicts with the older versioned yq from the image:
     # Write /tmp/bin/yq as a tiny script (#!/bin/sh; exit 1), so yq --version fails and ExitTrap EnsureReqs downloads latest yq (replacing the stub).
@@ -49,7 +59,7 @@ if [ "${MAP_TESTS}" = "true" ]; then
             ExitTrap--PostProcessPrep junit--odf__interop-tests__ocs-tests__interop-tests-ocs-tests.xml
     ' EXIT
 else
-    trap 'cleanup' EXIT
+    trap 'cleanup; _propagate_junit' EXIT
 fi
 
 #