diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-4.22-lpMainline-lp-interop.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-4.22-lpMainline-lp-interop.yaml index 0d2a09b2ac787..dfc7f2fe3e4c1 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-4.22-lpMainline-lp-interop.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-4.22-lpMainline-lp-interop.yaml @@ -19,10 +19,18 @@ base_images: name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp + ocs-ci-tests: + name: ocs-ci-container + namespace: ci + tag: stable tests-private: name: tests-private namespace: ci tag: "4.22" + upi-installer: + name: "4.22" + namespace: ocp + tag: upi-installer build_root: image_stream_tag: name: release @@ -97,8 +105,9 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: ftan@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "4.22" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -139,6 +148,7 @@ tests: - ref: acm-opp-app - ref: interop-opp-odf-health - ref: interop-opp-observability-odf + - ref: interop-tests-ocs-tests - ref: interop-tests-opp-quay-smoke - ref: interop-opp-skip-ratio-gate - as: cr--full-stack--vsphere @@ -167,8 +177,9 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: ftan@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "4.22" OPENSHIFT_REQUIRED_CORES: "72" OPENSHIFT_REQUIRED_MEMORY: "288" OPERATORS: | @@ -199,8 +210,9 @@ tests: - ref: interop-opp-wait-mcp - ref: interop-opp-odf-health - ref: interop-opp-observability-odf - - ref: acm-tests-observability + - ref: interop-tests-ocs-tests - ref: acm-opp-app + - ref: interop-opp-skip-ratio-gate zz_generated_metadata: branch: master org: RedHatQE diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-fips-lpMainline-lp-interop.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-fips-lpMainline-lp-interop.yaml new file mode 100644 index 0000000000000..f05f2c80a0756 --- /dev/null +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-fips-lpMainline-lp-interop.yaml @@ -0,0 +1,130 @@ +base_images: + clc-ui-e2e: + name: "2.17" + namespace: acm-qe + tag: clc-ui-e2e + cli: + name: "5.0" + namespace: ocp + tag: cli + fetch-managed-clusters: + name: autotest + namespace: acm-qe + tag: fetch-managed-clusters + ocs-ci-tests: + name: ocs-ci-container + namespace: ci + tag: stable + upi-installer: + name: "5.0" + namespace: ocp + tag: upi-installer +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.26-openshift-5.0 +images: + items: + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y git python39 jq + from: cli + optional: true + to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo +prowgen: + enable_secrets_store_csi_driver: true +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "5.0" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: interop-opp-aws + capabilities: + - intranet + cluster: build05 + cron: 0 5,17 * * * + reporter_config: + channel: '#opp-discussion' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed: + Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> + {{end}}' + steps: + allow_best_effort_post_steps: true + cluster_profile: aws-cspi-qe + env: + BASE_DOMAIN: cspilp.interop.ccitredhat.com + COMPUTE_NODE_REPLICAS: "6" + COMPUTE_NODE_TYPE: m6a.2xlarge + CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge + DR__RP__CR_COMP_NAME: lp-interop--OPP + FIPS_ENABLED: "true" + FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json + FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["5.0-lp","opp-aws-lp","opp-lp"]' + FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com + FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 + FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" + MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "4.22" + OPERATORS: | + [ + {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} + ] + SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status + ZONES_COUNT: "3" + post: + - ref: acm-fetch-operator-versions + - ref: acm-must-gather + - ref: acm-inspector + - ref: acm-tests-clc-destroy + - ref: gather-aws-console + - chain: ipi-deprovision + - ref: mpiit-data-router-reporter + - ref: firewatch-report-issues + pre: + - ref: ipi-conf + - ref: ipi-conf-telemetry + - ref: ipi-conf-aws-custom-az + - ref: ipi-conf-aws + - ref: ipi-install-monitoringpvc + - chain: ipi-install + test: + - ref: install-operators + - ref: acm-mch + - ref: acm-policies-openshift-plus-setup + - ref: acm-policies-openshift-plus + - ref: interop-opp-wait-mcp + - ref: stackrox-opp-readiness + - ref: stackrox-opp-smoke + - ref: acm-tests-clc-smoke + - ref: acm-fetch-managed-clusters + - ref: acm-opp-app + - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf + - ref: interop-tests-ocs-tests + - ref: interop-tests-opp-quay-smoke + - ref: interop-opp-skip-ratio-gate +zz_generated_metadata: + branch: master + org: RedHatQE + repo: interop-testing + variant: opp--ocp-5.0-fips-lpMainline-lp-interop diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-lpMainline-lp-interop.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-lpMainline-lp-interop.yaml index aafca4863ef95..d6c62447ddeaa 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-lpMainline-lp-interop.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.0-lpMainline-lp-interop.yaml @@ -15,6 +15,10 @@ base_images: name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp + ocs-ci-tests: + name: ocs-ci-container + namespace: ci + tag: stable tests-private: name: tests-private namespace: ci @@ -85,8 +89,9 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "4.22" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -115,14 +120,17 @@ tests: - ref: acm-policies-openshift-plus-setup - ref: acm-policies-openshift-plus - ref: interop-opp-wait-mcp + - ref: stackrox-opp-readiness + - ref: stackrox-opp-smoke - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health - ref: interop-opp-observability-odf + - ref: interop-tests-ocs-tests - ref: interop-tests-opp-quay-smoke - - ref: acm-tests-observability -- as: cr--full-stack--vsphere + - ref: interop-opp-skip-ratio-gate +- as: interop-opp-vsphere capabilities: - intranet cluster: vsphere02 @@ -148,8 +156,9 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9323 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "4.22" OPENSHIFT_REQUIRED_CORES: "72" OPENSHIFT_REQUIRED_MEMORY: "288" OPERATORS: | @@ -179,8 +188,9 @@ tests: - ref: interop-opp-wait-mcp - ref: interop-opp-odf-health - ref: interop-opp-observability-odf - - ref: acm-tests-observability + - ref: interop-tests-ocs-tests - ref: acm-opp-app + - ref: interop-opp-skip-ratio-gate workflow: acm-ipi-vsphere zz_generated_metadata: branch: master diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-fips-lpMainline-lp-interop.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-fips-lpMainline-lp-interop.yaml new file mode 100644 index 0000000000000..881dae7432ca6 --- /dev/null +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-fips-lpMainline-lp-interop.yaml @@ -0,0 +1,130 @@ +base_images: + clc-ui-e2e: + name: "2.17" + namespace: acm-qe + tag: clc-ui-e2e + cli: + name: "5.1" + namespace: ocp + tag: cli + fetch-managed-clusters: + name: autotest + namespace: acm-qe + tag: fetch-managed-clusters + ocs-ci-tests: + name: ocs-ci-container + namespace: ci + tag: stable + upi-installer: + name: "5.1" + namespace: ocp + tag: upi-installer +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.26-openshift-5.1 +images: + items: + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y git python39 jq + from: cli + optional: true + to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo +prowgen: + enable_secrets_store_csi_driver: true +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "5.1" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: interop-opp-aws + capabilities: + - intranet + cluster: build05 + cron: 30 5,17 * * * + reporter_config: + channel: '#opp-discussion' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed: + Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> + {{end}}' + steps: + allow_best_effort_post_steps: true + cluster_profile: aws-cspi-qe + env: + BASE_DOMAIN: cspilp.interop.ccitredhat.com + COMPUTE_NODE_REPLICAS: "6" + COMPUTE_NODE_TYPE: m6a.2xlarge + CONTROL_PLANE_INSTANCE_TYPE: m6a.2xlarge + DR__RP__CR_COMP_NAME: lp-interop--OPP + FIPS_ENABLED: "true" + FIREWATCH_CONFIG_FILE_PATH: https://raw.githubusercontent.com/CSPI-QE/cspi-utils/refs/heads/main/firewatch-base-configs/opp/lp-interop-aws.json + FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS: '["5.1-lp","opp-aws-lp","opp-lp"]' + FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com + FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9181 + FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" + MAP_TESTS: "true" + ODF_VERSION_MAJOR_MINOR: "5.1" + OPERATORS: | + [ + {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} + ] + SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status + ZONES_COUNT: "3" + post: + - ref: acm-fetch-operator-versions + - ref: acm-must-gather + - ref: acm-inspector + - ref: acm-tests-clc-destroy + - ref: gather-aws-console + - chain: ipi-deprovision + - ref: mpiit-data-router-reporter + - ref: firewatch-report-issues + pre: + - ref: ipi-conf + - ref: ipi-conf-telemetry + - ref: ipi-conf-aws-custom-az + - ref: ipi-conf-aws + - ref: ipi-install-monitoringpvc + - chain: ipi-install + test: + - ref: install-operators + - ref: acm-mch + - ref: acm-policies-openshift-plus-setup + - ref: acm-policies-openshift-plus + - ref: interop-opp-wait-mcp + - ref: stackrox-opp-readiness + - ref: stackrox-opp-smoke + - ref: acm-tests-clc-smoke + - ref: acm-fetch-managed-clusters + - ref: acm-opp-app + - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf + - ref: interop-tests-ocs-tests + - ref: interop-tests-opp-quay-smoke + - ref: interop-opp-skip-ratio-gate +zz_generated_metadata: + branch: master + org: RedHatQE + repo: interop-testing + variant: opp--ocp-5.1-fips-lpMainline-lp-interop diff --git a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-lpMainline-lp-interop.yaml b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-lpMainline-lp-interop.yaml index e6bda1be6e3b0..c1b989bd26a4b 100644 --- a/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-lpMainline-lp-interop.yaml +++ b/ci-operator/config/RedHatQE/interop-testing/RedHatQE-interop-testing-master__opp--ocp-5.1-lpMainline-lp-interop.yaml @@ -85,13 +85,14 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9181 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" ODF_VERSION_MAJOR_MINOR: "5.1" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} ] + SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status ZONES_COUNT: "3" post: - ref: acm-fetch-operator-versions @@ -115,13 +116,17 @@ tests: - ref: acm-policies-openshift-plus-setup - ref: acm-policies-openshift-plus - ref: interop-opp-wait-mcp - - ref: acm-tests-clc-create + - ref: stackrox-opp-readiness + - ref: stackrox-opp-smoke + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app + - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-ocs-tests - ref: interop-tests-opp-quay-smoke - - ref: acm-tests-observability -- as: cr--full-stack--vsphere + - ref: interop-opp-skip-ratio-gate +- as: interop-opp-vsphere capabilities: - intranet cluster: vsphere02 @@ -148,7 +153,7 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9181 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" MAP_TESTS: "true" ODF_VERSION_MAJOR_MINOR: "5.1" OPENSHIFT_REQUIRED_CORES: "72" @@ -158,6 +163,7 @@ tests: {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} ] SIZE_VARIANT: large + SKIP_POLICIES: policy-acs,policy-acs-central-ca-bundle,policy-acs-central-status,policy-acs-monitor-certs,policy-acs-operator-central,policy-acs-sync-resources,policy-advanced-managed-cluster-security,policy-advanced-managed-cluster-status,policy-compliance-operator-install,policy-config-quay,policy-hub-quay-bridge,policy-install-quay,policy-observability-operator,policy-observability-storage,policy-observability-storage-status,policy-odf,policy-odf-cluster,policy-odf-noobaa,policy-odf-status,policy-quay-bridge,policy-quay-status post: - ref: acm-fetch-operator-versions - ref: acm-must-gather @@ -176,9 +182,11 @@ tests: - ref: acm-policies-openshift-plus-setup - ref: acm-policies-openshift-plus - ref: interop-opp-wait-mcp + - ref: interop-opp-odf-health + - ref: interop-opp-observability-odf - ref: interop-tests-ocs-tests - - ref: acm-tests-observability - ref: acm-opp-app + - ref: interop-opp-skip-ratio-gate workflow: acm-ipi-vsphere zz_generated_metadata: branch: master diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml index 61ee76745ad14..728b7a23a5749 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml @@ -19,6 +19,10 @@ base_images: name: "2.17" namespace: acm-qe tag: multicluster-observability-operator-opp + ocs-ci-tests: + name: ocs-ci-container + namespace: ci + tag: stable tests-private: name: tests-private namespace: ci @@ -71,7 +75,7 @@ tests: capabilities: - intranet cluster: build05 - cron: 0 23 31 2 * + cron: 0 3,15 * * * reporter_config: channel: '#opp-discussion' job_states_to_report: @@ -98,7 +102,8 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-9104 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" + ODF_VERSION_MAJOR_MINOR: "4.22" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"} @@ -132,11 +137,12 @@ tests: - ref: interop-opp-wait-mcp - ref: stackrox-opp-readiness - ref: stackrox-opp-smoke - - ref: acm-tests-clc-create + - ref: acm-tests-clc-smoke - ref: acm-fetch-managed-clusters - ref: acm-opp-app - ref: interop-opp-odf-health - ref: interop-opp-observability-odf + - ref: interop-tests-ocs-tests - ref: interop-tests-opp-quay-smoke - ref: interop-opp-skip-ratio-gate zz_generated_metadata: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml index 167da6248be25..7427ff7b99305 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-upgrade.yaml @@ -30,7 +30,7 @@ tests: capabilities: - intranet cluster: build05 - cron: 0 23 31 2 * + cron: 30 3,15 * * * reporter_config: channel: '#opp-discussion' job_states_to_report: @@ -59,7 +59,7 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-8942 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" ODF_SUBSCRIPTION_NAMESPACE: openshift-storage OPERATORS: | [ diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml index c6052d04bf870..a2723c6309ca1 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0-upgrade.yaml @@ -3,6 +3,10 @@ base_images: name: "5.0" namespace: ocp tag: cli + upi-installer: + name: "5.0" + namespace: ocp + tag: upi-installer build_root: image_stream_tag: name: release @@ -30,7 +34,7 @@ tests: capabilities: - intranet cluster: build05 - cron: 0 23 31 2 * + cron: 0 4,16 * * * reporter_config: channel: '#opp-discussion' job_states_to_report: @@ -44,8 +48,6 @@ tests: steps: allow_best_effort_post_steps: true cluster_profile: aws-cspi-qe - dependencies: - OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial env: ACM_SUBSCRIPTION_NAMESPACE: ocm ACS_SUBSCRIPTION_NAMESPACE: openshift-operators @@ -60,7 +62,7 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-8942 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, @@ -81,7 +83,7 @@ tests: - ref: ipi-conf-aws - ref: ipi-conf-aws-rootvolume - ref: ipi-install-monitoringpvc - - chain: ipi-install + - chain: ipi-install-stableinitial - ref: interop-opp-disk-diag - ref: interop-opp-kubelet-config - ref: interop-opp-wait-for-api diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1-upgrade.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1-upgrade.yaml index 73929715cd94d..670e402ecdbfe 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1-upgrade.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1-upgrade.yaml @@ -3,6 +3,10 @@ base_images: name: "5.1" namespace: ocp tag: cli + upi-installer: + name: "5.1" + namespace: ocp + tag: upi-installer build_root: image_stream_tag: name: release @@ -30,7 +34,7 @@ tests: capabilities: - intranet cluster: build05 - cron: 0 23 31 2 * + cron: 30 4,16 * * * reporter_config: channel: '#opp-discussion' job_states_to_report: @@ -44,9 +48,8 @@ tests: steps: allow_best_effort_post_steps: true cluster_profile: aws-cspi-qe - dependencies: - OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: release:initial env: + ACM_SUBSCRIPTION_NAMESPACE: ocm AWS_COMPUTE_VOLUME_SIZE: "500" BASE_DOMAIN: cspilp.interop.ccitredhat.com COMPUTE_NODE_REPLICAS: "6" @@ -58,7 +61,7 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_EPIC: INTEROP-8942 FIREWATCH_DEFAULT_JIRA_PROJECT: LPINTEROP - FIREWATCH_FAIL_WITH_TEST_FAILURES: "true" + FIREWATCH_FAIL_WITH_TEST_FAILURES: "false" OPERATORS: | [ {"name": "advanced-cluster-management", "source": "redhat-operators", "channel": "release-2.17", "install_namespace": "ocm", "target_namespaces": "ocm", "operator_group": "acm-operator-group"}, @@ -76,7 +79,7 @@ tests: - ref: ipi-conf-aws - ref: ipi-conf-aws-rootvolume - ref: ipi-install-monitoringpvc - - chain: ipi-install + - chain: ipi-install-stableinitial - ref: interop-opp-disk-diag - ref: interop-opp-kubelet-config - ref: interop-opp-wait-for-api @@ -86,6 +89,7 @@ tests: - ref: interop-opp-preflight - ref: interop-opp-upgrade - ref: cucushift-upgrade-healthcheck + - ref: interop-opp-product-upgrade-acm - ref: interop-opp-smoke - ref: interop-opp-skip-ratio-gate zz_generated_metadata: diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml index 43d83fd7cd6cf..a8d8f582def39 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-periodics.yaml @@ -2627,6 +2627,121 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 0 5,17 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: RedHatQE + repo: interop-testing + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe + ci-operator.openshift.io/cluster: build05 + ci-operator.openshift.io/variant: opp--ocp-5.0-fips-lpMainline-lp-interop + ci.openshift.io/generator: prowgen + job-release: "5.0" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.0-fips-lpMainline-lp-interop-interop-opp-aws + reporter_config: + slack: + channel: '#opp-discussion' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed: + Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View + logs> {{end}}' + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=interop-opp-aws + - --variant=opp--ocp-5.0-fips-lpMainline-lp-interop + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 0 3,15 * * * @@ -2761,7 +2876,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "5.0" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.0-lpMainline-lp-interop-cr--full-stack--vsphere + name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.0-lpMainline-lp-interop-interop-opp-vsphere reporter_config: slack: channel: '#opp-discussion' @@ -2785,7 +2900,7 @@ periodics: - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --target=cr--full-stack--vsphere + - --target=interop-opp-vsphere - --variant=opp--ocp-5.0-lpMainline-lp-interop command: - ci-operator @@ -2857,6 +2972,121 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 30 5,17 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: RedHatQE + repo: interop-testing + labels: + capability/intranet: intranet + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe + ci-operator.openshift.io/cluster: build05 + ci-operator.openshift.io/variant: opp--ocp-5.1-fips-lpMainline-lp-interop + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.1-fips-lpMainline-lp-interop-interop-opp-aws + reporter_config: + slack: + channel: '#opp-discussion' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed: + Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View + logs> {{end}}' + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=interop-opp-aws + - --variant=opp--ocp-5.1-fips-lpMainline-lp-interop + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 0 3,15 * * * @@ -2991,7 +3221,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "5.1" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.1-lpMainline-lp-interop-cr--full-stack--vsphere + name: periodic-ci-RedHatQE-interop-testing-master-opp--ocp-5.1-lpMainline-lp-interop-interop-opp-vsphere reporter_config: slack: channel: '#opp-discussion' @@ -3015,7 +3245,7 @@ periodics: - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --target=cr--full-stack--vsphere + - --target=interop-opp-vsphere - --variant=opp--ocp-5.1-lpMainline-lp-interop command: - ci-operator diff --git a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml index 05778977c493f..3eb025b8d047e 100644 --- a/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml +++ b/ci-operator/jobs/RedHatQE/interop-testing/RedHatQE-interop-testing-master-presubmits.yaml @@ -877,6 +877,65 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )opp--ocp-4.22-lpMainline-lp-interop-images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + cluster: build04 + context: ci/prow/opp--ocp-5.0-fips-lpMainline-lp-interop-images + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/variant: opp--ocp-5.0-fips-lpMainline-lp-interop + ci.openshift.io/generator: prowgen + job-release: "5.0" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-RedHatQE-interop-testing-master-opp--ocp-5.0-fips-lpMainline-lp-interop-images + optional: true + rerun_command: /test opp--ocp-5.0-fips-lpMainline-lp-interop-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp--ocp-5.0-fips-lpMainline-lp-interop + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )opp--ocp-5.0-fips-lpMainline-lp-interop-images,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -936,6 +995,65 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )opp--ocp-5.0-lpMainline-lp-interop-images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + cluster: build04 + context: ci/prow/opp--ocp-5.1-fips-lpMainline-lp-interop-images + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/variant: opp--ocp-5.1-fips-lpMainline-lp-interop + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-RedHatQE-interop-testing-master-opp--ocp-5.1-fips-lpMainline-lp-interop-images + optional: true + rerun_command: /test opp--ocp-5.1-fips-lpMainline-lp-interop-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=opp--ocp-5.1-fips-lpMainline-lp-interop + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )opp--ocp-5.1-fips-lpMainline-lp-interop-images,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/jobs/stolostron/policy-collection/stolostron-policy-collection-main-periodics.yaml b/ci-operator/jobs/stolostron/policy-collection/stolostron-policy-collection-main-periodics.yaml index 12fa47c251ed7..cf0bf9c1047c2 100644 --- a/ci-operator/jobs/stolostron/policy-collection/stolostron-policy-collection-main-periodics.yaml +++ b/ci-operator/jobs/stolostron/policy-collection/stolostron-policy-collection-main-periodics.yaml @@ -1,7 +1,7 @@ periodics: - agent: kubernetes cluster: build05 - cron: 0 23 31 2 * + cron: 0 3,15 * * * decorate: true decoration_config: skip_cloning: true @@ -116,7 +116,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 - cron: 0 23 31 2 * + cron: 30 3,15 * * * decorate: true decoration_config: skip_cloning: true @@ -231,7 +231,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 - cron: 0 23 31 2 * + cron: 0 4,16 * * * decorate: true decoration_config: skip_cloning: true @@ -346,7 +346,7 @@ periodics: secretName: result-aggregator - agent: kubernetes cluster: build05 - cron: 0 23 31 2 * + cron: 30 4,16 * * * decorate: true decoration_config: skip_cloning: true diff --git a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh index 82a36317e492c..8038b869a3480 100644 --- a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh +++ b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh @@ -1,8 +1,33 @@ #!/bin/bash -set -o nounset -set -o errexit -set -o pipefail +set -euo pipefail + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" ODF_INSTALL_NAMESPACE=openshift-storage DEFAULT_ODF_OPERATOR_CHANNEL="stable-${ODF_VERSION_MAJOR_MINOR}" diff --git a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml index 7885f4c3504ca..e30976b4331b6 100644 --- a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml +++ b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml @@ -6,6 +6,7 @@ ref: tag: latest commands: interop-tests-deploy-odf-commands.sh timeout: 3h0m0s + grace_period: 10m resources: requests: cpu: 100m @@ -31,4 +32,4 @@ ref: default: "ocs-storagecluster" - name: ODF_VOLUME_SIZE documentation: The size of the ODF volume in Gi - default: "50" \ No newline at end of file + default: "50" diff --git a/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh b/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh index 98e05e4fceb76..135df66da430b 100644 --- a/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh +++ b/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh @@ -1,8 +1,35 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + CLUSTER_VERSION=$(oc get clusterVersion version -o jsonpath='{$.status.desired.version}') OCP_MAJOR_MINOR=$(echo "${CLUSTER_VERSION}" | cut -d '.' -f1,2) OCP_VERSION="${OCP_MAJOR_MINOR}" @@ -23,13 +50,14 @@ fi CLUSTER_NAME=$([[ -f "${SHARED_DIR}/CLUSTER_NAME" ]] && cat "${SHARED_DIR}/CLUSTER_NAME" || echo "cluster-name") CLUSTER_DOMAIN="${CLUSTER_DOMAIN:-release-ci.cnv-qe.rhood.us}" LOGS_FOLDER="${ARTIFACT_DIR}/ocs-tests" -LOGS_CONFIG="${LOGS_FOLDER}/ocs-tests-config.yaml" +LOGS_CONFIG="$(mktemp /tmp/ocs-tests-config.XXXXXX.yaml)" CLUSTER_PATH="${ARTIFACT_DIR}/ocs-tests" export BIN_FOLDER="${LOGS_FOLDER}/bin" # Function to clean up folders cleanup() { + rm -f "${LOGS_CONFIG}" # Tear down local auth copy created for run-ci. [[ -d "${CLUSTER_PATH}/auth" ]] && rm -rf "${CLUSTER_PATH}/auth" } @@ -51,6 +79,7 @@ if [ "${MAP_TESTS}" = "true" ]; then curl -fsSL https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" trap ' + _opp_cleanup cleanup _propagate_junit mkdir -p /tmp/bin @@ -60,7 +89,7 @@ if [ "${MAP_TESTS}" = "true" ]; then ExitTrap--PostProcessPrep junit--odf__interop-tests__ocs-tests__interop-tests-ocs-tests.xml ' EXIT else - trap 'cleanup; _propagate_junit' EXIT + trap '_opp_cleanup; cleanup; _propagate_junit' EXIT fi # @@ -113,7 +142,6 @@ if [[ -f "${SHARED_DIR}/vsphere_context.sh" ]]; then set +x source "${SHARED_DIR}/vsphere_context.sh" source "${SHARED_DIR}/govc.sh" - set -x cat >> "${LOGS_CONFIG}" << __APPENDED_ENV_DATA__ ENV_DATA: @@ -124,6 +152,7 @@ ENV_DATA: vsphere_cluster: "${vsphere_cluster}" vsphere_datastore: "${vsphere_datastore}" __APPENDED_ENV_DATA__ + set -x fi EXTRA_ARGS="" diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 46e61e1e64c23..c90eaf367a3c8 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,5 +1,32 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" @@ -83,7 +110,7 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( GenerateJunit; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; GenerateJunit; _propagate_junit' EXIT function DiscoverQuay () { typeset registryJson="" discoverErr="" @@ -514,16 +541,21 @@ function RunAcsScan () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" + # Mask credentials: function call args expand in xtrace + set +x 2>/dev/null if ! RegisterQuayInAcs "${acsHost}" "${acsPassword}"; then + "${xtraceOn}" && set -x elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Failed to register Quay in ACS" "${elapsed}" return 1 fi if ! RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"; then + "${xtraceOn}" && set -x elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS scan request failed" "${elapsed}" return 1 fi + "${xtraceOn}" && set -x typeset -i attempts=0 maxAttempts=40 @@ -546,7 +578,9 @@ sys.exit(0 if len(images) > 0 else 1) fi if (( attempts % 4 == 3 )); then + set +x 2>/dev/null RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" || true + "${xtraceOn}" && set -x fi attempts=$((attempts + 1)) diff --git a/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh b/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh index a5ea8129d32f3..fce83de75b9bc 100644 --- a/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh +++ b/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + # NOTE: BACKUP_TIMEOUT and OPP_OPERATORS are set via step config YAML (naming deviates from OPP__ convention) BACKUP_TIMEOUT="${BACKUP_TIMEOUT:-300}" OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" @@ -43,15 +70,15 @@ TimeoutMonitor() { # Start timeout monitor in background TimeoutMonitor & typeset timeoutPid=$! -trap 'kill ${timeoutPid} || true' EXIT +trap '_opp_cleanup; kill ${timeoutPid} || true' EXIT trap 'kill ${timeoutPid} || true; exit 124' TERM -: "=== Pre-Upgrade Cluster Backup ===" +echo ">>> PHASE: Pre-Upgrade Cluster Backup" : "Start time: $(date '+%F %T')" : "Backup timeout: ${BACKUP_TIMEOUT}s" # --- Etcd snapshot --- -: "--- Etcd Snapshot ---" +echo ">>> PHASE: Etcd Snapshot" typeset controlPlaneNode="" controlPlaneNode=$(oc get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{.items[0].metadata.name}') || true if [[ -n "${controlPlaneNode}" ]]; then @@ -104,7 +131,7 @@ else fi # --- Control plane resource state --- -: "--- Control Plane State ---" +echo ">>> PHASE: Control Plane State" Capture "ClusterVersion" "${backupDir}/clusterversion.yaml" \ oc get clusterversion version -o yaml @@ -118,7 +145,7 @@ Capture "MachineConfigPools" "${backupDir}/machineconfigpools.yaml" \ oc get machineconfigpools -o yaml # --- OPP operator state --- -: "--- OPP Operator State ---" +echo ">>> PHASE: OPP Operator State" Capture "CSVs" "${backupDir}/csvs.yaml" \ oc get csv -A -o yaml @@ -129,7 +156,7 @@ Capture "InstallPlans" "${backupDir}/installplans.yaml" \ oc get installplans -A -o yaml # --- Backup manifest --- -: "--- Generating Backup Manifest ---" +echo ">>> PHASE: Generating Backup Manifest" typeset clusterVersion="" clusterVersion=$(oc get clusterversion version -o jsonpath='{.status.desired.version}') || true @@ -160,7 +187,7 @@ EOF : "OK: backup-manifest.json" # --- Summary --- -: "=== Backup Summary ===" +echo ">>> PHASE: Backup Summary" : "End time: $(date '+%F %T')" : "Cluster version: ${clusterVersion:-unknown}" : "Node count: ${nodeCount}" diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 10f0f11e5e3cb..ba1faa8f86797 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -1,5 +1,38 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: INTEROP-9455 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # ACM Observability + ODF Interop Validation (6-point gate) @@ -35,13 +68,21 @@ function AddResult () { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. function XmlEscape () { typeset text="${1:-}"; (($#)) && shift + if shopt -q patsub_replacement 2>/dev/null; then + shopt -u patsub_replacement + local _restore_patsub=true + fi text="${text//&/&}" text="${text///>}" text="${text//\"/"}" text="${text//\'/'}" + [[ "${_restore_patsub:-}" == true ]] && shopt -s patsub_replacement printf '%s' "${text}" true } @@ -103,14 +144,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Check 1: ODF Ceph RGW infrastructure ready # --------------------------------------------------------------------------- function CheckRgwReady () { - : "=== Check 1: ODF Ceph RGW infrastructure ===" + echo ">>> PHASE: Check 1 — ODF Ceph RGW infrastructure" typeset rgwPhase="" typeset rgwJson="" rgwErr="" @@ -201,34 +242,93 @@ print(items[0].get('status',{}).get('phase','') if items else '') # --------------------------------------------------------------------------- function CheckMcoReady () { - : "=== Check 2: MultiClusterObservability CR ===" - - typeset mcoStatus="" - if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ - --all-namespaces -o json | python3 -c " + echo ">>> PHASE: Check 2 — MultiClusterObservability CR readiness poll" + + typeset -i maxAttempts=24 + typeset -i sleepSeconds=30 + # Timeout budget: 24 x 30s poll = 720s max + ~180s margin within 900s step timeout + typeset -i attempt=0 + typeset -i startTime=0 + startTime=$(date +%s) + typeset _last_mco_error="" + + # Fast-path: if the CR doesn't exist at all, skip immediately. + # Capture exit status separately so RBAC/connectivity errors are not + # silently treated as "CR absent". + typeset _mco_probe="" _mco_probe_rc=0 + _mco_probe="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + observability --ignore-not-found -o name 2>&1)" || _mco_probe_rc=$? + if (( _mco_probe_rc != 0 )); then + echo "WARNING: MCO CR query failed (exit ${_mco_probe_rc})" + echo "Proceeding to poll loop (may be transient)" + elif [[ -z "${_mco_probe}" ]]; then + echo "MultiClusterObservability CR 'observability' not found" + echo "Observability may not be deployed — skipping MCO readiness check" + AddResult "mco-ready" "skip" "MCO CR not found — observability may not be deployed" + return 0 + fi + + while (( attempt < maxAttempts )); do + (( attempt += 1 )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - startTime )) + echo ">>> MCO poll ${attempt}/${maxAttempts} (${elapsed}s elapsed)…" + + typeset mcoStatus="" mcoConditions="" mcoError="" queryFailed="false" + if ! mcoConditions="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + observability -o jsonpath='{.status.conditions}' 2>&1)"; then + mcoError="${mcoConditions}" + if [[ "${mcoError}" == *"(NotFound)"* && "${mcoError}" == *'"observability" not found'* ]]; then + AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" + return 0 + fi + queryFailed="true" + _last_mco_error="${mcoError}" + elif ! mcoStatus="$(printf '%s' "${mcoConditions}" | python3 -c " import sys,json -d=json.load(sys.stdin) -items=d.get('items',[]) -if not items: - print('NotFound') -else: - conds=items[0].get('status',{}).get('conditions',[]) - ready=[c for c in conds if c.get('type')=='Ready'] - print(ready[0].get('status','Unknown') if ready else 'NoCondition') +raw=sys.stdin.read().strip() +if not raw: + print('NoCondition') + sys.exit(0) +conds=json.loads(raw) +ready=[c for c in conds if c.get('type')=='Ready'] +print(ready[0].get('status','Unknown') if ready else 'NoCondition') ")"; then - AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR" - return - fi + queryFailed="true" + fi - if [[ "${mcoStatus}" == "True" ]]; then - : "PASS: MultiClusterObservability Ready=True" - AddResult "mco-ready" "pass" - elif [[ "${mcoStatus}" == "NotFound" ]]; then - AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" - else - AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)" + if [[ "${queryFailed}" == "true" ]]; then + # Query or parsing failed — might be transient; retry unless last attempt + if (( attempt >= maxAttempts )); then + elapsed=$(( $(date +%s) - startTime )) + AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR after ${elapsed}s" + return 1 + fi + sleep "${sleepSeconds}" + continue + fi + + if [[ "${mcoStatus}" == "True" ]]; then + elapsed=$(( $(date +%s) - startTime )) + : "PASS: MultiClusterObservability Ready=True after ${elapsed}s" + AddResult "mco-ready" "pass" "MultiClusterObservability is Ready after ${elapsed}s" + return 0 + fi + + # Not ready yet — sleep and retry + if (( attempt < maxAttempts )); then + sleep "${sleepSeconds}" + fi + done + + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - startTime )) + typeset _mco_detail="MultiClusterObservability not Ready after ${elapsed}s (last status=${mcoStatus:-unknown})" + if [[ -n "${_last_mco_error}" ]]; then + _mco_detail="${_mco_detail}; last error: ${_last_mco_error:0:200}" fi - true + AddResult "mco-ready" "fail" "${_mco_detail}" + return 1 } # --------------------------------------------------------------------------- @@ -236,7 +336,7 @@ else: # --------------------------------------------------------------------------- function CheckStorageEndpoint () { - : "=== Check 3: Object storage endpoint ===" + echo ">>> PHASE: Check 3 — Object storage endpoint" typeset storageConfig="" if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ @@ -337,7 +437,7 @@ print('odf-backed' if odf_pat.search(endpoint) else 'external') # --------------------------------------------------------------------------- function CheckThanosHealth () { - : "=== Check 4: Thanos components healthy ===" + echo ">>> PHASE: Check 4 — Thanos components healthy" if ! oc get namespace "${obsNamespace}" -o name; then AddResult "thanos-health" "skip" "Observability namespace ${obsNamespace} does not exist" @@ -419,7 +519,7 @@ function CheckThanosHealth () { # --------------------------------------------------------------------------- function CheckObcBound () { - : "=== Check 5: Observability ObjectBucketClaim ===" + echo ">>> PHASE: Check 5 — Observability ObjectBucketClaim" typeset obcList="" obcList="$(oc get obc -n "${obsNamespace}" -o json 2>/dev/null)" || true @@ -541,7 +641,7 @@ print('ok') } function CheckThanosQuery () { - : "=== Check 6: Thanos query functional ===" + echo ">>> PHASE: Check 6 — Thanos query functional" typeset routeJson="" routeJson="$(oc get routes -n "${obsNamespace}" -o json)" || true @@ -668,12 +768,38 @@ print(items[0]['metadata']['name'] if items else '') # Main # --------------------------------------------------------------------------- +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(XmlEscape "${bug_id}")" + safe_desc="$(XmlEscape "${bug_description}")" + safe_error="$(XmlEscape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "ACM Observability + ODF Interop Validation starting" + echo ">>> PHASE: ACM Observability + ODF Interop Validation starting" : "ACM namespace: ${acmNamespace}" : "Observability namespace: ${obsNamespace}" : "ODF namespace: ${odfNamespace}" @@ -686,22 +812,31 @@ function Main () { CheckObcBound || true CheckThanosQuery || true + typeset -i _idx=0 + for _idx in "${!tcResultsArr[@]}"; do + if [[ "${tcNamesArr[$_idx]}" == "thanos-query" \ + && "${tcResultsArr[$_idx]}" == "fail" \ + && "${tcMessagesArr[$_idx]}" == *"returned empty result vector"* ]]; then + # Known-issue skip: INTEROP-9455 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when INTEROP-9455 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${tcMessagesArr[$_idx]}" "INTEROP-9455" \ + "Thanos query returns empty result during observability convergence" + tcResultsArr[$_idx]="skip" + fi + done + WriteJunit - typeset -i hasAnyFail=0 typeset r="" for r in "${tcResultsArr[@]}"; do if [[ "${r}" == "fail" ]]; then - hasAnyFail=1 - break + : "ACM Observability + ODF Interop: SOME CHECKS FAILED" + exit 1 fi done - if (( hasAnyFail )); then - : "ACM Observability + ODF Interop: SOME CHECKS FAILED" - exit 1 - fi - : "ACM Observability + ODF Interop: ALL PASSED" exit 0 } diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml index 07e847330aa08..e8f25f5d1fe5f 100644 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml @@ -1,6 +1,5 @@ ref: as: interop-opp-observability-odf - best_effort: true commands: interop-opp-observability-odf-commands.sh documentation: |- Validates the cross-product integration surface between ACM Observability @@ -27,10 +26,12 @@ ref: - default: "openshift-storage" documentation: Namespace where ODF is installed name: ODF_NAMESPACE + # NOTE: best_effort was intentionally removed to surface step failures. + # Known-issue skips (INTEROP-9455) now handle expected failures explicitly. from: cli grace_period: 30s resources: requests: cpu: 100m memory: 200Mi - timeout: 10m + timeout: 15m diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index c77500c40fae9..cc4cb60692320 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,5 +1,32 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -64,13 +91,23 @@ function AddResult () { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. function XmlEscape () { typeset text="${1:-}"; (($#)) && shift - text="${text//&/&}" - text="${text///>}" - text="${text//\"/"}" - text="${text//\'/'}" + # Disable patsub_replacement if available (bash 5.2+) + # so literal '&' in replacement is not special + if shopt -q patsub_replacement 2>/dev/null; then + shopt -u patsub_replacement + local _restore_patsub=true + fi + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" + [[ "${_restore_patsub:-}" == true ]] && shopt -s patsub_replacement printf '%s' "${text}" true } @@ -152,14 +189,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Check 1: ODF Operator CSV in Succeeded phase # --------------------------------------------------------------------------- function CheckOdfCsv () { - : "=== Check 1: ODF Operator CSV ===" + echo ">>> PHASE: Check 1 — ODF Operator CSV" typeset csvPhase="" if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -187,7 +224,7 @@ print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound') # --------------------------------------------------------------------------- function CheckStorageCluster () { - : "=== Check 2: StorageCluster Ready ===" + echo ">>> PHASE: Check 2 — StorageCluster Ready" typeset scPhase="" if ! scPhase="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -214,7 +251,7 @@ print(d['items'][0]['status'].get('phase','NotFound') if d.get('items') else 'No # --------------------------------------------------------------------------- function CheckCephCluster () { - : "=== Check 3: CephCluster health ===" + echo ">>> PHASE: Check 3 — CephCluster health" typeset cephHealth="" if ! cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -241,7 +278,7 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','NotFound') if # --------------------------------------------------------------------------- function CheckStorageClasses () { - : "=== Check 4: StorageClasses ===" + echo ">>> PHASE: Check 4 — StorageClasses" typeset failMsg="" typeset scName="" @@ -270,7 +307,7 @@ function CheckStorageClasses () { # --------------------------------------------------------------------------- function CheckPvcProvision () { - : "=== Check 5: PVC provisioning (RBD + CephFS) ===" + echo ">>> PHASE: Check 5 — PVC provisioning" typeset -a scTests=("ocs-storagecluster-ceph-rbd" "ocs-storagecluster-cephfs") typeset -a scModes=("ReadWriteOnce" "ReadWriteMany") @@ -333,7 +370,7 @@ EOF # --------------------------------------------------------------------------- function CheckNoobaa () { - : "=== Check 6: NooBaa S3 functional ===" + echo ">>> PHASE: Check 6 — NooBaa S3 functional" typeset nbPhase="" if ! nbPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -521,7 +558,7 @@ EOF # --------------------------------------------------------------------------- function CheckCephHealth () { - : "=== Check 7: Ceph health detail ===" + echo ">>> PHASE: Check 7 — Ceph health detail" typeset cephDetail="" if ! cephDetail="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -563,7 +600,7 @@ function WaitForOdfReady () { typeset -i deadline=$(( SECONDS + timeout )) typeset -i pollInterval=15 - : "Waiting up to ${timeout}s for StorageCluster and NooBaa to reach Ready..." + echo ">>> PHASE: Waiting for ODF subsystems to reach Ready" typeset scPhase="" nbPhase="" while (( SECONDS < deadline )); do @@ -643,7 +680,7 @@ function Main () { export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "ODF Health Check (7-point gate) starting" + echo ">>> PHASE: ODF Health Check (7-point gate) starting" : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" @@ -666,12 +703,12 @@ function Main () { typeset scJson="" scCount="" set +x # suppress xtrace for API response if ! scJson="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then - set -x # restore xtrace + set -x : "Failed to query StorageClusters in ${ODF_NAMESPACE}" exit 1 fi if [[ -z "${scJson}" ]]; then - set -x # restore xtrace + set -x : "StorageCluster query returned empty output in ${ODF_NAMESPACE}" exit 1 fi @@ -681,11 +718,11 @@ items=d.get('items') if not isinstance(items,list): raise ValueError('StorageCluster items is not a list') print(len(items)) ")"; then - set -x # restore xtrace + set -x : "Failed to parse StorageCluster JSON from ${ODF_NAMESPACE}" exit 1 fi - set -x # restore xtrace + set -x if (( scCount == 0 )); then AddResult "odf-csv-phase" "pass" SkipAllChecks "ODF operator installed but no StorageCluster configured in ${ODF_NAMESPACE}" \ diff --git a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh index 4cc43f782c5a8..a64520bebe65b 100755 --- a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh +++ b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh @@ -1,8 +1,35 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" export HOME="${HOME:-/tmp/home}" @@ -45,7 +72,7 @@ function DebugOnExit () { true } -trap '{ EXIT_CODE=$?; DebugOnExit; true; }' EXIT +trap '_opp_cleanup; EXIT_CODE=${_exit_code}; DebugOnExit' EXIT trap '{ EXIT_CODE=143; DebugOnExit; trap - EXIT; exit 143; }' TERM # ────────────────────────────────────────────────────────────────────── @@ -101,7 +128,7 @@ with open(sys.argv[1], 'w') as f: } function CheckApiDeprecations () { - : "=== Check 1: API deprecation scan ===" + echo ">>> PHASE: Check 1 — API deprecation scan" typeset targetMinor="${1}" typeset ocpDisplay="${2:-4.${targetMinor}}" @@ -147,7 +174,13 @@ function CheckApiDeprecations () { } function CheckOppCompatibility () { - : "=== Check 2: OPP operator compatibility matrix ===" + echo ">>> PHASE: Check 2 — OPP operator compatibility matrix" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: OPP compatibility matrix check (IGNORE_SECONDARY_POLICIES=true)" + echo "opp_compatibility_matrix" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AppendCheck "opp_compatibility_matrix" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset ocpKey="${1}" typeset compatSpec="${OPP_COMPAT[${ocpKey}]:-}" @@ -219,7 +252,7 @@ function CheckOppCompatibility () { } function CheckClusterHealth () { - : "=== Check 3: Cluster health baseline ===" + echo ">>> PHASE: Check 3 — Cluster health baseline" typeset failed=0 details="" @@ -314,7 +347,7 @@ print('\n'.join(names)) } function CheckMcpReadiness () { - : "=== Check 4: MachineConfigPool readiness ===" + echo ">>> PHASE: Check 4 — MachineConfigPool readiness" typeset failed=0 details="" @@ -404,7 +437,7 @@ function Main () { sourceVersion="$(oc get clusterversion --no-headers | awk '{print $2}')" : "Source OCP version: ${sourceVersion}" - : "=== Starting OPP pre-flight validation ===" + echo ">>> PHASE: Starting OPP pre-flight validation" InitReport diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh index f1f28f1c71cef..b2a98feed755b 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: ACM-45920 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ACM_TARGET_CHANNEL="${ACM_TARGET_CHANNEL:-}" ACM_UPGRADE_TIMEOUT="${ACM_UPGRADE_TIMEOUT:-30m}" ACM_SUBSCRIPTION_NAME="${ACM_SUBSCRIPTION_NAME:-advanced-cluster-management}" @@ -29,7 +56,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ @@ -310,6 +337,45 @@ function ValidateHubHealth () { return 0 } +# _xml_escape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. +_xml_escape() { + local text="$1" + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" + printf '%s' "${text}" +} + +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(_xml_escape "${bug_id}")" + safe_desc="$(_xml_escape "${bug_description}")" + safe_error="$(_xml_escape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + # === Main === function Main () { @@ -404,8 +470,38 @@ function Main () { newVersion="$(GetInstalledVersion)" echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})" - ValidateMceUpgrade - ValidateHubHealth + typeset _acm_upgrade_output="" + if ! _acm_upgrade_output="$(ValidateMceUpgrade 2>&1)"; then + echo "${_acm_upgrade_output}" + if echo "${_acm_upgrade_output}" | grep -q "cannot unmarshal string into Go struct"; then + # Known-issue skip: ACM-45920 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when ACM-45920 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acm_upgrade_output}" "ACM-45920" \ + "YAML unmarshal error on OCP 5.0 — ACM team fix in progress" + else + exit 1 + fi + else + echo "${_acm_upgrade_output}" + fi + + if ! _acm_upgrade_output="$(ValidateHubHealth 2>&1)"; then + echo "${_acm_upgrade_output}" + if echo "${_acm_upgrade_output}" | grep -q "cannot unmarshal string into Go struct"; then + # Known-issue skip: ACM-45920 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when ACM-45920 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acm_upgrade_output}" "ACM-45920" \ + "YAML unmarshal error on OCP 5.0 — ACM team fix in progress" + else + exit 1 + fi + else + echo "${_acm_upgrade_output}" + fi { printf '=== ACM Operator Upgrade Summary ===\n' diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh index f953e5d4a610c..de85a1b391c4a 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: INTEROP-9466 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ACS_TARGET_CHANNEL="${ACS_TARGET_CHANNEL:-}" ACS_UPGRADE_TIMEOUT="${ACS_UPGRADE_TIMEOUT:-30m}" ACS_SUBSCRIPTION_NAME="${ACS_SUBSCRIPTION_NAME:-rhacs-operator}" @@ -30,7 +57,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ACS_SUBSCRIPTION_NAME}" \ @@ -287,6 +314,45 @@ function ValidateAcsHealth () { return 0 } +# _xml_escape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. +_xml_escape() { + local text="$1" + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" + printf '%s' "${text}" +} + +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(_xml_escape "${bug_id}")" + safe_desc="$(_xml_escape "${bug_description}")" + safe_error="$(_xml_escape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + # === Main === function Main () { @@ -381,7 +447,22 @@ function Main () { newVersion="$(GetInstalledVersion)" echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})" - ValidateAcsHealth + typeset _acs_upgrade_output="" + if ! _acs_upgrade_output="$(ValidateAcsHealth 2>&1)"; then + echo "${_acs_upgrade_output}" + if echo "${_acs_upgrade_output}" | grep -q "SecuredCluster did not reach Deployed"; then + # Known-issue skip: INTEROP-9466 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when INTEROP-9466 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acs_upgrade_output}" "INTEROP-9466" \ + "SecuredCluster reconciliation delay after ACS upgrade" + else + exit 1 + fi + else + echo "${_acs_upgrade_output}" + fi { printf '=== ACS Operator Upgrade Summary ===\n' diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh index a2fc00ad31251..e163f794700c2 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ODF_TARGET_CHANNEL="${ODF_TARGET_CHANNEL:-}" ODF_UPGRADE_TIMEOUT="${ODF_UPGRADE_TIMEOUT:-45m}" ODF_SUBSCRIPTION_NAME="${ODF_SUBSCRIPTION_NAME:-odf-operator}" @@ -30,7 +51,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ODF_SUBSCRIPTION_NAME}" \ diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh index 4ad5329d1c011..a308ddb493da0 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + QUAY_TARGET_CHANNEL="${QUAY_TARGET_CHANNEL:-}" QUAY_UPGRADE_TIMEOUT="${QUAY_UPGRADE_TIMEOUT:-30m}" QUAY_SUBSCRIPTION_NAME="${QUAY_SUBSCRIPTION_NAME:-quay-operator}" @@ -30,7 +51,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${QUAY_SUBSCRIPTION_NAME}" \ diff --git a/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh b/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh index f18ded33d9fd3..9ac13aa8f9810 100644 --- a/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh +++ b/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh @@ -1,5 +1,32 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # OPP post-upgrade smoke tests @@ -34,13 +61,21 @@ AddResult() { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. XmlEscape() { typeset text="${1:-}"; (($#)) && shift + if shopt -q patsub_replacement 2>/dev/null; then + shopt -u patsub_replacement + local _restore_patsub=true + fi text="${text//&/&}" text="${text///>}" text="${text//\"/"}" text="${text//\'/'}" + [[ "${_restore_patsub:-}" == true ]] && shopt -s patsub_replacement printf '%s' "${text}" } @@ -93,14 +128,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Test 1: cluster-health # --------------------------------------------------------------------------- TestClusterHealth() { - : "=== Test: cluster-health ===" + echo ">>> PHASE: Test — cluster-health" typeset failMsg="" # ClusterOperators: Available=True, Degraded!=True @@ -183,7 +218,13 @@ TestClusterHealth() { # --------------------------------------------------------------------------- TestOppOperators() { - : "=== Test: opp-operators ===" + echo ">>> PHASE: Test — opp-operators" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: opp-operators check (IGNORE_SECONDARY_POLICIES=true)" + echo "opp-operators" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "opp-operators" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" typeset -a operatorsArr=() @@ -277,7 +318,13 @@ TestOppOperators() { # --------------------------------------------------------------------------- TestAcmConnectivity() { - : "=== Test: acm-connectivity ===" + echo ">>> PHASE: Test — acm-connectivity" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: acm-connectivity check (IGNORE_SECONDARY_POLICIES=true)" + echo "acm-connectivity" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "acm-connectivity" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Check if ManagedCluster resources exist @@ -324,7 +371,13 @@ TestAcmConnectivity() { # --------------------------------------------------------------------------- TestAcsSensors() { - : "=== Test: acs-sensors ===" + echo ">>> PHASE: Test — acs-sensors" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: acs-sensors check (IGNORE_SECONDARY_POLICIES=true)" + echo "acs-sensors" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "acs-sensors" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Check SecuredCluster CR status first @@ -394,7 +447,13 @@ TestAcsSensors() { # --------------------------------------------------------------------------- TestQuayPull() { - : "=== Test: quay-pull ===" + echo ">>> PHASE: Test — quay-pull" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: quay-pull check (IGNORE_SECONDARY_POLICIES=true)" + echo "quay-pull" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "quay-pull" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Find the Quay registry route @@ -471,7 +530,7 @@ Main() { export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "OPP Smoke Tests starting" + echo ">>> PHASE: OPP Smoke Tests starting" : "Operators: ${OPP_OPERATORS}" : "Settle window: ${SMOKE_SETTLE_SECONDS}s" : "Artifacts dir: ${ARTIFACT_DIR}" diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 79d8ae4f7d5c8..cef2e064152f2 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + # NOTE: UPGRADE_TIMEOUT, POLL_INTERVAL, STALL_WINDOW, OPP_OPERATORS are set via step config YAML # (naming deviates from OPP__ convention) UPGRADE_TIMEOUT="${UPGRADE_TIMEOUT:-130}" @@ -67,7 +94,7 @@ function DebugOnExit () { true } -trap '{ exitCode=$?; DebugOnExit; true; }' EXIT +trap '_opp_cleanup; exitCode=${_exit_code}; DebugOnExit' EXIT trap '{ exitCode=143; DebugOnExit; trap - EXIT; exit 143; }' TERM set +x @@ -217,6 +244,7 @@ function UpdateCcoAnnotation () { function InitiateUpgrade () { typeset isForce="${1:-}"; (($#)) && shift + echo ">>> PHASE: Initiating upgrade" : "Initiating upgrade to ${upgradeTarget}" : "Force flag: ${isForce}" oc adm upgrade --to-image="${upgradeTarget}" --allow-explicit-upgrade --force="${isForce}" @@ -234,6 +262,7 @@ function InitiateUpgrade () { } function MonitorUpgrade () { + echo ">>> PHASE: Monitoring upgrade" typeset -i pollCount=0 typeset -i lastProgressChange=0 lastProgressChange=$(date +%s) @@ -299,6 +328,7 @@ function MonitorUpgrade () { } function StabilizeCluster () { + echo ">>> PHASE: Stabilizing cluster" : "Waiting for cluster stability (minimum-stable-period=5m, timeout=30m)" if ! oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m; then : "Cluster stabilization failed; gathering diagnostics" @@ -312,6 +342,7 @@ function StabilizeCluster () { } function ValidatePlatformHealth () { + echo ">>> PHASE: Validating platform health" : "Validating platform health" typeset avail="" progressing="" degraded="" @@ -351,6 +382,7 @@ function ValidatePlatformHealth () { } function ValidateOppOperators () { + echo ">>> PHASE: Validating OPP operators" : "Validating OPP operator health" typeset -a operatorsArr=() IFS=',' read -ra operatorsArr <<< "${OPP_OPERATORS}" @@ -430,6 +462,8 @@ function Main () { sourceMinorVersion="$(echo "${sourceVersion}" | cut -f2 -d.)" : "Source release: ${sourceVersion} (minor: ${sourceMinorVersion})" + echo ">>> PHASE: OCP upgrade starting" + isForceUpdate="false" if ! CheckSigned "${upgradeTarget}"; then : "Target is unsigned; will use --force" diff --git a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh index 05c434fa3fb11..8806c42cfb32d 100755 --- a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh +++ b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh @@ -1,8 +1,35 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E \ + -e 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' \ + -e 's/Bearer [A-Za-z0-9._~+\/=-]+/Bearer [REDACTED]/g' \ + -e 's/password=[^ &]+/password=[REDACTED]/g' \ + -e 's/token=[^ &]+/token=[REDACTED]/g' \ + -e 's|://[^:@/]*:[^:@/]*@|://[REDACTED]:[REDACTED]@|g' \ + "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + typeset -ri mcpWaitTimeout="${MCP_WAIT_TIMEOUT:-3600}" typeset -ri consecutiveRequired="${MCP_CONSECUTIVE_CHECKS:-3}" typeset -ri settleDelay="${MCP_SETTLE_DELAY:-90}" diff --git a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml index aea0e6302035d..d1d69934db280 100644 --- a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml +++ b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml @@ -39,6 +39,7 @@ ref: documentation: Max seconds to wait for all non-paused MCPs to stabilize (default 60 min) name: MCP_WAIT_TIMEOUT from: cli + grace_period: 1m resources: requests: cpu: 100m