diff --git a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh index 82a36317e492c..a4db8aded1414 100644 --- a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh +++ b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh @@ -1,8 +1,27 @@ #!/bin/bash -set -o nounset -set -o errexit -set -o pipefail +set -euo pipefail + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" ODF_INSTALL_NAMESPACE=openshift-storage DEFAULT_ODF_OPERATOR_CHANNEL="stable-${ODF_VERSION_MAJOR_MINOR}" diff --git a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml index 7885f4c3504ca..e30976b4331b6 100644 --- a/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml +++ b/ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml @@ -6,6 +6,7 @@ ref: tag: latest commands: interop-tests-deploy-odf-commands.sh timeout: 3h0m0s + grace_period: 10m resources: requests: cpu: 100m @@ -31,4 +32,4 @@ ref: default: "ocs-storagecluster" - name: ODF_VOLUME_SIZE documentation: The size of the ODF volume in Gi - default: "50" \ No newline at end of file + default: "50" diff --git a/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh b/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh index 98e05e4fceb76..e7998ee10d27a 100644 --- a/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh +++ b/ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh @@ -1,8 +1,29 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + CLUSTER_VERSION=$(oc get clusterVersion version -o jsonpath='{$.status.desired.version}') OCP_MAJOR_MINOR=$(echo "${CLUSTER_VERSION}" | cut -d '.' -f1,2) OCP_VERSION="${OCP_MAJOR_MINOR}" @@ -60,7 +81,7 @@ if [ "${MAP_TESTS}" = "true" ]; then ExitTrap--PostProcessPrep junit--odf__interop-tests__ocs-tests__interop-tests-ocs-tests.xml ' EXIT else - trap 'cleanup; _propagate_junit' EXIT + trap '_opp_cleanup; cleanup; _propagate_junit' EXIT fi # diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 46e61e1e64c23..1121bad984b07 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,5 +1,26 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" @@ -83,7 +104,7 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( GenerateJunit; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; GenerateJunit; _propagate_junit' EXIT function DiscoverQuay () { typeset registryJson="" discoverErr="" @@ -514,16 +535,21 @@ function RunAcsScan () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" + # Mask credentials: function call args expand in xtrace + set +x 2>/dev/null if ! RegisterQuayInAcs "${acsHost}" "${acsPassword}"; then + "${xtraceOn}" && set -x elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Failed to register Quay in ACS" "${elapsed}" return 1 fi if ! RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"; then + "${xtraceOn}" && set -x elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS scan request failed" "${elapsed}" return 1 fi + "${xtraceOn}" && set -x typeset -i attempts=0 maxAttempts=40 @@ -546,7 +572,9 @@ sys.exit(0 if len(images) > 0 else 1) fi if (( attempts % 4 == 3 )); then + set +x 2>/dev/null RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" || true + "${xtraceOn}" && set -x fi attempts=$((attempts + 1)) diff --git a/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh b/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh index a5ea8129d32f3..45d2cc82acef7 100644 --- a/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh +++ b/ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + # NOTE: BACKUP_TIMEOUT and OPP_OPERATORS are set via step config YAML (naming deviates from OPP__ convention) BACKUP_TIMEOUT="${BACKUP_TIMEOUT:-300}" OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" @@ -43,15 +64,15 @@ TimeoutMonitor() { # Start timeout monitor in background TimeoutMonitor & typeset timeoutPid=$! -trap 'kill ${timeoutPid} || true' EXIT +trap '_opp_cleanup; kill ${timeoutPid} || true' EXIT trap 'kill ${timeoutPid} || true; exit 124' TERM -: "=== Pre-Upgrade Cluster Backup ===" +echo ">>> PHASE: Pre-Upgrade Cluster Backup" : "Start time: $(date '+%F %T')" : "Backup timeout: ${BACKUP_TIMEOUT}s" # --- Etcd snapshot --- -: "--- Etcd Snapshot ---" +echo ">>> PHASE: Etcd Snapshot" typeset controlPlaneNode="" controlPlaneNode=$(oc get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{.items[0].metadata.name}') || true if [[ -n "${controlPlaneNode}" ]]; then @@ -104,7 +125,7 @@ else fi # --- Control plane resource state --- -: "--- Control Plane State ---" +echo ">>> PHASE: Control Plane State" Capture "ClusterVersion" "${backupDir}/clusterversion.yaml" \ oc get clusterversion version -o yaml @@ -118,7 +139,7 @@ Capture "MachineConfigPools" "${backupDir}/machineconfigpools.yaml" \ oc get machineconfigpools -o yaml # --- OPP operator state --- -: "--- OPP Operator State ---" +echo ">>> PHASE: OPP Operator State" Capture "CSVs" "${backupDir}/csvs.yaml" \ oc get csv -A -o yaml @@ -129,7 +150,7 @@ Capture "InstallPlans" "${backupDir}/installplans.yaml" \ oc get installplans -A -o yaml # --- Backup manifest --- -: "--- Generating Backup Manifest ---" +echo ">>> PHASE: Generating Backup Manifest" typeset clusterVersion="" clusterVersion=$(oc get clusterversion version -o jsonpath='{.status.desired.version}') || true @@ -160,7 +181,7 @@ EOF : "OK: backup-manifest.json" # --- Summary --- -: "=== Backup Summary ===" +echo ">>> PHASE: Backup Summary" : "End time: $(date '+%F %T')" : "Cluster version: ${clusterVersion:-unknown}" : "Node count: ${nodeCount}" diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh index 10f0f11e5e3cb..c5f8bbe33d808 100755 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-commands.sh @@ -1,5 +1,32 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: INTEROP-9455 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # ACM Observability + ODF Interop Validation (6-point gate) @@ -35,13 +62,16 @@ function AddResult () { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. function XmlEscape () { typeset text="${1:-}"; (($#)) && shift - text="${text//&/&}" - text="${text///>}" - text="${text//\"/"}" - text="${text//\'/'}" + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" printf '%s' "${text}" true } @@ -103,14 +133,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Check 1: ODF Ceph RGW infrastructure ready # --------------------------------------------------------------------------- function CheckRgwReady () { - : "=== Check 1: ODF Ceph RGW infrastructure ===" + echo ">>> PHASE: Check 1 — ODF Ceph RGW infrastructure" typeset rgwPhase="" typeset rgwJson="" rgwErr="" @@ -201,34 +231,93 @@ print(items[0].get('status',{}).get('phase','') if items else '') # --------------------------------------------------------------------------- function CheckMcoReady () { - : "=== Check 2: MultiClusterObservability CR ===" - - typeset mcoStatus="" - if ! mcoStatus="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ - --all-namespaces -o json | python3 -c " + echo ">>> PHASE: Check 2 — MultiClusterObservability CR readiness poll" + + typeset -i maxAttempts=24 + typeset -i sleepSeconds=30 + # Timeout budget: 24 x 30s poll = 720s max + ~180s margin within 900s step timeout + typeset -i attempt=0 + typeset -i startTime=0 + startTime=$(date +%s) + typeset _last_mco_error="" + + # Fast-path: if the CR doesn't exist at all, skip immediately. + # Capture exit status separately so RBAC/connectivity errors are not + # silently treated as "CR absent". + typeset _mco_probe="" _mco_probe_rc=0 + _mco_probe="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + observability --ignore-not-found -o name 2>&1)" || _mco_probe_rc=$? + if (( _mco_probe_rc != 0 )); then + echo "WARNING: oc get MCO CR failed (exit ${_mco_probe_rc}): ${_mco_probe}" + echo "Proceeding to poll loop (may be transient)" + elif [[ -z "${_mco_probe}" ]]; then + echo "MultiClusterObservability CR 'observability' not found" + echo "Observability may not be deployed — skipping MCO readiness check" + AddResult "mco-ready" "skip" "MCO CR not found — observability may not be deployed" + return 0 + fi + + while (( attempt < maxAttempts )); do + (( attempt += 1 )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - startTime )) + echo ">>> MCO poll ${attempt}/${maxAttempts} (${elapsed}s elapsed)…" + + typeset mcoStatus="" mcoConditions="" mcoError="" queryFailed="false" + if ! mcoConditions="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ + observability -o jsonpath='{.status.conditions}' 2>&1)"; then + mcoError="${mcoConditions}" + if [[ "${mcoError}" == *"(NotFound)"* && "${mcoError}" == *'"observability" not found'* ]]; then + AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" + return 0 + fi + queryFailed="true" + _last_mco_error="${mcoError}" + elif ! mcoStatus="$(printf '%s' "${mcoConditions}" | python3 -c " import sys,json -d=json.load(sys.stdin) -items=d.get('items',[]) -if not items: - print('NotFound') -else: - conds=items[0].get('status',{}).get('conditions',[]) - ready=[c for c in conds if c.get('type')=='Ready'] - print(ready[0].get('status','Unknown') if ready else 'NoCondition') +raw=sys.stdin.read().strip() +if not raw: + print('NoCondition') + sys.exit(0) +conds=json.loads(raw) +ready=[c for c in conds if c.get('type')=='Ready'] +print(ready[0].get('status','Unknown') if ready else 'NoCondition') ")"; then - AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR" - return - fi + queryFailed="true" + fi - if [[ "${mcoStatus}" == "True" ]]; then - : "PASS: MultiClusterObservability Ready=True" - AddResult "mco-ready" "pass" - elif [[ "${mcoStatus}" == "NotFound" ]]; then - AddResult "mco-ready" "skip" "MultiClusterObservability CR not found; observability not deployed" - else - AddResult "mco-ready" "fail" "MultiClusterObservability Ready=${mcoStatus} (expected True)" + if [[ "${queryFailed}" == "true" ]]; then + # Query or parsing failed — might be transient; retry unless last attempt + if (( attempt >= maxAttempts )); then + elapsed=$(( $(date +%s) - startTime )) + AddResult "mco-ready" "fail" "Failed to query MultiClusterObservability CR after ${elapsed}s" + return 1 + fi + sleep "${sleepSeconds}" + continue + fi + + if [[ "${mcoStatus}" == "True" ]]; then + elapsed=$(( $(date +%s) - startTime )) + : "PASS: MultiClusterObservability Ready=True after ${elapsed}s" + AddResult "mco-ready" "pass" "MultiClusterObservability is Ready after ${elapsed}s" + return 0 + fi + + # Not ready yet — sleep and retry + if (( attempt < maxAttempts )); then + sleep "${sleepSeconds}" + fi + done + + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - startTime )) + typeset _mco_detail="MultiClusterObservability not Ready after ${elapsed}s (last status=${mcoStatus:-unknown})" + if [[ -n "${_last_mco_error}" ]]; then + _mco_detail="${_mco_detail}; last error: ${_last_mco_error:0:200}" fi - true + AddResult "mco-ready" "fail" "${_mco_detail}" + return 1 } # --------------------------------------------------------------------------- @@ -236,7 +325,7 @@ else: # --------------------------------------------------------------------------- function CheckStorageEndpoint () { - : "=== Check 3: Object storage endpoint ===" + echo ">>> PHASE: Check 3 — Object storage endpoint" typeset storageConfig="" if ! storageConfig="$(oc get multiclusterobservabilities.observability.open-cluster-management.io \ @@ -337,7 +426,7 @@ print('odf-backed' if odf_pat.search(endpoint) else 'external') # --------------------------------------------------------------------------- function CheckThanosHealth () { - : "=== Check 4: Thanos components healthy ===" + echo ">>> PHASE: Check 4 — Thanos components healthy" if ! oc get namespace "${obsNamespace}" -o name; then AddResult "thanos-health" "skip" "Observability namespace ${obsNamespace} does not exist" @@ -419,7 +508,7 @@ function CheckThanosHealth () { # --------------------------------------------------------------------------- function CheckObcBound () { - : "=== Check 5: Observability ObjectBucketClaim ===" + echo ">>> PHASE: Check 5 — Observability ObjectBucketClaim" typeset obcList="" obcList="$(oc get obc -n "${obsNamespace}" -o json 2>/dev/null)" || true @@ -541,7 +630,7 @@ print('ok') } function CheckThanosQuery () { - : "=== Check 6: Thanos query functional ===" + echo ">>> PHASE: Check 6 — Thanos query functional" typeset routeJson="" routeJson="$(oc get routes -n "${obsNamespace}" -o json)" || true @@ -668,12 +757,38 @@ print(items[0]['metadata']['name'] if items else '') # Main # --------------------------------------------------------------------------- +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(XmlEscape "${bug_id}")" + safe_desc="$(XmlEscape "${bug_description}")" + safe_error="$(XmlEscape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + function Main () { if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "ACM Observability + ODF Interop Validation starting" + echo ">>> PHASE: ACM Observability + ODF Interop Validation starting" : "ACM namespace: ${acmNamespace}" : "Observability namespace: ${obsNamespace}" : "ODF namespace: ${odfNamespace}" @@ -686,22 +801,31 @@ function Main () { CheckObcBound || true CheckThanosQuery || true + typeset -i _idx=0 + for _idx in "${!tcResultsArr[@]}"; do + if [[ "${tcNamesArr[$_idx]}" == "thanos-query" \ + && "${tcResultsArr[$_idx]}" == "fail" \ + && "${tcMessagesArr[$_idx]}" == *"returned empty result vector"* ]]; then + # Known-issue skip: INTEROP-9455 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when INTEROP-9455 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${tcMessagesArr[$_idx]}" "INTEROP-9455" \ + "Thanos query returns empty result during observability convergence" + tcResultsArr[$_idx]="skip" + fi + done + WriteJunit - typeset -i hasAnyFail=0 typeset r="" for r in "${tcResultsArr[@]}"; do if [[ "${r}" == "fail" ]]; then - hasAnyFail=1 - break + : "ACM Observability + ODF Interop: SOME CHECKS FAILED" + exit 1 fi done - if (( hasAnyFail )); then - : "ACM Observability + ODF Interop: SOME CHECKS FAILED" - exit 1 - fi - : "ACM Observability + ODF Interop: ALL PASSED" exit 0 } diff --git a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml index 07e847330aa08..e8f25f5d1fe5f 100644 --- a/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml +++ b/ci-operator/step-registry/interop/opp/observability-odf/interop-opp-observability-odf-ref.yaml @@ -1,6 +1,5 @@ ref: as: interop-opp-observability-odf - best_effort: true commands: interop-opp-observability-odf-commands.sh documentation: |- Validates the cross-product integration surface between ACM Observability @@ -27,10 +26,12 @@ ref: - default: "openshift-storage" documentation: Namespace where ODF is installed name: ODF_NAMESPACE + # NOTE: best_effort was intentionally removed to surface step failures. + # Known-issue skips (INTEROP-9455) now handle expected failures explicitly. from: cli grace_period: 30s resources: requests: cpu: 100m memory: 200Mi - timeout: 10m + timeout: 15m diff --git a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh index c77500c40fae9..9a1bce2247881 100755 --- a/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh +++ b/ci-operator/step-registry/interop/opp/odf-health/interop-opp-odf-health-commands.sh @@ -1,5 +1,26 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # ODF Health Check (7-point gate) @@ -64,6 +85,9 @@ function AddResult () { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. function XmlEscape () { typeset text="${1:-}"; (($#)) && shift text="${text//&/&}" @@ -152,14 +176,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Check 1: ODF Operator CSV in Succeeded phase # --------------------------------------------------------------------------- function CheckOdfCsv () { - : "=== Check 1: ODF Operator CSV ===" + echo ">>> PHASE: Check 1 — ODF Operator CSV" typeset csvPhase="" if ! csvPhase="$(oc get csv -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -187,7 +211,7 @@ print((m[0].get('status',{}).get('phase','NotFound')) if m else 'NotFound') # --------------------------------------------------------------------------- function CheckStorageCluster () { - : "=== Check 2: StorageCluster Ready ===" + echo ">>> PHASE: Check 2 — StorageCluster Ready" typeset scPhase="" if ! scPhase="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -214,7 +238,7 @@ print(d['items'][0]['status'].get('phase','NotFound') if d.get('items') else 'No # --------------------------------------------------------------------------- function CheckCephCluster () { - : "=== Check 3: CephCluster health ===" + echo ">>> PHASE: Check 3 — CephCluster health" typeset cephHealth="" if ! cephHealth="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -241,7 +265,7 @@ print(d['items'][0].get('status',{}).get('ceph',{}).get('health','NotFound') if # --------------------------------------------------------------------------- function CheckStorageClasses () { - : "=== Check 4: StorageClasses ===" + echo ">>> PHASE: Check 4 — StorageClasses" typeset failMsg="" typeset scName="" @@ -270,7 +294,7 @@ function CheckStorageClasses () { # --------------------------------------------------------------------------- function CheckPvcProvision () { - : "=== Check 5: PVC provisioning (RBD + CephFS) ===" + echo ">>> PHASE: Check 5 — PVC provisioning" typeset -a scTests=("ocs-storagecluster-ceph-rbd" "ocs-storagecluster-cephfs") typeset -a scModes=("ReadWriteOnce" "ReadWriteMany") @@ -333,7 +357,7 @@ EOF # --------------------------------------------------------------------------- function CheckNoobaa () { - : "=== Check 6: NooBaa S3 functional ===" + echo ">>> PHASE: Check 6 — NooBaa S3 functional" typeset nbPhase="" if ! nbPhase="$(oc get noobaa -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -521,7 +545,7 @@ EOF # --------------------------------------------------------------------------- function CheckCephHealth () { - : "=== Check 7: Ceph health detail ===" + echo ">>> PHASE: Check 7 — Ceph health detail" typeset cephDetail="" if ! cephDetail="$(oc get cephcluster -n "${ODF_NAMESPACE}" -o json | python3 -c " @@ -563,7 +587,7 @@ function WaitForOdfReady () { typeset -i deadline=$(( SECONDS + timeout )) typeset -i pollInterval=15 - : "Waiting up to ${timeout}s for StorageCluster and NooBaa to reach Ready..." + echo ">>> PHASE: Waiting for ODF subsystems to reach Ready" typeset scPhase="" nbPhase="" while (( SECONDS < deadline )); do @@ -643,7 +667,7 @@ function Main () { export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "ODF Health Check (7-point gate) starting" + echo ">>> PHASE: ODF Health Check (7-point gate) starting" : "Namespace: ${ODF_NAMESPACE}" : "Artifacts dir: ${ARTIFACT_DIR}" @@ -666,12 +690,12 @@ function Main () { typeset scJson="" scCount="" set +x # suppress xtrace for API response if ! scJson="$(oc get storagecluster -n "${ODF_NAMESPACE}" -o json 2>/dev/null)"; then - set -x # restore xtrace + set -x : "Failed to query StorageClusters in ${ODF_NAMESPACE}" exit 1 fi if [[ -z "${scJson}" ]]; then - set -x # restore xtrace + set -x : "StorageCluster query returned empty output in ${ODF_NAMESPACE}" exit 1 fi @@ -681,11 +705,11 @@ items=d.get('items') if not isinstance(items,list): raise ValueError('StorageCluster items is not a list') print(len(items)) ")"; then - set -x # restore xtrace + set -x : "Failed to parse StorageCluster JSON from ${ODF_NAMESPACE}" exit 1 fi - set -x # restore xtrace + set -x if (( scCount == 0 )); then AddResult "odf-csv-phase" "pass" SkipAllChecks "ODF operator installed but no StorageCluster configured in ${ODF_NAMESPACE}" \ diff --git a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh index 4cc43f782c5a8..afc97064e3979 100755 --- a/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh +++ b/ci-operator/step-registry/interop/opp/preflight/interop-opp-preflight-commands.sh @@ -1,8 +1,29 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}" export HOME="${HOME:-/tmp/home}" @@ -45,7 +66,7 @@ function DebugOnExit () { true } -trap '{ EXIT_CODE=$?; DebugOnExit; true; }' EXIT +trap '_opp_cleanup; EXIT_CODE=${_exit_code}; DebugOnExit' EXIT trap '{ EXIT_CODE=143; DebugOnExit; trap - EXIT; exit 143; }' TERM # ────────────────────────────────────────────────────────────────────── @@ -101,7 +122,7 @@ with open(sys.argv[1], 'w') as f: } function CheckApiDeprecations () { - : "=== Check 1: API deprecation scan ===" + echo ">>> PHASE: Check 1 — API deprecation scan" typeset targetMinor="${1}" typeset ocpDisplay="${2:-4.${targetMinor}}" @@ -147,7 +168,13 @@ function CheckApiDeprecations () { } function CheckOppCompatibility () { - : "=== Check 2: OPP operator compatibility matrix ===" + echo ">>> PHASE: Check 2 — OPP operator compatibility matrix" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: OPP compatibility matrix check (IGNORE_SECONDARY_POLICIES=true)" + echo "opp_compatibility_matrix" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AppendCheck "opp_compatibility_matrix" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset ocpKey="${1}" typeset compatSpec="${OPP_COMPAT[${ocpKey}]:-}" @@ -219,7 +246,7 @@ function CheckOppCompatibility () { } function CheckClusterHealth () { - : "=== Check 3: Cluster health baseline ===" + echo ">>> PHASE: Check 3 — Cluster health baseline" typeset failed=0 details="" @@ -314,7 +341,7 @@ print('\n'.join(names)) } function CheckMcpReadiness () { - : "=== Check 4: MachineConfigPool readiness ===" + echo ">>> PHASE: Check 4 — MachineConfigPool readiness" typeset failed=0 details="" @@ -404,7 +431,7 @@ function Main () { sourceVersion="$(oc get clusterversion --no-headers | awk '{print $2}')" : "Source OCP version: ${sourceVersion}" - : "=== Starting OPP pre-flight validation ===" + echo ">>> PHASE: Starting OPP pre-flight validation" InitReport diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh index f1f28f1c71cef..b2a98feed755b 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acm/interop-opp-product-upgrade-acm-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: ACM-45920 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ACM_TARGET_CHANNEL="${ACM_TARGET_CHANNEL:-}" ACM_UPGRADE_TIMEOUT="${ACM_UPGRADE_TIMEOUT:-30m}" ACM_SUBSCRIPTION_NAME="${ACM_SUBSCRIPTION_NAME:-advanced-cluster-management}" @@ -29,7 +56,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ACM_SUBSCRIPTION_NAME}" \ @@ -310,6 +337,45 @@ function ValidateHubHealth () { return 0 } +# _xml_escape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. +_xml_escape() { + local text="$1" + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" + printf '%s' "${text}" +} + +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(_xml_escape "${bug_id}")" + safe_desc="$(_xml_escape "${bug_description}")" + safe_error="$(_xml_escape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + # === Main === function Main () { @@ -404,8 +470,38 @@ function Main () { newVersion="$(GetInstalledVersion)" echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})" - ValidateMceUpgrade - ValidateHubHealth + typeset _acm_upgrade_output="" + if ! _acm_upgrade_output="$(ValidateMceUpgrade 2>&1)"; then + echo "${_acm_upgrade_output}" + if echo "${_acm_upgrade_output}" | grep -q "cannot unmarshal string into Go struct"; then + # Known-issue skip: ACM-45920 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when ACM-45920 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acm_upgrade_output}" "ACM-45920" \ + "YAML unmarshal error on OCP 5.0 — ACM team fix in progress" + else + exit 1 + fi + else + echo "${_acm_upgrade_output}" + fi + + if ! _acm_upgrade_output="$(ValidateHubHealth 2>&1)"; then + echo "${_acm_upgrade_output}" + if echo "${_acm_upgrade_output}" | grep -q "cannot unmarshal string into Go struct"; then + # Known-issue skip: ACM-45920 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when ACM-45920 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acm_upgrade_output}" "ACM-45920" \ + "YAML unmarshal error on OCP 5.0 — ACM team fix in progress" + else + exit 1 + fi + else + echo "${_acm_upgrade_output}" + fi { printf '=== ACM Operator Upgrade Summary ===\n' diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh index f953e5d4a610c..de85a1b391c4a 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/acs/interop-opp-product-upgrade-acs-commands.sh @@ -1,7 +1,34 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# === Known-Issue Skip Framework === +# This script uses _detect_known_issue() to emit JUnit SKIPPED results +# for tracked bugs instead of failing the job. Unknown failures still FAIL. +# Tracked issues: INTEROP-9466 +# See PR review Fix 3 for rationale. + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ACS_TARGET_CHANNEL="${ACS_TARGET_CHANNEL:-}" ACS_UPGRADE_TIMEOUT="${ACS_UPGRADE_TIMEOUT:-30m}" ACS_SUBSCRIPTION_NAME="${ACS_SUBSCRIPTION_NAME:-rhacs-operator}" @@ -30,7 +57,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ACS_SUBSCRIPTION_NAME}" \ @@ -287,6 +314,45 @@ function ValidateAcsHealth () { return 0 } +# _xml_escape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. +_xml_escape() { + local text="$1" + text="${text//&/\&}" + text="${text///\>}" + text="${text//\"/\"}" + text="${text//\'/\'}" + printf '%s' "${text}" +} + +# JUnit fragment contract: ci-operator's junit_report.go accepts both +# standalone fragments and full -wrapped documents. +# Fragments are appended to junit_known_issues.xml and consumed correctly. +_detect_known_issue() { + local error_output="$1" + local bug_id="$2" + local bug_description="$3" + local safe_bug_id safe_desc safe_error + + safe_bug_id="$(_xml_escape "${bug_id}")" + safe_desc="$(_xml_escape "${bug_description}")" + safe_error="$(_xml_escape "${error_output:0:500}")" + + echo ">>> KNOWN ISSUE: ${bug_id} — ${bug_description}" + echo ">>> Marking as SKIPPED (tracked: https://issues.redhat.com/browse/${bug_id})" + + cat <> "${ARTIFACT_DIR}/junit_known_issues.xml" + + + Tracked at https://issues.redhat.com/browse/${safe_bug_id} + Error: ${safe_error} + + +JUNIT_EOF +} + # === Main === function Main () { @@ -381,7 +447,22 @@ function Main () { newVersion="$(GetInstalledVersion)" echo "Upgrade complete: ${currentVersion} -> ${newVersion} (CSV: ${newCsv})" - ValidateAcsHealth + typeset _acs_upgrade_output="" + if ! _acs_upgrade_output="$(ValidateAcsHealth 2>&1)"; then + echo "${_acs_upgrade_output}" + if echo "${_acs_upgrade_output}" | grep -q "SecuredCluster did not reach Deployed"; then + # Known-issue skip: INTEROP-9466 + # Added: 2026-09-21 + # Review-by: 2026-12-21 (or when INTEROP-9466 is resolved) + # Owner: OPP-interop team + _detect_known_issue "${_acs_upgrade_output}" "INTEROP-9466" \ + "SecuredCluster reconciliation delay after ACS upgrade" + else + exit 1 + fi + else + echo "${_acs_upgrade_output}" + fi { printf '=== ACS Operator Upgrade Summary ===\n' diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh index a2fc00ad31251..e163f794700c2 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/odf/interop-opp-product-upgrade-odf-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + ODF_TARGET_CHANNEL="${ODF_TARGET_CHANNEL:-}" ODF_UPGRADE_TIMEOUT="${ODF_UPGRADE_TIMEOUT:-45m}" ODF_SUBSCRIPTION_NAME="${ODF_SUBSCRIPTION_NAME:-odf-operator}" @@ -30,7 +51,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${ODF_SUBSCRIPTION_NAME}" \ diff --git a/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh b/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh index 4ad5329d1c011..a308ddb493da0 100755 --- a/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh +++ b/ci-operator/step-registry/interop/opp/product-upgrade/quay/interop-opp-product-upgrade-quay-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + QUAY_TARGET_CHANNEL="${QUAY_TARGET_CHANNEL:-}" QUAY_UPGRADE_TIMEOUT="${QUAY_UPGRADE_TIMEOUT:-30m}" QUAY_SUBSCRIPTION_NAME="${QUAY_SUBSCRIPTION_NAME:-quay-operator}" @@ -30,7 +51,7 @@ function CollectDiagnostics () { true } -trap 'if (( $? != 0 )); then CollectDiagnostics; fi' EXIT +trap '_opp_cleanup; if (( _exit_code != 0 )); then CollectDiagnostics; fi' EXIT function GetCurrentCsv () { oc get subscription "${QUAY_SUBSCRIPTION_NAME}" \ diff --git a/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh b/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh index f18ded33d9fd3..1e406137330c7 100644 --- a/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh +++ b/ci-operator/step-registry/interop/opp/smoke/interop-opp-smoke-commands.sh @@ -1,5 +1,26 @@ #!/bin/bash -set -euxo pipefail; shopt -s inherit_errexit +set -euo pipefail; shopt -s inherit_errexit + +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" # --------------------------------------------------------------------------- # OPP post-upgrade smoke tests @@ -34,6 +55,9 @@ AddResult() { true } +# XmlEscape: Required for bash 5.x where patsub_replacement is enabled +# by default, changing how ${var//pattern/replacement} handles & and \ in +# the replacement string. Without escaping, JUnit XML output is malformed. XmlEscape() { typeset text="${1:-}"; (($#)) && shift text="${text//&/&}" @@ -93,14 +117,14 @@ _propagate_junit () { find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true } -trap '{( CollectExitArtifacts; _propagate_junit; true )}' EXIT +trap '_opp_cleanup; CollectExitArtifacts; _propagate_junit' EXIT # --------------------------------------------------------------------------- # Test 1: cluster-health # --------------------------------------------------------------------------- TestClusterHealth() { - : "=== Test: cluster-health ===" + echo ">>> PHASE: Test — cluster-health" typeset failMsg="" # ClusterOperators: Available=True, Degraded!=True @@ -183,7 +207,13 @@ TestClusterHealth() { # --------------------------------------------------------------------------- TestOppOperators() { - : "=== Test: opp-operators ===" + echo ">>> PHASE: Test — opp-operators" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: opp-operators check (IGNORE_SECONDARY_POLICIES=true)" + echo "opp-operators" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "opp-operators" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" typeset -a operatorsArr=() @@ -277,7 +307,13 @@ TestOppOperators() { # --------------------------------------------------------------------------- TestAcmConnectivity() { - : "=== Test: acm-connectivity ===" + echo ">>> PHASE: Test — acm-connectivity" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: acm-connectivity check (IGNORE_SECONDARY_POLICIES=true)" + echo "acm-connectivity" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "acm-connectivity" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Check if ManagedCluster resources exist @@ -324,7 +360,13 @@ TestAcmConnectivity() { # --------------------------------------------------------------------------- TestAcsSensors() { - : "=== Test: acs-sensors ===" + echo ">>> PHASE: Test — acs-sensors" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: acs-sensors check (IGNORE_SECONDARY_POLICIES=true)" + echo "acs-sensors" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "acs-sensors" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Check SecuredCluster CR status first @@ -394,7 +436,13 @@ TestAcsSensors() { # --------------------------------------------------------------------------- TestQuayPull() { - : "=== Test: quay-pull ===" + echo ">>> PHASE: Test — quay-pull" + if [[ "${IGNORE_SECONDARY_POLICIES:-false}" == "true" ]]; then + echo "SKIP: quay-pull check (IGNORE_SECONDARY_POLICIES=true)" + echo "quay-pull" >> "${ARTIFACT_DIR}/skipped-policies.json" 2>/dev/null || true + AddResult "quay-pull" "skip" "Skipped (IGNORE_SECONDARY_POLICIES=true)" + return 0 + fi typeset failMsg="" # Find the Quay registry route @@ -471,7 +519,7 @@ Main() { export KUBECONFIG="${SHARED_DIR}/kubeconfig" fi - : "OPP Smoke Tests starting" + echo ">>> PHASE: OPP Smoke Tests starting" : "Operators: ${OPP_OPERATORS}" : "Settle window: ${SMOKE_SETTLE_SECONDS}s" : "Artifacts dir: ${ARTIFACT_DIR}" diff --git a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh index 79d8ae4f7d5c8..af756e5ff73a4 100644 --- a/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh +++ b/ci-operator/step-registry/interop/opp/upgrade/interop-opp-upgrade-commands.sh @@ -1,7 +1,28 @@ #!/bin/bash -set -eux -o pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + # NOTE: UPGRADE_TIMEOUT, POLL_INTERVAL, STALL_WINDOW, OPP_OPERATORS are set via step config YAML # (naming deviates from OPP__ convention) UPGRADE_TIMEOUT="${UPGRADE_TIMEOUT:-130}" @@ -67,7 +88,7 @@ function DebugOnExit () { true } -trap '{ exitCode=$?; DebugOnExit; true; }' EXIT +trap '_opp_cleanup; exitCode=${_exit_code}; DebugOnExit' EXIT trap '{ exitCode=143; DebugOnExit; trap - EXIT; exit 143; }' TERM set +x @@ -217,6 +238,7 @@ function UpdateCcoAnnotation () { function InitiateUpgrade () { typeset isForce="${1:-}"; (($#)) && shift + echo ">>> PHASE: Initiating upgrade" : "Initiating upgrade to ${upgradeTarget}" : "Force flag: ${isForce}" oc adm upgrade --to-image="${upgradeTarget}" --allow-explicit-upgrade --force="${isForce}" @@ -234,6 +256,7 @@ function InitiateUpgrade () { } function MonitorUpgrade () { + echo ">>> PHASE: Monitoring upgrade" typeset -i pollCount=0 typeset -i lastProgressChange=0 lastProgressChange=$(date +%s) @@ -299,6 +322,7 @@ function MonitorUpgrade () { } function StabilizeCluster () { + echo ">>> PHASE: Stabilizing cluster" : "Waiting for cluster stability (minimum-stable-period=5m, timeout=30m)" if ! oc adm wait-for-stable-cluster --minimum-stable-period=5m --timeout=30m; then : "Cluster stabilization failed; gathering diagnostics" @@ -312,6 +336,7 @@ function StabilizeCluster () { } function ValidatePlatformHealth () { + echo ">>> PHASE: Validating platform health" : "Validating platform health" typeset avail="" progressing="" degraded="" @@ -351,6 +376,7 @@ function ValidatePlatformHealth () { } function ValidateOppOperators () { + echo ">>> PHASE: Validating OPP operators" : "Validating OPP operator health" typeset -a operatorsArr=() IFS=',' read -ra operatorsArr <<< "${OPP_OPERATORS}" @@ -430,6 +456,8 @@ function Main () { sourceMinorVersion="$(echo "${sourceVersion}" | cut -f2 -d.)" : "Source release: ${sourceVersion} (minor: ${sourceMinorVersion})" + echo ">>> PHASE: OCP upgrade starting" + isForceUpdate="false" if ! CheckSigned "${upgradeTarget}"; then : "Target is unsigned; will use --force" diff --git a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh index 05c434fa3fb11..fd9aaa9cced81 100755 --- a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh +++ b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-commands.sh @@ -1,8 +1,29 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail shopt -s inherit_errexit +# --- Trace-to-file: always capture, dump on failure only --- +_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log" +exec {_xtrace_fd}>"${_xtrace_log}" +BASH_XTRACEFD=${_xtrace_fd} +set -x + +# shellcheck disable=SC2154 +_opp_cleanup() { + _exit_code=$? + set +x 2>/dev/null + # Scrub credentials before copying + sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true + if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then + cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true + echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})" + fi +} +trap '_opp_cleanup' EXIT + +echo ">>> PHASE: initialization" + typeset -ri mcpWaitTimeout="${MCP_WAIT_TIMEOUT:-3600}" typeset -ri consecutiveRequired="${MCP_CONSECUTIVE_CHECKS:-3}" typeset -ri settleDelay="${MCP_SETTLE_DELAY:-90}" diff --git a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml index aea0e6302035d..d1d69934db280 100644 --- a/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml +++ b/ci-operator/step-registry/interop/opp/wait-mcp/interop-opp-wait-mcp-ref.yaml @@ -39,6 +39,7 @@ ref: documentation: Max seconds to wait for all non-paused MCPs to stabilize (default 60 min) name: MCP_WAIT_TIMEOUT from: cli + grace_period: 1m resources: requests: cpu: 100m