INTEROP-9511: Signal-integrity improvements for OPP interop steps

ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-commands.sh

@@ -1,8 +1,27 @@
 #!/bin/bash
 
-set -o nounset
-set -o errexit
-set -o pipefail
+set -euo pipefail
+
+# --- Trace-to-file: always capture, dump on failure only ---
+_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log"
+exec {_xtrace_fd}>"${_xtrace_log}"
+BASH_XTRACEFD=${_xtrace_fd}
+set -x
+
+# shellcheck disable=SC2154
+_opp_cleanup() {
+  _exit_code=$?
+  set +x 2>/dev/null
+  # Scrub credentials before copying
+  sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true
+  if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then
+    cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+    echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})"
+  fi
+}
+trap '_opp_cleanup' EXIT
+
+echo ">>> PHASE: initialization"
 
 ODF_INSTALL_NAMESPACE=openshift-storage
 DEFAULT_ODF_OPERATOR_CHANNEL="stable-${ODF_VERSION_MAJOR_MINOR}"

ci-operator/step-registry/interop-tests/deploy-odf/interop-tests-deploy-odf-ref.yaml

@@ -6,6 +6,7 @@ ref:
     tag: latest
   commands: interop-tests-deploy-odf-commands.sh
   timeout: 3h0m0s
+  grace_period: 10m
   resources:
     requests:
       cpu: 100m
@@ -31,4 +32,4 @@ ref:
     default: "ocs-storagecluster"
   - name: ODF_VOLUME_SIZE
     documentation: The size of the ODF volume in Gi
-    default: "50"
+    default: "50"

ci-operator/step-registry/interop-tests/ocs-tests/interop-tests-ocs-tests-commands.sh

@@ -1,8 +1,29 @@
 #!/bin/bash
 
-set -euxo pipefail
+set -euo pipefail
 shopt -s inherit_errexit
 
+# --- Trace-to-file: always capture, dump on failure only ---
+_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log"
+exec {_xtrace_fd}>"${_xtrace_log}"
+BASH_XTRACEFD=${_xtrace_fd}
+set -x
+
+# shellcheck disable=SC2154
+_opp_cleanup() {
+  _exit_code=$?
+  set +x 2>/dev/null
+  # Scrub credentials before copying
+  sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true
+  if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then
+    cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+    echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})"
+  fi
+}
+trap '_opp_cleanup' EXIT
+
+echo ">>> PHASE: initialization"
+
 CLUSTER_VERSION=$(oc get clusterVersion version -o jsonpath='{$.status.desired.version}')
 OCP_MAJOR_MINOR=$(echo "${CLUSTER_VERSION}" | cut -d '.' -f1,2)
 OCP_VERSION="${OCP_MAJOR_MINOR}"
@@ -60,7 +81,7 @@ if [ "${MAP_TESTS}" = "true" ]; then
             ExitTrap--PostProcessPrep junit--odf__interop-tests__ocs-tests__interop-tests-ocs-tests.xml
     ' EXIT
 else
-    trap 'cleanup; _propagate_junit' EXIT
+    trap '_opp_cleanup; cleanup; _propagate_junit' EXIT
 fi
 
 #

ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh

@@ -1,5 +1,26 @@
 #!/bin/bash
-set -euxo pipefail; shopt -s inherit_errexit
+set -euo pipefail; shopt -s inherit_errexit
+
+# --- Trace-to-file: always capture, dump on failure only ---
+_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log"
+exec {_xtrace_fd}>"${_xtrace_log}"
+BASH_XTRACEFD=${_xtrace_fd}
+set -x
+
+# shellcheck disable=SC2154
+_opp_cleanup() {
+  _exit_code=$?
+  set +x 2>/dev/null
+  # Scrub credentials before copying
+  sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true
+  if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then
+    cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+    echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})"
+  fi
+}
+trap '_opp_cleanup' EXIT
+
+echo ">>> PHASE: initialization"
 
 ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}"
 mkdir -p "${ARTIFACT_DIR}"
@@ -83,7 +104,7 @@ _propagate_junit () {
     find "${ARTIFACT_DIR}" -name '*.xml' -exec cp {} "${SHARED_DIR}/junit/" \; 2>/dev/null || true
 }
 
-trap '{( GenerateJunit; _propagate_junit; true )}' EXIT
+trap '_opp_cleanup; GenerateJunit; _propagate_junit' EXIT
 
 function DiscoverQuay () {
     typeset registryJson="" discoverErr=""
@@ -514,16 +535,21 @@ function RunAcsScan () {
 
     typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}"
 
+    # Mask credentials: function call args expand in xtrace
+    set +x 2>/dev/null
     if ! RegisterQuayInAcs "${acsHost}" "${acsPassword}"; then
+        "${xtraceOn}" && set -x
         elapsed=$(( $(date +%s) - start ))
         RecordResult "${testName}" "failed" "Failed to register Quay in ACS" "${elapsed}"
         return 1
     fi
     if ! RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"; then
+        "${xtraceOn}" && set -x
         elapsed=$(( $(date +%s) - start ))
         RecordResult "${testName}" "failed" "ACS scan request failed" "${elapsed}"
         return 1
     fi
+    "${xtraceOn}" && set -x
 
     typeset -i attempts=0 maxAttempts=40
 
@@ -546,7 +572,9 @@ sys.exit(0 if len(images) > 0 else 1)
         fi
 
         if (( attempts % 4 == 3 )); then
+            set +x 2>/dev/null
             RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" || true
+            "${xtraceOn}" && set -x
         fi
 
         attempts=$((attempts + 1))

ci-operator/step-registry/interop/opp/backup/interop-opp-backup-commands.sh

@@ -1,7 +1,28 @@
 #!/bin/bash
-set -euxo pipefail
+set -euo pipefail
 shopt -s inherit_errexit
 
+# --- Trace-to-file: always capture, dump on failure only ---
+_xtrace_log="/tmp/xtrace-$(basename "$0" .sh).log"
+exec {_xtrace_fd}>"${_xtrace_log}"
+BASH_XTRACEFD=${_xtrace_fd}
+set -x
+
+# shellcheck disable=SC2154
+_opp_cleanup() {
+  _exit_code=$?
+  set +x 2>/dev/null
+  # Scrub credentials before copying
+  sed -i -E 's/(password|token|secret|key|credential)=[^ ]*/\1=REDACTED/gi' "${_xtrace_log}" 2>/dev/null || true
+  if [[ ${_exit_code} -ne 0 && -n "${ARTIFACT_DIR:-}" ]]; then
+    cp "${_xtrace_log}" "${ARTIFACT_DIR}/" 2>/dev/null || true
+    echo ">>> TRACE: xtrace log saved to artifacts (exit code ${_exit_code})"
+  fi
+}
+trap '_opp_cleanup' EXIT
+
+echo ">>> PHASE: initialization"
+
 # NOTE: BACKUP_TIMEOUT and OPP_OPERATORS are set via step config YAML (naming deviates from OPP__ convention)
 BACKUP_TIMEOUT="${BACKUP_TIMEOUT:-300}"
 OPP_OPERATORS="${OPP_OPERATORS:-advanced-cluster-management,rhacs-operator,odf-operator,quay-operator}"
@@ -43,15 +64,15 @@ TimeoutMonitor() {
 # Start timeout monitor in background
 TimeoutMonitor &
 typeset timeoutPid=$!
-trap 'kill ${timeoutPid} || true' EXIT
+trap '_opp_cleanup; kill ${timeoutPid} || true' EXIT
 trap 'kill ${timeoutPid} || true; exit 124' TERM
 
-: "=== Pre-Upgrade Cluster Backup ==="
+echo ">>> PHASE: Pre-Upgrade Cluster Backup"
 : "Start time: $(date '+%F %T')"
 : "Backup timeout: ${BACKUP_TIMEOUT}s"
 
 # --- Etcd snapshot ---
-: "--- Etcd Snapshot ---"
+echo ">>> PHASE: Etcd Snapshot"
 typeset controlPlaneNode=""
 controlPlaneNode=$(oc get nodes -l node-role.kubernetes.io/master="" -o jsonpath='{.items[0].metadata.name}') || true
 if [[ -n "${controlPlaneNode}" ]]; then
@@ -104,7 +125,7 @@ else
 fi
 
 # --- Control plane resource state ---
-: "--- Control Plane State ---"
+echo ">>> PHASE: Control Plane State"
 Capture "ClusterVersion" "${backupDir}/clusterversion.yaml" \
     oc get clusterversion version -o yaml
 
@@ -118,7 +139,7 @@ Capture "MachineConfigPools" "${backupDir}/machineconfigpools.yaml" \
     oc get machineconfigpools -o yaml
 
 # --- OPP operator state ---
-: "--- OPP Operator State ---"
+echo ">>> PHASE: OPP Operator State"
 Capture "CSVs" "${backupDir}/csvs.yaml" \
     oc get csv -A -o yaml
 
@@ -129,7 +150,7 @@ Capture "InstallPlans" "${backupDir}/installplans.yaml" \
     oc get installplans -A -o yaml
 
 # --- Backup manifest ---
-: "--- Generating Backup Manifest ---"
+echo ">>> PHASE: Generating Backup Manifest"
 typeset clusterVersion=""
 clusterVersion=$(oc get clusterversion version -o jsonpath='{.status.desired.version}') || true
 
@@ -160,7 +181,7 @@ EOF
 : "OK: backup-manifest.json"
 
 # --- Summary ---
-: "=== Backup Summary ==="
+echo ">>> PHASE: Backup Summary"
 : "End time: $(date '+%F %T')"
 : "Cluster version: ${clusterVersion:-unknown}"
 : "Node count: ${nodeCount}"