From c3a9caf41a6f52c649be751a641e01538be9feca Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 10:55:44 -0700 Subject: [PATCH 1/6] Add script to automate running baseline and testing versions of a suite It takes refs for compass and MALI for the baseline and testing versions, It then sets up compass and MALI for baseline and testing. Then submits jobs for both, with the testing one being dependent on the first. It was written with MALI in mind, but should work for MPAS-Ocean as well. --- ...ompass_integration_baseline_and_testing.sh | 334 ++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100755 utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh new file mode 100755 index 0000000000..dc38e0904a --- /dev/null +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -0,0 +1,334 @@ +#!/bin/bash +# +# run_compass_integration.sh +# +# Automates a baseline + testing compass "full_integration" landice suite +# comparison on Perlmutter (pm-cpu). +# +# ALL configuration (paths, refs, remotes, account, qos, etc.) is set by +# editing the variables in the CONFIG section below. The only supported +# command-line arguments are --force-baseline / --force-testing / --force, +# used to force re-cloning/re-building a stage that was already set up by a +# previous run of this script. +# +# Workflow (one command, no manual steps): +# 1. BASELINE prep (synchronous, on the login node): +# - clone compass from COMPASS_REMOTE_BASELINE @ COMPASS_REF_BASELINE +# (skipped if already present, unless --force/--force-baseline) +# - ./deploy.py --with-albany ... (skipped if load script exists) +# - source the generated load script +# - clone MALI-Dev from MALI_REMOTE_BASELINE @ MALI_REF_BASELINE +# - compile MALI (mpas-albany-landice) +# - `compass suite -c landice -t full_integration -w -s` +# 2. Submit the compass-generated suite job script for BASELINE via sbatch. +# 3. TESTING prep (synchronous, on the login node): +# - reuse the BASELINE compass checkout if COMPASS_REMOTE_TESTING/ +# COMPASS_REF_TESTING are left blank; otherwise clone/deploy a +# separate compass copy for testing +# - clone/build a *separate* MALI-Dev copy from MALI_REMOTE_TESTING @ +# MALI_REF_TESTING +# - `compass suite -c landice -t full_integration -b \ +# -w -s` +# 4. Submit the compass-generated suite job script for TESTING via sbatch +# with `--dependency=afterok:` so it automatically +# waits for the baseline run to finish successfully before starting -- +# no manual waiting required. +# +# Hardcoded directory layout under ROOT_WORK_DIR: +# /compass-baseline/ compass checkout used for baseline +# /compass-testing/ compass checkout used for testing (only +# created if a separate compass ref/remote +# is requested for testing) +# /MALI-baseline/ MALI-Dev checkout/build for baseline +# /MALI-testing/ MALI-Dev checkout/build for testing +# /suite-baseline/ compass suite -w for baseline +# /suite-testing/ compass suite -w for testing +# +set -euo pipefail + +# ============================================================================= +# CONFIG -- edit these values for your run. Nothing here is a CLI argument. +# ============================================================================= + +# Single root location; everything else lives in a hardcoded structure below it. +ROOT_WORK_DIR="/pscratch/sd/h/hoffman2/COMPASS/run" + +# SLURM settings for the compass suite *run* job (the actual integration test +# execution). The suite itself typically completes in ~12 minutes. +ACCOUNT="m4274" +QOS="debug" +RUN_WALLTIME="00:30:00" + +# compass deploy.py / build settings (shared by baseline and testing). +MACHINE="pm-cpu" +COMPILER="gnu" +MPI="mpich" + +# compass suite selection. +SUITE_CORE="landice" +SUITE_NAME="full_integration" + +# --- compass source ---------------------------------------------------------- +COMPASS_REMOTE_BASELINE="git@github.com:MPAS-Dev/compass.git" +COMPASS_REF_BASELINE="main" + +# Leave both of these blank to reuse the baseline compass checkout for testing +# (recommended when you are only testing a MALI change). Set both to test a +# different compass remote/ref as well (a separate compass-testing/ checkout +# will be created). +COMPASS_REMOTE_TESTING="" +COMPASS_REF_TESTING="" + +# --- MALI-Dev source ---------------------------------------------------------- +MALI_REMOTE_BASELINE="git@github.com:MALI-Dev/E3SM.git" +MALI_REF_BASELINE="develop" + +MALI_REMOTE_TESTING="git@github.com:MALI-Dev/E3SM.git" +MALI_REF_TESTING="develop" + +# ============================================================================= +# End of CONFIG. You should not need to edit anything below this line. +# ============================================================================= + +FORCE_BASELINE=false +FORCE_TESTING=false + +usage() { + cat < +clone_or_checkout() { + local dir="$1" remote_url="$2" ref="$3" force="$4" + + if [[ -d "${dir}/.git" && "${force}" == "false" ]]; then + log "Repo already present at ${dir}, skipping clone (use --force to redo)." + return 0 + fi + + if [[ -d "${dir}" ]]; then + log "Removing existing ${dir} (force redo)." + rm -rf "${dir}" + fi + + log "Cloning ${remote_url} into ${dir}" + git clone "${remote_url}" "${dir}" + + ( + cd "${dir}" + local current_branch + current_branch=$(git rev-parse --abbrev-ref HEAD) + if [[ -n "${ref}" && "${ref}" != "${current_branch}" ]]; then + log "Checking out ${ref} in ${dir}" + git fetch origin + git checkout "${ref}" + fi + ) +} + +# deploy_compass_env -> echoes path to load script on success +deploy_compass_env() { + local compass_dir="$1" force="$2" + local existing + existing=$(find "${compass_dir}" -maxdepth 1 -name "load_compass_${MACHINE}_${COMPILER}_${MPI}.sh" 2>/dev/null | head -n1 || true) + + if [[ -n "${existing}" && "${force}" == "false" ]]; then + log "Compass env already deployed in ${compass_dir}, reusing ${existing}." + echo "${existing}" + return 0 + fi + + log "Running deploy.py in ${compass_dir} (machine=${MACHINE} compiler=${COMPILER} mpi=${MPI})" + ( + cd "${compass_dir}" + ./deploy.py --with-albany --compiler "${COMPILER}" --mpi "${MPI}" --machine "${MACHINE}" + ) + + existing=$(find "${compass_dir}" -maxdepth 1 -name "load_compass_${MACHINE}_${COMPILER}_${MPI}.sh" 2>/dev/null | head -n1 || true) + if [[ -z "${existing}" ]]; then + log "ERROR: could not find generated load_compass_*.sh in ${compass_dir}" + exit 1 + fi + echo "${existing}" +} + +# compile_mali +compile_mali() { + local mali_dir="$1" force="$2" + local landice_dir="${mali_dir}/components/mpas-albany-landice" + local binary="${landice_dir}/landice_model" + + if [[ -f "${binary}" && "${force}" == "false" ]]; then + log "MALI already built at ${binary}, skipping (use --force to redo)." + return 0 + fi + + log "Compiling MALI in ${landice_dir}" + ( + cd "${landice_dir}" + make -j 4 gnu-cray ALBANY=true DEBUG=true + ) +} + +# setup_suite +# Returns (echoes) the path to the compass-generated suite job script. +setup_suite() { + local compass_dir="$1" work_dir="$2" baseline_dir="$3" + mkdir -p "${work_dir}" + + log "Setting up ${SUITE_CORE}/${SUITE_NAME} suite in ${work_dir}" + ( + cd "${compass_dir}" + if [[ -n "${baseline_dir}" ]]; then + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -b "${baseline_dir}" -w "${work_dir}" -s + else + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -w "${work_dir}" -s + fi + ) + + local job_script + job_script=$(find "${work_dir}" -maxdepth 1 -name "job_script.*.sh" 2>/dev/null | head -n1 || true) + if [[ -z "${job_script}" ]]; then + log "ERROR: could not find compass-generated job_script.*.sh in ${work_dir}" + exit 1 + fi + echo "${job_script}" +} + +# patch_and_submit -> echoes new jobid +patch_and_submit() { + local job_script="$1" dep_jobid="$2" + + # Patch account / qos / walltime directives defensively, whatever compass put there. + sed -i -E \ + -e "s/^#SBATCH[[:space:]]+(-A|--account=?)[[:space:]]*.*/#SBATCH -A ${ACCOUNT}/" \ + -e "s/^#SBATCH[[:space:]]+(--qos=?)[[:space:]]*.*/#SBATCH --qos=${QOS}/" \ + -e "s/^#SBATCH[[:space:]]+(-t|--time=?)[[:space:]]*.*/#SBATCH --time=${RUN_WALLTIME}/" \ + "${job_script}" + + # Add directives if they weren't present at all. + grep -q '^#SBATCH -A' "${job_script}" || sed -i "1a #SBATCH -A ${ACCOUNT}" "${job_script}" + grep -q '^#SBATCH --qos' "${job_script}" || sed -i "1a #SBATCH --qos=${QOS}" "${job_script}" + grep -q '^#SBATCH --time' "${job_script}" || sed -i "1a #SBATCH --time=${RUN_WALLTIME}" "${job_script}" + + local dep_args=() + if [[ -n "${dep_jobid}" ]]; then + dep_args=(--dependency="afterok:${dep_jobid}") + log "Submitting ${job_script} with dependency afterok:${dep_jobid}" + else + log "Submitting ${job_script}" + fi + + local jobid + jobid=$(sbatch --parsable "${dep_args[@]}" "${job_script}") + echo "${jobid}" +} + +# --------------------------------------------------------------------------- +# BASELINE +# --------------------------------------------------------------------------- +log "=== BASELINE: compass checkout ===" +clone_or_checkout "${BASELINE_COMPASS_DIR}" "${COMPASS_REMOTE_BASELINE}" \ + "${COMPASS_REF_BASELINE}" "${FORCE_BASELINE}" + +log "=== BASELINE: deploy compass env ===" +BASELINE_LOAD_SCRIPT=$(deploy_compass_env "${BASELINE_COMPASS_DIR}" "${FORCE_BASELINE}") +log "Sourcing ${BASELINE_LOAD_SCRIPT}" +# shellcheck disable=SC1090 +source "${BASELINE_LOAD_SCRIPT}" + +log "=== BASELINE: MALI-Dev checkout ===" +clone_or_checkout "${BASELINE_MALI_DIR}" "${MALI_REMOTE_BASELINE}" \ + "${MALI_REF_BASELINE}" "${FORCE_BASELINE}" + +log "=== BASELINE: compile MALI ===" +compile_mali "${BASELINE_MALI_DIR}" "${FORCE_BASELINE}" + +log "=== BASELINE: compass suite setup ===" +BASELINE_JOB_SCRIPT=$(setup_suite "${BASELINE_COMPASS_DIR}" "${BASELINE_WORK_DIR}" "") + +log "=== BASELINE: submit suite run job ===" +BASELINE_RUN_JOBID=$(patch_and_submit "${BASELINE_JOB_SCRIPT}" "") +echo "${BASELINE_RUN_JOBID}" > "${ROOT_WORK_DIR}/.baseline_run_jobid" +log "Baseline suite run job submitted: ${BASELINE_RUN_JOBID}" + +# --------------------------------------------------------------------------- +# TESTING +# --------------------------------------------------------------------------- +if [[ "${SAME_COMPASS}" == "true" ]]; then + TESTING_COMPASS_DIR="${BASELINE_COMPASS_DIR}" + TESTING_LOAD_SCRIPT="${BASELINE_LOAD_SCRIPT}" + log "=== TESTING: reusing baseline compass checkout (${TESTING_COMPASS_DIR}) ===" +else + log "=== TESTING: separate compass checkout (remote=${COMPASS_REMOTE_TESTING} ref=${COMPASS_REF_TESTING}) ===" + clone_or_checkout "${TESTING_COMPASS_DIR}" "${COMPASS_REMOTE_TESTING}" \ + "${COMPASS_REF_TESTING}" "${FORCE_TESTING}" + + log "=== TESTING: deploy compass env ===" + TESTING_LOAD_SCRIPT=$(deploy_compass_env "${TESTING_COMPASS_DIR}" "${FORCE_TESTING}") + log "Sourcing ${TESTING_LOAD_SCRIPT}" + # shellcheck disable=SC1090 + source "${TESTING_LOAD_SCRIPT}" +fi + +log "=== TESTING: MALI-Dev checkout ===" +clone_or_checkout "${TESTING_MALI_DIR}" "${MALI_REMOTE_TESTING}" \ + "${MALI_REF_TESTING}" "${FORCE_TESTING}" + +log "=== TESTING: compile MALI ===" +compile_mali "${TESTING_MALI_DIR}" "${FORCE_TESTING}" + +log "=== TESTING: compass suite setup (baseline=${BASELINE_WORK_DIR}) ===" +TESTING_JOB_SCRIPT=$(setup_suite "${TESTING_COMPASS_DIR}" "${TESTING_WORK_DIR}" "${BASELINE_WORK_DIR}") + +log "=== TESTING: submit suite run job (depends on baseline run ${BASELINE_RUN_JOBID}) ===" +TESTING_RUN_JOBID=$(patch_and_submit "${TESTING_JOB_SCRIPT}" "${BASELINE_RUN_JOBID}") +echo "${TESTING_RUN_JOBID}" > "${ROOT_WORK_DIR}/.testing_run_jobid" + +log "=== DONE ===" +log "Baseline suite run job: ${BASELINE_RUN_JOBID} (work dir: ${BASELINE_WORK_DIR})" +log "Testing suite run job: ${TESTING_RUN_JOBID} (work dir: ${TESTING_WORK_DIR}, depends on ${BASELINE_RUN_JOBID})" +log "Monitor with: squeue -u \$USER" From c568e3608fdd6609a93301b32fe5fb90b6d89b97 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 11:24:09 -0700 Subject: [PATCH 2/6] Add existing-compass-dir support to integration suite automation script Add EXISTING_COMPASS_DIR_BASELINE and EXISTING_COMPASS_DIR_TESTING config options so the script can reuse an already-cloned (and optionally already-deployed) compass checkout instead of always managing its own clone under ROOT_WORK_DIR. When set, git clone/checkout is skipped entirely to avoid touching the existing working tree, and deploy.py is only run if no matching load script is found. --force is ignored for these directories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ompass_integration_baseline_and_testing.sh | 66 ++++++++++++++++--- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh index dc38e0904a..af54c5be5d 100755 --- a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -72,6 +72,15 @@ SUITE_NAME="full_integration" COMPASS_REMOTE_BASELINE="git@github.com:MPAS-Dev/compass.git" COMPASS_REF_BASELINE="main" +# If set, use this already-cloned (and optionally already-deployed) compass +# directory for baseline instead of managing a clone under ROOT_WORK_DIR. +# git clone/checkout is never performed on this directory (your working tree +# is left untouched); COMPASS_REMOTE_BASELINE/COMPASS_REF_BASELINE above are +# ignored when this is set. deploy.py is still run here if no matching +# load_compass___.sh is found (i.e. env not yet +# deployed), otherwise it is skipped. +EXISTING_COMPASS_DIR_BASELINE="" + # Leave both of these blank to reuse the baseline compass checkout for testing # (recommended when you are only testing a MALI change). Set both to test a # different compass remote/ref as well (a separate compass-testing/ checkout @@ -79,12 +88,18 @@ COMPASS_REF_BASELINE="main" COMPASS_REMOTE_TESTING="" COMPASS_REF_TESTING="" +# Same as EXISTING_COMPASS_DIR_BASELINE, but for testing. Only relevant when a +# separate testing compass checkout is in use (i.e. COMPASS_REMOTE_TESTING or +# COMPASS_REF_TESTING is set above); ignored when testing reuses the baseline +# compass checkout. +EXISTING_COMPASS_DIR_TESTING="" + # --- MALI-Dev source ---------------------------------------------------------- MALI_REMOTE_BASELINE="git@github.com:MALI-Dev/E3SM.git" MALI_REF_BASELINE="develop" MALI_REMOTE_TESTING="git@github.com:MALI-Dev/E3SM.git" -MALI_REF_TESTING="develop" +MALI_REF_TESTING="matthewhoffman/mali/spatial-damage-threshold" # ============================================================================= # End of CONFIG. You should not need to edit anything below this line. @@ -126,9 +141,18 @@ fi mkdir -p "${ROOT_WORK_DIR}" ROOT_WORK_DIR="$(cd "${ROOT_WORK_DIR}" && pwd)" -# Hardcoded organizational structure under ROOT_WORK_DIR. -BASELINE_COMPASS_DIR="${ROOT_WORK_DIR}/compass-baseline" -TESTING_COMPASS_DIR="${ROOT_WORK_DIR}/compass-testing" +# Hardcoded organizational structure under ROOT_WORK_DIR (unless an +# EXISTING_COMPASS_DIR_* override is given above). +if [[ -n "${EXISTING_COMPASS_DIR_BASELINE}" ]]; then + BASELINE_COMPASS_DIR="${EXISTING_COMPASS_DIR_BASELINE}" +else + BASELINE_COMPASS_DIR="${ROOT_WORK_DIR}/compass-baseline" +fi +if [[ -n "${EXISTING_COMPASS_DIR_TESTING}" ]]; then + TESTING_COMPASS_DIR="${EXISTING_COMPASS_DIR_TESTING}" +else + TESTING_COMPASS_DIR="${ROOT_WORK_DIR}/compass-testing" +fi BASELINE_MALI_DIR="${ROOT_WORK_DIR}/MALI-baseline" TESTING_MALI_DIR="${ROOT_WORK_DIR}/MALI-testing" BASELINE_WORK_DIR="${ROOT_WORK_DIR}/suite-baseline" @@ -172,6 +196,12 @@ clone_or_checkout() { # deploy_compass_env -> echoes path to load script on success deploy_compass_env() { local compass_dir="$1" force="$2" + + if [[ ! -f "${compass_dir}/deploy.py" ]]; then + log "ERROR: ${compass_dir} does not look like a compass checkout (no deploy.py found)" + exit 1 + fi + local existing existing=$(find "${compass_dir}" -maxdepth 1 -name "load_compass_${MACHINE}_${COMPILER}_${MPI}.sh" 2>/dev/null | head -n1 || true) @@ -271,11 +301,19 @@ patch_and_submit() { # BASELINE # --------------------------------------------------------------------------- log "=== BASELINE: compass checkout ===" -clone_or_checkout "${BASELINE_COMPASS_DIR}" "${COMPASS_REMOTE_BASELINE}" \ - "${COMPASS_REF_BASELINE}" "${FORCE_BASELINE}" +if [[ -n "${EXISTING_COMPASS_DIR_BASELINE}" ]]; then + log "Using existing compass checkout at ${BASELINE_COMPASS_DIR} (no clone/checkout performed)" +else + clone_or_checkout "${BASELINE_COMPASS_DIR}" "${COMPASS_REMOTE_BASELINE}" \ + "${COMPASS_REF_BASELINE}" "${FORCE_BASELINE}" +fi log "=== BASELINE: deploy compass env ===" -BASELINE_LOAD_SCRIPT=$(deploy_compass_env "${BASELINE_COMPASS_DIR}" "${FORCE_BASELINE}") +if [[ -n "${EXISTING_COMPASS_DIR_BASELINE}" ]]; then + BASELINE_LOAD_SCRIPT=$(deploy_compass_env "${BASELINE_COMPASS_DIR}" "false") +else + BASELINE_LOAD_SCRIPT=$(deploy_compass_env "${BASELINE_COMPASS_DIR}" "${FORCE_BASELINE}") +fi log "Sourcing ${BASELINE_LOAD_SCRIPT}" # shellcheck disable=SC1090 source "${BASELINE_LOAD_SCRIPT}" @@ -304,11 +342,19 @@ if [[ "${SAME_COMPASS}" == "true" ]]; then log "=== TESTING: reusing baseline compass checkout (${TESTING_COMPASS_DIR}) ===" else log "=== TESTING: separate compass checkout (remote=${COMPASS_REMOTE_TESTING} ref=${COMPASS_REF_TESTING}) ===" - clone_or_checkout "${TESTING_COMPASS_DIR}" "${COMPASS_REMOTE_TESTING}" \ - "${COMPASS_REF_TESTING}" "${FORCE_TESTING}" + if [[ -n "${EXISTING_COMPASS_DIR_TESTING}" ]]; then + log "Using existing compass checkout at ${TESTING_COMPASS_DIR} (no clone/checkout performed)" + else + clone_or_checkout "${TESTING_COMPASS_DIR}" "${COMPASS_REMOTE_TESTING}" \ + "${COMPASS_REF_TESTING}" "${FORCE_TESTING}" + fi log "=== TESTING: deploy compass env ===" - TESTING_LOAD_SCRIPT=$(deploy_compass_env "${TESTING_COMPASS_DIR}" "${FORCE_TESTING}") + if [[ -n "${EXISTING_COMPASS_DIR_TESTING}" ]]; then + TESTING_LOAD_SCRIPT=$(deploy_compass_env "${TESTING_COMPASS_DIR}" "false") + else + TESTING_LOAD_SCRIPT=$(deploy_compass_env "${TESTING_COMPASS_DIR}" "${FORCE_TESTING}") + fi log "Sourcing ${TESTING_LOAD_SCRIPT}" # shellcheck disable=SC1090 source "${TESTING_LOAD_SCRIPT}" From 084e07e07580ae8e75bdf0f269bf9dd02166b827 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 11:26:02 -0700 Subject: [PATCH 3/6] Fix stdout/stderr leak corrupting captured return values log() and the noisy deploy.py/compass suite subcommands were writing to stdout inside functions whose return value is captured via command substitution (deploy_compass_env, setup_suite). This caused all of their console output to be concatenated into the "returned" path string, producing a bogus multi-line value that triggered "File name too long" when later used with `source`. Route log() to stderr and explicitly redirect deploy.py/compass suite stdout to stderr so only the intended echoed value is captured. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run_compass_integration_baseline_and_testing.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh index af54c5be5d..464bf3b97d 100755 --- a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -158,7 +158,7 @@ TESTING_MALI_DIR="${ROOT_WORK_DIR}/MALI-testing" BASELINE_WORK_DIR="${ROOT_WORK_DIR}/suite-baseline" TESTING_WORK_DIR="${ROOT_WORK_DIR}/suite-testing" -log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2; } # --------------------------------------------------------------------------- # Helpers @@ -214,7 +214,7 @@ deploy_compass_env() { log "Running deploy.py in ${compass_dir} (machine=${MACHINE} compiler=${COMPILER} mpi=${MPI})" ( cd "${compass_dir}" - ./deploy.py --with-albany --compiler "${COMPILER}" --mpi "${MPI}" --machine "${MACHINE}" + ./deploy.py --with-albany --compiler "${COMPILER}" --mpi "${MPI}" --machine "${MACHINE}" 1>&2 ) existing=$(find "${compass_dir}" -maxdepth 1 -name "load_compass_${MACHINE}_${COMPILER}_${MPI}.sh" 2>/dev/null | head -n1 || true) @@ -253,9 +253,9 @@ setup_suite() { ( cd "${compass_dir}" if [[ -n "${baseline_dir}" ]]; then - compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -b "${baseline_dir}" -w "${work_dir}" -s + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -b "${baseline_dir}" -w "${work_dir}" -s 1>&2 else - compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -w "${work_dir}" -s + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -w "${work_dir}" -s 1>&2 fi ) From 7872f6c6ef785620f88305616e12e15f97ab685f Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 11:27:28 -0700 Subject: [PATCH 4/6] Relax set -u/-e while sourcing compass load scripts The compass-generated load script chains into third-party pixi/conda activation hooks (e.g. cartopy_offline_data-activate.sh) that reference variables without guarding against nounset, causing our strict `set -euo pipefail` to abort with "unbound variable" errors that have nothing to do with this script's own logic. Add a source_load_script helper that temporarily disables -u/-e for the duration of the source call, then restores our strict settings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ompass_integration_baseline_and_testing.sh | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh index 464bf3b97d..6b74a92cad 100755 --- a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -297,6 +297,23 @@ patch_and_submit() { echo "${jobid}" } +# source_load_script +# +# Sources a compass-generated load script with our strict `set -euo pipefail` +# temporarily relaxed, since these scripts (and the pixi/conda activation +# hooks they chain into) are third-party and not written against nounset/ +# errexit conventions (e.g. they may reference variables that are only +# conditionally set). +source_load_script() { + local script="$1" + set +u +e + # shellcheck disable=SC1090 + source "${script}" + local rc=$? + set -u -e + return "${rc}" +} + # --------------------------------------------------------------------------- # BASELINE # --------------------------------------------------------------------------- @@ -315,8 +332,7 @@ else BASELINE_LOAD_SCRIPT=$(deploy_compass_env "${BASELINE_COMPASS_DIR}" "${FORCE_BASELINE}") fi log "Sourcing ${BASELINE_LOAD_SCRIPT}" -# shellcheck disable=SC1090 -source "${BASELINE_LOAD_SCRIPT}" +source_load_script "${BASELINE_LOAD_SCRIPT}" log "=== BASELINE: MALI-Dev checkout ===" clone_or_checkout "${BASELINE_MALI_DIR}" "${MALI_REMOTE_BASELINE}" \ @@ -356,8 +372,7 @@ else TESTING_LOAD_SCRIPT=$(deploy_compass_env "${TESTING_COMPASS_DIR}" "${FORCE_TESTING}") fi log "Sourcing ${TESTING_LOAD_SCRIPT}" - # shellcheck disable=SC1090 - source "${TESTING_LOAD_SCRIPT}" + source_load_script "${TESTING_LOAD_SCRIPT}" fi log "=== TESTING: MALI-Dev checkout ===" From 36f9fd9e58a73ace5e261c25803dc11235568463 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 11:35:39 -0700 Subject: [PATCH 5/6] Pass -p (MALI build path) to compass suite setup compass suite requires -p pointing at the compiled MALI component directory. Add a mali_path parameter to setup_suite and pass /components/mpas-albany-landice for both baseline and testing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ompass_integration_baseline_and_testing.sh | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh index 6b74a92cad..fc6f2efe27 100755 --- a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -19,7 +19,8 @@ # - source the generated load script # - clone MALI-Dev from MALI_REMOTE_BASELINE @ MALI_REF_BASELINE # - compile MALI (mpas-albany-landice) -# - `compass suite -c landice -t full_integration -w -s` +# - `compass suite -c landice -t full_integration -p \ +# -w -s` # 2. Submit the compass-generated suite job script for BASELINE via sbatch. # 3. TESTING prep (synchronous, on the login node): # - reuse the BASELINE compass checkout if COMPASS_REMOTE_TESTING/ @@ -27,8 +28,8 @@ # separate compass copy for testing # - clone/build a *separate* MALI-Dev copy from MALI_REMOTE_TESTING @ # MALI_REF_TESTING -# - `compass suite -c landice -t full_integration -b \ -# -w -s` +# - `compass suite -c landice -t full_integration -p \ +# -b -w -s` # 4. Submit the compass-generated suite job script for TESTING via sbatch # with `--dependency=afterok:` so it automatically # waits for the baseline run to finish successfully before starting -- @@ -243,19 +244,19 @@ compile_mali() { ) } -# setup_suite +# setup_suite # Returns (echoes) the path to the compass-generated suite job script. setup_suite() { - local compass_dir="$1" work_dir="$2" baseline_dir="$3" + local compass_dir="$1" work_dir="$2" mali_path="$3" baseline_dir="$4" mkdir -p "${work_dir}" - log "Setting up ${SUITE_CORE}/${SUITE_NAME} suite in ${work_dir}" + log "Setting up ${SUITE_CORE}/${SUITE_NAME} suite in ${work_dir} (MALI path: ${mali_path})" ( cd "${compass_dir}" if [[ -n "${baseline_dir}" ]]; then - compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -b "${baseline_dir}" -w "${work_dir}" -s 1>&2 + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -p "${mali_path}" -b "${baseline_dir}" -w "${work_dir}" -s 1>&2 else - compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -w "${work_dir}" -s 1>&2 + compass suite -c "${SUITE_CORE}" -t "${SUITE_NAME}" -p "${mali_path}" -w "${work_dir}" -s 1>&2 fi ) @@ -342,7 +343,8 @@ log "=== BASELINE: compile MALI ===" compile_mali "${BASELINE_MALI_DIR}" "${FORCE_BASELINE}" log "=== BASELINE: compass suite setup ===" -BASELINE_JOB_SCRIPT=$(setup_suite "${BASELINE_COMPASS_DIR}" "${BASELINE_WORK_DIR}" "") +BASELINE_JOB_SCRIPT=$(setup_suite "${BASELINE_COMPASS_DIR}" "${BASELINE_WORK_DIR}" \ + "${BASELINE_MALI_DIR}/components/mpas-albany-landice" "") log "=== BASELINE: submit suite run job ===" BASELINE_RUN_JOBID=$(patch_and_submit "${BASELINE_JOB_SCRIPT}" "") @@ -383,7 +385,8 @@ log "=== TESTING: compile MALI ===" compile_mali "${TESTING_MALI_DIR}" "${FORCE_TESTING}" log "=== TESTING: compass suite setup (baseline=${BASELINE_WORK_DIR}) ===" -TESTING_JOB_SCRIPT=$(setup_suite "${TESTING_COMPASS_DIR}" "${TESTING_WORK_DIR}" "${BASELINE_WORK_DIR}") +TESTING_JOB_SCRIPT=$(setup_suite "${TESTING_COMPASS_DIR}" "${TESTING_WORK_DIR}" \ + "${TESTING_MALI_DIR}/components/mpas-albany-landice" "${BASELINE_WORK_DIR}") log "=== TESTING: submit suite run job (depends on baseline run ${BASELINE_RUN_JOBID}) ===" TESTING_RUN_JOBID=$(patch_and_submit "${TESTING_JOB_SCRIPT}" "${BASELINE_RUN_JOBID}") From 7f03281573dff9ff4e164331c3fd67bb34834600 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 12 Sep 2026 11:44:49 -0700 Subject: [PATCH 6/6] Submit compass suite job scripts from their own directory compass suite job scripts expect to be submitted from the suite work dir they were written into (e.g. for relative paths used inside the script). Cd into that directory before calling sbatch instead of submitting from wherever this script happens to be run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run_compass_integration_baseline_and_testing.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh index fc6f2efe27..0887961582 100755 --- a/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh +++ b/utils/automate_integration_suite_comparison/run_compass_integration_baseline_and_testing.sh @@ -293,8 +293,13 @@ patch_and_submit() { log "Submitting ${job_script}" fi - local jobid - jobid=$(sbatch --parsable "${dep_args[@]}" "${job_script}") + # compass suite job scripts assume they are submitted from the directory + # they were written into (the suite work dir), so cd there before calling + # sbatch rather than submitting from wherever this script happens to run. + local job_dir job_base jobid + job_dir=$(dirname "${job_script}") + job_base=$(basename "${job_script}") + jobid=$(cd "${job_dir}" && sbatch --parsable "${dep_args[@]}" "${job_base}") echo "${jobid}" }