From 8b4e0a510140eaa2bf9c12ed597d7863a41f59dc Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 19:29:30 -0700 Subject: [PATCH 01/19] Update spack to v1.2.2 --- .gitignore | 1 + .uberenv_config.json | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..11d6a5ee 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026" } From ff13119379c69f7f7d6db4c8bff51d30c6dc78e6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:39:01 -0400 Subject: [PATCH 02/19] Add unified build.sh and HIP support to the TPL superbuild - build.sh: single driver for z5tux (cpu/cuda/hip) and LC (matrix, tioga, tuo-cpu, tuo-gpu), cmake+ninja by default, --spack for uberenv builds - CMakeLists: HIP plumbing for raja, chai, caliper and hypre - Make python explicit in the py-sphinx externals (spack v1.2 deprecation) --- .gitignore | 1 + CMakeLists.txt | 120 ++- build.sh | 813 ++++++++++++++++++ docker/pecan-spack.yaml | 2 + docker/ubuntu-spack.yaml | 2 + .../spack_configs/toss_4_x86_64_ib/spack.yaml | 2 + 6 files changed, 937 insertions(+), 3 deletions(-) create mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 488ea52a..36d0b675 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build* +!/build.sh /install* .cproject .project diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dc910be1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ option( ENABLE_TRILINOS "" ON ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -737,7 +814,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +861,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +892,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +925,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +969,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +994,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..723ab3f3 --- /dev/null +++ b/build.sh @@ -0,0 +1,813 @@ +#!/usr/bin/env bash +# +# Unified driver to build the GEOS third-party libraries. +# +# Supported configurations: +# z5tux gcc + mpich, host only +# z5tux-cuda gcc + mpich + CUDA (sm_120) +# z5tux-hip amdclang + mpich + ROCm (gfx1100) +# z5tux-sycl not supported yet (see NOTES below) +# matrix gcc + mvapich2 + CUDA (sm_90) +# tioga amdclang + cray-mpich + ROCm (gfx90a) +# tuo-cpu amdclang + cray-mpich, host only +# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) +# +# By default the TPLs are built with the CMake superbuild of this repository +# (CMakeLists.txt), configured with the Ninja generator both for the superbuild +# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same +# configurations are built through uberenv/spack instead. +# +# The script is meant to be run *on* the target machine: on the LC systems, +# clone/copy this repository there and run it from a login node (optionally +# with --alloc to push the compilation onto a compute node). +# +# NOTES +# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack +# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. +# * Trilinos is off by default (GEOS defaults to the hypre interface); enable +# it with --trilinos. +# * The ROCm configurations compile Fortran with gfortran, not amdflang: the +# flang runtime is not pulled in when clang++ links Fortran objects, which +# breaks superlu_dist. Override with --fc if needed. +# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels +# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect +# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. +# +# Usage: ./build.sh [config] [options] [-D= ...] +# Run ./build.sh --help for the full option list. + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +# ------------------------------------------------------------------------------ +# Defaults +# ------------------------------------------------------------------------------ +CONFIG="" +BUILD_TYPE="Release" +BUILD_DIR="" +INSTALL_DIR="" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" +TOP_JOBS=1 +GENERATOR="auto" # auto | ninja | make +GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target +ROCM_DIR_OPT="" +CUDA_DIR_OPT="" +MPI_DIR_OPT="" +FC_OPT="" +USE_ALLOC=0 +ALLOC_CMD="" +ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" +LOAD_MODULES=1 +MODULES_OPT="" +CLEAN=0 +DO_CONFIGURE=1 +DO_BUILD=1 +USE_SPACK=0 +SPACK_SPEC_OPT="" +ENABLE_TRILINOS="OFF" +ENABLE_OPENMP="" # per-config default unless set here +ENABLE_ROCTX="OFF" +ENABLE_SUPERLU_DIST="ON" +ENABLE_HYPRE_GPU_PROFILING="OFF" +DRY_RUN=0 +LOG_FILE="" +EXTRA_CMAKE_ARGS=() +FORWARDED_ARGS=() + +# Populated by select_config() +CFG_DESC="" +CFG_SUPPORTED=1 +CFG_UNSUPPORTED_REASON="" +CMAKE_ARGS=() +MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip + +# ------------------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------------------ +log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } +warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } +die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } + +run() { + printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" + if [[ ${DRY_RUN} -eq 0 ]]; then + "$@" + fi +} + +# Newest (version-sorted) existing path matching the given glob, empty if none. +newest_path() { + local match + # shellcheck disable=SC2086 # deliberate globbing + match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" + printf '%s' "${match}" +} + +require_file() { + [[ -e "$1" ]] || die "${2:-required file} not found: $1" +} + +require_exec() { + [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" +} + +add_arg() { CMAKE_ARGS+=( "$@" ); } + +usage() { + cat <= ...] [-- ] + +Configurations (auto-detected from the hostname when omitted): + z5tux gcc + mpich, host only + z5tux-cuda gcc + mpich + CUDA, sm_120 + z5tux-hip amdclang + mpich + ROCm, gfx1100 + z5tux-sycl not supported yet + matrix gcc + mvapich2 + CUDA, sm_90 (LC) + tioga amdclang + cray-mpich + ROCm, gfx90a (LC) + tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) + tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) + +Options: + -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) + -b, --build-dir build directory (default: /build--) + -i, --install-dir install directory (default: /install--) + -j, --jobs parallelism inside each TPL (default: ${JOBS}) + --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) + --generator auto|ninja|make (default: auto, prefers ninja) + --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) + --rocm ROCm installation to use + --cuda CUDA toolkit to use + --mpi MPI installation to use (expects /bin/mpicc) + --fc Fortran compiler to use + --trilinos build Trilinos as well (off by default) + --openmp override the per-configuration OpenMP setting + --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) + --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) + --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) + --alloc run the build step through the machine's scheduler + --alloc-cmd scheduler command to use with --alloc + --account bank used by --alloc (default: ${ACCOUNT}) + --modules "" replace the default module list + --no-modules do not touch modules at all + --spack build through uberenv/spack instead of the cmake superbuild + --spec "" spack spec to build (implies --spack) + --clean remove the build and install directories first + --configure-only stop after the cmake configure step + --build-only skip the configure step (reuse an existing build directory) + --log also write the output to + -n, --dry-run print the commands instead of running them + -l, --list list the configurations and exit + -h, --help show this help and exit + + Any -D= argument is forwarded to cmake (or, with --spack, any + argument after -- is forwarded to uberenv.py). + +Examples: + ./${SCRIPT_NAME} z5tux + ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 + ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 + ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node + ./${SCRIPT_NAME} matrix --spack # same config, through spack + ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF +EOF +} + +# Static description of each configuration: used by --list and by the run banner, +# so that neither needs the toolchain to be resolved first. +describe_config() { + case "$1" in + z5tux) printf 'z5tux: gcc + mpich, host only' ;; + z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; + z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; + z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; + matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; + tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; + *) printf '' ;; + esac +} + +# Modules to load on the LC machines. Kept separate from select_config() so that +# they are loaded *before* the compilers and cmake/ninja are looked up. +modules_for_config() { + case "$1" in + matrix) + printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; + tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) + printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; + *) + ;; + esac +} + +# Machine a configuration runs on (used for the scheduler defaults). +machine_for_config() { + case "$1" in + z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; + matrix) printf 'matrix' ;; + tioga|tioga-gpu) printf 'tioga' ;; + tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; + *) printf '' ;; + esac +} + +# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. +# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). +spack_spec_for_config() { + case "$1" in + z5tux) + printf '~docs %%gcc ^vtk generator=ninja' ;; + z5tux-cuda) + printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; + z5tux-hip) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; + matrix) + printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) + printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; + tuo-gpu|tuolumne-gpu|tuolumne) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; + *) + printf '' ;; + esac +} + +# Configurations that cannot be built (yet); prints the reason. +unsupported_reason() { + case "$1" in + z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; + *) printf '' ;; + esac +} + +list_configs() { + local cfg + printf '%-12s %s\n' "CONFIG" "DESCRIPTION" + for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do + printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" + done + printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ + "no SYCL support in the superbuild nor in the spack packages" +} + +# ------------------------------------------------------------------------------ +# Argument parsing +# ------------------------------------------------------------------------------ +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -l|--list) list_configs; exit 0 ;; + -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; + -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; + --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --trilinos) ENABLE_TRILINOS="ON"; shift ;; + --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; + --roctx) ENABLE_ROCTX="ON"; shift ;; + --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; + --alloc) USE_ALLOC=1; shift ;; + --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; + --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; + --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --no-modules) LOAD_MODULES=0; shift ;; + --spack) USE_SPACK=1; shift ;; + --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; + --clean) CLEAN=1; shift ;; + --configure-only) DO_BUILD=0; shift ;; + --build-only) DO_CONFIGURE=0; shift ;; + --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; + -n|--dry-run) DRY_RUN=1; shift ;; + -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; + --) shift; FORWARDED_ARGS+=( "$@" ); break ;; + -*) die "unknown option '$1' (see --help)" ;; + *) + [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" + CONFIG="$1"; shift ;; + esac +done + +# ------------------------------------------------------------------------------ +# Configurations +# ------------------------------------------------------------------------------ + +# Compilers + MPI wrappers of the mpich installation used on z5tux. +setup_z5tux_mpi() { + local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" + require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ + -DMPIEXEC_NUMPROC_FLAG="-np" + MPI_PREFIX="${mpi_dir}" +} + +# The mpich/cray-mpich wrappers pick their back-end compiler from these. +export_mpich_backend() { + export MPICH_CC="$1" + export MPICH_CXX="$2" + export MPICH_FC="$3" + export MPICH_F90="$3" +} + +# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. +resolve_rocm() { + local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" + if [[ -z "${rocm}" ]]; then + rocm="$(newest_path '/opt/rocm-*')" + [[ -n "${rocm}" ]] || rocm="/opt/rocm" + fi + [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" + printf '%s' "${rocm}" +} + +# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. +resolve_cuda() { + local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" + if [[ -z "${cuda}" ]]; then + cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" + [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" + [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" + fi + [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" + printf '%s' "${cuda}" +} + +# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over +# amdflang: clang++ links Fortran objects without the flang runtime, which +# breaks the mixed-language TPLs (superlu_dist). +resolve_fortran() { + local rocm="$1" fc + if [[ -n "${FC_OPT}" ]]; then + printf '%s' "${FC_OPT}"; return 0 + fi + for fc in /usr/bin/gfortran \ + "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ + "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do + if [[ -n "${fc}" && -x "${fc}" ]]; then + printf '%s' "${fc}"; return 0 + fi + done + warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" + printf '%s' "${rocm}/llvm/bin/amdflang" +} + +# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. +setup_rocm_compilers() { + local rocm="$1" + local cc="${rocm}/llvm/bin/amdclang" + local cxx="${rocm}/llvm/bin/amdclang++" + local fc + fc="$(resolve_fortran "${rocm}")" + require_exec "${cc}" "amdclang" + require_exec "${cxx}" "amdclang++" + require_exec "${fc}" "Fortran compiler (use --fc )" + add_arg -DCMAKE_C_COMPILER="${cc}" \ + -DCMAKE_CXX_COMPILER="${cxx}" \ + -DCMAKE_Fortran_COMPILER="${fc}" + export_mpich_backend "${cc}" "${cxx}" "${fc}" +} + +# HIP settings shared by z5tux-hip, tioga and tuo-gpu. +setup_hip() { + local rocm="$1" arch="$2" + # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. + require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" + add_arg -DENABLE_HIP=ON \ + -DROCM_PATH="${rocm}" \ + -DHIP_ROOT="${rocm}" \ + -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ + -DCMAKE_HIP_ARCHITECTURES="${arch}" \ + -DCMAKE_HIP_STANDARD=17 \ + -DENABLE_ROCTX="${ENABLE_ROCTX}" \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=HIP + # hypre compiles its device sources outside of cmake and needs the MPI headers + [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" +} + +# CUDA settings shared by z5tux-cuda and matrix. +setup_cuda() { + local cuda="$1" arch="$2" + require_exec "${cuda}/bin/nvcc" "nvcc" + add_arg -DENABLE_CUDA=ON \ + -DENABLE_CUDA_NVTOOLSEXT=ON \ + -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ + -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ + -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ + -DCUDA_ARCH="sm_${arch}" \ + -DCMAKE_CUDA_STANDARD=17 \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=CUDA +} + +# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the +# compiler family ('amd' for amdclang, 'cray' for cce). +setup_cray_mpi() { + local flavor="${1:-amd}" + local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" + if [[ -z "${mpi_dir}" ]]; then + mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" + fi + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" +} + +# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. +setup_blas_lapack() { + local candidates=( "$@" ) lib + for lib in "${candidates[@]}"; do + if [[ -e "${lib}" ]]; then + add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" + return 0 + fi + done + if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then + add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" + return 0 + fi + warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" +} + +# Resolves the toolchain of a configuration into cmake cache arguments. +select_config() { + local cfg="$1" + CMAKE_ARGS=() + + case "${cfg}" in + # --------------------------------------------------------------- z5tux --- + z5tux) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + ;; + + z5tux-cuda) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" + ;; + + z5tux-hip) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" + ;; + + # ------------------------------------------------------------------ LC --- + matrix) + + local gcc_dir mpi_dir + gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" + [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" + [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" + add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ + -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + + local gcc_ver + gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" + mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" + [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" + + setup_blas_lapack /usr/lib64/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" + ;; + + tioga|tioga-gpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" + ;; + + tuo-cpu|tuolumne-cpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + ;; + + tuo-gpu|tuolumne-gpu|tuolumne) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" + ;; + + *) + die "unknown configuration '${cfg}' (run --list to see the supported ones)" + ;; + esac +} + +# Scheduler command used by --alloc, per machine. +default_alloc_cmd() { + case "${CFG_MACHINE}" in + matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tioga) printf 'flux run -N 1 -t 180m' ;; + *) printf '' ;; + esac +} + +detect_config() { + local host="${HOSTNAME:-$(hostname)}" + case "${host}" in + z5tux*) printf 'z5tux' ;; + matrix*|rzmatrix*) printf 'matrix' ;; + tioga*|rzvernal*) printf 'tioga' ;; + tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; + *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; + esac +} + +# ------------------------------------------------------------------------------ +# Modules +# ------------------------------------------------------------------------------ +init_modules() { + if ! declare -F module >/dev/null 2>&1; then + local init + for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do + if [[ -r "${init}" ]]; then + # shellcheck disable=SC1090 + source "${init}" + break + fi + done + fi + declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 +} + +load_modules() { + local -a mods=() + local m + if [[ -n "${MODULES_OPT}" ]]; then + read -r -a mods <<< "${MODULES_OPT}" + else + while IFS= read -r m; do + [[ -n "${m}" ]] && mods+=( "${m}" ) + done < <(modules_for_config "${CONFIG}") + fi + [[ ${#mods[@]} -gt 0 ]] || return 0 + + if ! init_modules; then + warn "no module command available, skipping module loading" + return 0 + fi + + local m + for m in "${mods[@]}"; do + if [[ ${DRY_RUN} -eq 1 ]]; then + log "+ module load ${m}" + continue + fi + if module load "${m}" >/dev/null 2>&1; then + log "module load ${m}" + else + # Retry without the version: LC renames those more often than the toolchains move. + local base="${m%%/*}" + if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then + log "module load ${base} (fallback, '${m}' unavailable)" + else + warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" + fi + fi + done +} + +# ------------------------------------------------------------------------------ +# Build back-ends +# ------------------------------------------------------------------------------ +resolve_generator() { + case "${GENERATOR}" in + ninja) + command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" + ;; + make) + ;; + auto) + if command -v ninja >/dev/null 2>&1; then + GENERATOR="ninja" + else + warn "ninja not found in PATH, falling back to Unix Makefiles" + GENERATOR="make" + fi + ;; + *) + die "unknown generator '${GENERATOR}' (auto|ninja|make)" + ;; + esac +} + +configure_cmake() { + local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) + + if [[ "${GENERATOR}" == "ninja" ]]; then + cmd+=( -G Ninja -DENABLE_NINJA=ON ) + else + cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) + fi + + cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DNUM_PROC="${JOBS}" + -DENABLE_TRILINOS="${ENABLE_TRILINOS}" + -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" + -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" + -DBLT_CXX_STD=c++17 + -DCMAKE_CXX_STANDARD=17 ) + cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) + cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) + cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) + + run "${cmd[@]}" + write_build_env +} + +# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and +# the TPLs configured outside of cmake (hypre) inherit them from this script. +# Dump them next to the build so that running ninja by hand stays equivalent. +write_build_env() { + [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 + local env_file="${BUILD_DIR}/build-env.sh" + cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" + log "generator : ${GENERATOR}" + log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" + + if [[ ${DO_CONFIGURE} -eq 1 ]]; then + configure_cmake + else + [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" + fi + + if [[ ${DO_BUILD} -eq 1 ]]; then + build_cmake + fi + fi + + log "done." + if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then + log "TPLs installed in ${INSTALL_DIR}" + fi +} + +if [[ -n "${LOG_FILE}" ]]; then + main 2>&1 | tee "${LOG_FILE}" + exit "${PIPESTATUS[0]}" +else + main +fi diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..8b29be18 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -231,6 +231,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: From 39cc571c43ab666366870aa0d2b4a4cd6021558f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:40:30 -0400 Subject: [PATCH 03/19] Remove build.sh from versioning --- .gitignore | 1 - build.sh | 813 ----------------------------------------------------- 2 files changed, 814 deletions(-) delete mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 36d0b675..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /build* -!/build.sh /install* .cproject .project diff --git a/build.sh b/build.sh deleted file mode 100755 index 723ab3f3..00000000 --- a/build.sh +++ /dev/null @@ -1,813 +0,0 @@ -#!/usr/bin/env bash -# -# Unified driver to build the GEOS third-party libraries. -# -# Supported configurations: -# z5tux gcc + mpich, host only -# z5tux-cuda gcc + mpich + CUDA (sm_120) -# z5tux-hip amdclang + mpich + ROCm (gfx1100) -# z5tux-sycl not supported yet (see NOTES below) -# matrix gcc + mvapich2 + CUDA (sm_90) -# tioga amdclang + cray-mpich + ROCm (gfx90a) -# tuo-cpu amdclang + cray-mpich, host only -# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) -# -# By default the TPLs are built with the CMake superbuild of this repository -# (CMakeLists.txt), configured with the Ninja generator both for the superbuild -# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same -# configurations are built through uberenv/spack instead. -# -# The script is meant to be run *on* the target machine: on the LC systems, -# clone/copy this repository there and run it from a login node (optionally -# with --alloc to push the compilation onto a compute node). -# -# NOTES -# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack -# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. -# * Trilinos is off by default (GEOS defaults to the hypre interface); enable -# it with --trilinos. -# * The ROCm configurations compile Fortran with gfortran, not amdflang: the -# flang runtime is not pulled in when clang++ links Fortran objects, which -# breaks superlu_dist. Override with --fc if needed. -# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels -# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect -# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. -# -# Usage: ./build.sh [config] [options] [-D= ...] -# Run ./build.sh --help for the full option list. - -set -euo pipefail - -SCRIPT_NAME="$(basename "$0")" -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# ------------------------------------------------------------------------------ -# Defaults -# ------------------------------------------------------------------------------ -CONFIG="" -BUILD_TYPE="Release" -BUILD_DIR="" -INSTALL_DIR="" -JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" -TOP_JOBS=1 -GENERATOR="auto" # auto | ninja | make -GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target -ROCM_DIR_OPT="" -CUDA_DIR_OPT="" -MPI_DIR_OPT="" -FC_OPT="" -USE_ALLOC=0 -ALLOC_CMD="" -ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" -LOAD_MODULES=1 -MODULES_OPT="" -CLEAN=0 -DO_CONFIGURE=1 -DO_BUILD=1 -USE_SPACK=0 -SPACK_SPEC_OPT="" -ENABLE_TRILINOS="OFF" -ENABLE_OPENMP="" # per-config default unless set here -ENABLE_ROCTX="OFF" -ENABLE_SUPERLU_DIST="ON" -ENABLE_HYPRE_GPU_PROFILING="OFF" -DRY_RUN=0 -LOG_FILE="" -EXTRA_CMAKE_ARGS=() -FORWARDED_ARGS=() - -# Populated by select_config() -CFG_DESC="" -CFG_SUPPORTED=1 -CFG_UNSUPPORTED_REASON="" -CMAKE_ARGS=() -MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip - -# ------------------------------------------------------------------------------ -# Helpers -# ------------------------------------------------------------------------------ -log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } -warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } -die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } - -run() { - printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" - if [[ ${DRY_RUN} -eq 0 ]]; then - "$@" - fi -} - -# Newest (version-sorted) existing path matching the given glob, empty if none. -newest_path() { - local match - # shellcheck disable=SC2086 # deliberate globbing - match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" - printf '%s' "${match}" -} - -require_file() { - [[ -e "$1" ]] || die "${2:-required file} not found: $1" -} - -require_exec() { - [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" -} - -add_arg() { CMAKE_ARGS+=( "$@" ); } - -usage() { - cat <= ...] [-- ] - -Configurations (auto-detected from the hostname when omitted): - z5tux gcc + mpich, host only - z5tux-cuda gcc + mpich + CUDA, sm_120 - z5tux-hip amdclang + mpich + ROCm, gfx1100 - z5tux-sycl not supported yet - matrix gcc + mvapich2 + CUDA, sm_90 (LC) - tioga amdclang + cray-mpich + ROCm, gfx90a (LC) - tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) - tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) - -Options: - -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) - -b, --build-dir build directory (default: /build--) - -i, --install-dir install directory (default: /install--) - -j, --jobs parallelism inside each TPL (default: ${JOBS}) - --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) - --generator auto|ninja|make (default: auto, prefers ninja) - --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) - --rocm ROCm installation to use - --cuda CUDA toolkit to use - --mpi MPI installation to use (expects /bin/mpicc) - --fc Fortran compiler to use - --trilinos build Trilinos as well (off by default) - --openmp override the per-configuration OpenMP setting - --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) - --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) - --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) - --alloc run the build step through the machine's scheduler - --alloc-cmd scheduler command to use with --alloc - --account bank used by --alloc (default: ${ACCOUNT}) - --modules "" replace the default module list - --no-modules do not touch modules at all - --spack build through uberenv/spack instead of the cmake superbuild - --spec "" spack spec to build (implies --spack) - --clean remove the build and install directories first - --configure-only stop after the cmake configure step - --build-only skip the configure step (reuse an existing build directory) - --log also write the output to - -n, --dry-run print the commands instead of running them - -l, --list list the configurations and exit - -h, --help show this help and exit - - Any -D= argument is forwarded to cmake (or, with --spack, any - argument after -- is forwarded to uberenv.py). - -Examples: - ./${SCRIPT_NAME} z5tux - ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 - ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 - ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node - ./${SCRIPT_NAME} matrix --spack # same config, through spack - ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF -EOF -} - -# Static description of each configuration: used by --list and by the run banner, -# so that neither needs the toolchain to be resolved first. -describe_config() { - case "$1" in - z5tux) printf 'z5tux: gcc + mpich, host only' ;; - z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; - z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; - z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; - matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; - tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; - *) printf '' ;; - esac -} - -# Modules to load on the LC machines. Kept separate from select_config() so that -# they are loaded *before* the compilers and cmake/ninja are looked up. -modules_for_config() { - case "$1" in - matrix) - printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; - tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) - printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; - *) - ;; - esac -} - -# Machine a configuration runs on (used for the scheduler defaults). -machine_for_config() { - case "$1" in - z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; - matrix) printf 'matrix' ;; - tioga|tioga-gpu) printf 'tioga' ;; - tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; - *) printf '' ;; - esac -} - -# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. -# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). -spack_spec_for_config() { - case "$1" in - z5tux) - printf '~docs %%gcc ^vtk generator=ninja' ;; - z5tux-cuda) - printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; - z5tux-hip) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; - matrix) - printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) - printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; - tuo-gpu|tuolumne-gpu|tuolumne) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; - *) - printf '' ;; - esac -} - -# Configurations that cannot be built (yet); prints the reason. -unsupported_reason() { - case "$1" in - z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; - *) printf '' ;; - esac -} - -list_configs() { - local cfg - printf '%-12s %s\n' "CONFIG" "DESCRIPTION" - for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do - printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" - done - printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ - "no SYCL support in the superbuild nor in the spack packages" -} - -# ------------------------------------------------------------------------------ -# Argument parsing -# ------------------------------------------------------------------------------ -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) usage; exit 0 ;; - -l|--list) list_configs; exit 0 ;; - -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; - -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; - --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --trilinos) ENABLE_TRILINOS="ON"; shift ;; - --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; - --roctx) ENABLE_ROCTX="ON"; shift ;; - --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; - --alloc) USE_ALLOC=1; shift ;; - --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; - --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; - --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --no-modules) LOAD_MODULES=0; shift ;; - --spack) USE_SPACK=1; shift ;; - --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; - --clean) CLEAN=1; shift ;; - --configure-only) DO_BUILD=0; shift ;; - --build-only) DO_CONFIGURE=0; shift ;; - --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; - -n|--dry-run) DRY_RUN=1; shift ;; - -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; - --) shift; FORWARDED_ARGS+=( "$@" ); break ;; - -*) die "unknown option '$1' (see --help)" ;; - *) - [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" - CONFIG="$1"; shift ;; - esac -done - -# ------------------------------------------------------------------------------ -# Configurations -# ------------------------------------------------------------------------------ - -# Compilers + MPI wrappers of the mpich installation used on z5tux. -setup_z5tux_mpi() { - local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" - require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ - -DMPIEXEC_NUMPROC_FLAG="-np" - MPI_PREFIX="${mpi_dir}" -} - -# The mpich/cray-mpich wrappers pick their back-end compiler from these. -export_mpich_backend() { - export MPICH_CC="$1" - export MPICH_CXX="$2" - export MPICH_FC="$3" - export MPICH_F90="$3" -} - -# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. -resolve_rocm() { - local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" - if [[ -z "${rocm}" ]]; then - rocm="$(newest_path '/opt/rocm-*')" - [[ -n "${rocm}" ]] || rocm="/opt/rocm" - fi - [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" - printf '%s' "${rocm}" -} - -# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. -resolve_cuda() { - local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" - if [[ -z "${cuda}" ]]; then - cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" - [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" - [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" - fi - [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" - printf '%s' "${cuda}" -} - -# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over -# amdflang: clang++ links Fortran objects without the flang runtime, which -# breaks the mixed-language TPLs (superlu_dist). -resolve_fortran() { - local rocm="$1" fc - if [[ -n "${FC_OPT}" ]]; then - printf '%s' "${FC_OPT}"; return 0 - fi - for fc in /usr/bin/gfortran \ - "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ - "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do - if [[ -n "${fc}" && -x "${fc}" ]]; then - printf '%s' "${fc}"; return 0 - fi - done - warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" - printf '%s' "${rocm}/llvm/bin/amdflang" -} - -# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. -setup_rocm_compilers() { - local rocm="$1" - local cc="${rocm}/llvm/bin/amdclang" - local cxx="${rocm}/llvm/bin/amdclang++" - local fc - fc="$(resolve_fortran "${rocm}")" - require_exec "${cc}" "amdclang" - require_exec "${cxx}" "amdclang++" - require_exec "${fc}" "Fortran compiler (use --fc )" - add_arg -DCMAKE_C_COMPILER="${cc}" \ - -DCMAKE_CXX_COMPILER="${cxx}" \ - -DCMAKE_Fortran_COMPILER="${fc}" - export_mpich_backend "${cc}" "${cxx}" "${fc}" -} - -# HIP settings shared by z5tux-hip, tioga and tuo-gpu. -setup_hip() { - local rocm="$1" arch="$2" - # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. - require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" - add_arg -DENABLE_HIP=ON \ - -DROCM_PATH="${rocm}" \ - -DHIP_ROOT="${rocm}" \ - -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ - -DCMAKE_HIP_ARCHITECTURES="${arch}" \ - -DCMAKE_HIP_STANDARD=17 \ - -DENABLE_ROCTX="${ENABLE_ROCTX}" \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=HIP - # hypre compiles its device sources outside of cmake and needs the MPI headers - [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" -} - -# CUDA settings shared by z5tux-cuda and matrix. -setup_cuda() { - local cuda="$1" arch="$2" - require_exec "${cuda}/bin/nvcc" "nvcc" - add_arg -DENABLE_CUDA=ON \ - -DENABLE_CUDA_NVTOOLSEXT=ON \ - -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ - -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ - -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ - -DCUDA_ARCH="sm_${arch}" \ - -DCMAKE_CUDA_STANDARD=17 \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=CUDA -} - -# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the -# compiler family ('amd' for amdclang, 'cray' for cce). -setup_cray_mpi() { - local flavor="${1:-amd}" - local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" - if [[ -z "${mpi_dir}" ]]; then - mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" - fi - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" -} - -# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. -setup_blas_lapack() { - local candidates=( "$@" ) lib - for lib in "${candidates[@]}"; do - if [[ -e "${lib}" ]]; then - add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" - return 0 - fi - done - if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then - add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" - return 0 - fi - warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" -} - -# Resolves the toolchain of a configuration into cmake cache arguments. -select_config() { - local cfg="$1" - CMAKE_ARGS=() - - case "${cfg}" in - # --------------------------------------------------------------- z5tux --- - z5tux) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - ;; - - z5tux-cuda) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" - ;; - - z5tux-hip) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" - ;; - - # ------------------------------------------------------------------ LC --- - matrix) - - local gcc_dir mpi_dir - gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" - [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" - [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" - add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ - -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - - local gcc_ver - gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" - mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" - [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" - - setup_blas_lapack /usr/lib64/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" - ;; - - tioga|tioga-gpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" - ;; - - tuo-cpu|tuolumne-cpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - ;; - - tuo-gpu|tuolumne-gpu|tuolumne) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" - ;; - - *) - die "unknown configuration '${cfg}' (run --list to see the supported ones)" - ;; - esac -} - -# Scheduler command used by --alloc, per machine. -default_alloc_cmd() { - case "${CFG_MACHINE}" in - matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tioga) printf 'flux run -N 1 -t 180m' ;; - *) printf '' ;; - esac -} - -detect_config() { - local host="${HOSTNAME:-$(hostname)}" - case "${host}" in - z5tux*) printf 'z5tux' ;; - matrix*|rzmatrix*) printf 'matrix' ;; - tioga*|rzvernal*) printf 'tioga' ;; - tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; - *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; - esac -} - -# ------------------------------------------------------------------------------ -# Modules -# ------------------------------------------------------------------------------ -init_modules() { - if ! declare -F module >/dev/null 2>&1; then - local init - for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do - if [[ -r "${init}" ]]; then - # shellcheck disable=SC1090 - source "${init}" - break - fi - done - fi - declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 -} - -load_modules() { - local -a mods=() - local m - if [[ -n "${MODULES_OPT}" ]]; then - read -r -a mods <<< "${MODULES_OPT}" - else - while IFS= read -r m; do - [[ -n "${m}" ]] && mods+=( "${m}" ) - done < <(modules_for_config "${CONFIG}") - fi - [[ ${#mods[@]} -gt 0 ]] || return 0 - - if ! init_modules; then - warn "no module command available, skipping module loading" - return 0 - fi - - local m - for m in "${mods[@]}"; do - if [[ ${DRY_RUN} -eq 1 ]]; then - log "+ module load ${m}" - continue - fi - if module load "${m}" >/dev/null 2>&1; then - log "module load ${m}" - else - # Retry without the version: LC renames those more often than the toolchains move. - local base="${m%%/*}" - if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then - log "module load ${base} (fallback, '${m}' unavailable)" - else - warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" - fi - fi - done -} - -# ------------------------------------------------------------------------------ -# Build back-ends -# ------------------------------------------------------------------------------ -resolve_generator() { - case "${GENERATOR}" in - ninja) - command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" - ;; - make) - ;; - auto) - if command -v ninja >/dev/null 2>&1; then - GENERATOR="ninja" - else - warn "ninja not found in PATH, falling back to Unix Makefiles" - GENERATOR="make" - fi - ;; - *) - die "unknown generator '${GENERATOR}' (auto|ninja|make)" - ;; - esac -} - -configure_cmake() { - local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) - - if [[ "${GENERATOR}" == "ninja" ]]; then - cmd+=( -G Ninja -DENABLE_NINJA=ON ) - else - cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) - fi - - cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DNUM_PROC="${JOBS}" - -DENABLE_TRILINOS="${ENABLE_TRILINOS}" - -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" - -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" - -DBLT_CXX_STD=c++17 - -DCMAKE_CXX_STANDARD=17 ) - cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) - cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) - cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) - - run "${cmd[@]}" - write_build_env -} - -# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and -# the TPLs configured outside of cmake (hypre) inherit them from this script. -# Dump them next to the build so that running ninja by hand stays equivalent. -write_build_env() { - [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 - local env_file="${BUILD_DIR}/build-env.sh" - cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" - log "generator : ${GENERATOR}" - log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" - - if [[ ${DO_CONFIGURE} -eq 1 ]]; then - configure_cmake - else - [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" - fi - - if [[ ${DO_BUILD} -eq 1 ]]; then - build_cmake - fi - fi - - log "done." - if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then - log "TPLs installed in ${INSTALL_DIR}" - fi -} - -if [[ -n "${LOG_FILE}" ]]; then - main 2>&1 | tee "${LOG_FILE}" - exit "${PIPESTATUS[0]}" -else - main -fi From 497f78ef30d2dabdb44d7a2e7e6f3152b026174d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:56:03 -0400 Subject: [PATCH 04/19] Remove blueos spack config --- .../blueos_3_ppc64le_ib_p9/spack.yaml | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr From ee8eafdf169c1d3c8e975755d0015bde0ced3ec1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:09 -0400 Subject: [PATCH 05/19] Remove Lassen and ESSL support --- CMakeLists.txt | 5 --- cmake/thirdparty/FindMathLibraries.cmake | 9 ---- scripts/createLapackForESSL.sh | 33 --------------- .../spack_packages/packages/essl/package.py | 41 ------------------- .../spack_packages/packages/geosx/package.py | 30 +------------- 5 files changed, 1 insertion(+), 117 deletions(-) delete mode 100755 scripts/createLapackForESSL.sh delete mode 100644 scripts/spack_packages/packages/essl/package.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dc910be1..379a583f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -666,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..d75a4ccc 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -356,9 +356,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -375,16 +372,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): + if sys_type in ('toss_4_x86_64_ib_cray'): cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) else: @@ -470,10 +458,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -568,12 +552,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_MKL', True)) cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -689,9 +667,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -769,9 +744,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) From 8bb038e041012f4163562791028fbf126fd426fd Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:25 -0400 Subject: [PATCH 06/19] Use the new spack installer on the LC machines --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 2 ++ scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 8b29be18..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..48af6ee7 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false From bc978545b488f7f30dd967433c4566ba3e7a74c0 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:12:47 -0400 Subject: [PATCH 07/19] Disable Trilinos by default --- CMakeLists.txt | 2 +- scripts/spack_packages/packages/geosx/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 379a583f..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index d75a4ccc..5bd2a7ae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -64,7 +64,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') From 7ac30d4b5f8453db87410455826de0b6c30e8d6b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 03:01:45 -0400 Subject: [PATCH 08/19] Install xz-devel in the Rocky TPL image --- docker/tpl-rockylinux.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..ac39daf7 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf From 317a1cd7e8a4815bf23697ca3dc080c2e42183f8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 11:32:19 -0400 Subject: [PATCH 09/19] Bump blt to 0.7.2 --- cmake/blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/blt b/cmake/blt index e783e30f..f08f95e0 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit e783e30f2823ee1a208f7f90741b41c1f5a08063 +Subproject commit f08f95e0f49b1d53db8be2dd577d1cdbfd9e9f90 From bdc6d264cf874565fb8118ca65d08ed86fb6b6ad Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 14:01:14 -0400 Subject: [PATCH 10/19] Fix Ubuntu CUDA TPL image builds - Inject the streak2 CA bundle into Ubuntu Dockerfiles, not only Rocky - Patch fmt 11.2.0 so nvcc + clang does not fail on _BitInt - Keep extra .dockerignore rules and add CPU/memory limits to reproduce_ci.sh --- .dockerignore | 13 +++++++ .github/workflows/docker_build_tpls.yml | 37 ++++++++++++++++++- scripts/docker-build.sh | 9 ++++- scripts/reproduce_ci.sh | 18 ++++++++- .../packages/fmt/nvcc-clang-bitint.patch | 17 +++++++++ .../spack_packages/packages/fmt/package.py | 8 ++++ 6 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch create mode 100644 scripts/spack_packages/packages/fmt/package.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d2a12691 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +build +build-* +install +install-* +*_tpls +.rocm-ci-local.* +*.cmake +*.log +__pycache__ +vtk-9.6.2-local/ +tplMirror/*.tar.gz +tplMirror/chai-develop-060425/ diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 22333d22..dabc2ff7 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -180,8 +180,43 @@ jobs: - name: Inject CA certificate into Docker build if: matrix.RUNS_ON == 'streak2' run: | + set -euo pipefail cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt - sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }} + dockerfile="${{ matrix.TPL_DOCKERFILE }}" + if grep -q 'RUN dnf clean all' "${dockerfile}"; then + sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}" + else + # Ubuntu CUDA jobs also run on streak2. Insert CA trust immediately + # after the first FROM so apt/git/pip can talk through the MITM proxy. + awk ' + BEGIN { inserted=0 } + { + print + if (!inserted && $0 ~ /^FROM /) { + print "COPY ca-bundle.crt /usr/local/share/ca-certificates/llnl-ca-bundle.crt" + print "RUN mkdir -p /etc/ssl/certs && \\" + print " if [ -f /etc/ssl/certs/ca-certificates.crt ]; then \\" + print " cat /usr/local/share/ca-certificates/llnl-ca-bundle.crt >> /etc/ssl/certs/ca-certificates.crt; \\" + print " else \\" + print " cp /usr/local/share/ca-certificates/llnl-ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; \\" + print " fi && \\" + print " (command -v update-ca-certificates >/dev/null 2>&1 && update-ca-certificates || true)" + print "ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \\" + print " REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \\" + print " CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \\" + print " GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \\" + print " PIP_CERT=/etc/ssl/certs/ca-certificates.crt" + inserted=1 + } + } + END { + if (!inserted) { + print "ERROR: no FROM line in Dockerfile" > "/dev/stderr" + exit 1 + } + } + ' "${dockerfile}" > "${dockerfile}.ca" && mv "${dockerfile}.ca" "${dockerfile}" + fi - name: Run the docker build docker script env: diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 59218646..93afb5fc 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,8 +1,13 @@ #!/bin/bash env -# We save memory for the docker context -echo .git > .dockerignore +# Keep .git out of the build context. Do not clobber an existing +# .dockerignore; local checkouts often ignore large untracked trees. +if [ -f .dockerignore ]; then + grep -qxF '.git' .dockerignore || echo '.git' >> .dockerignore +else + echo .git > .dockerignore +fi # Get uberenv submodule git submodule update --init scripts/uberenv diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 35ca6c48..5a4fb9d0 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -27,6 +27,8 @@ DOCKER_TAG=${DOCKER_TAG:-local} COMMIT=${COMMIT:-$(git rev-parse HEAD 2>/dev/null || echo HEAD)} INSTALL_DIR_ROOT=${INSTALL_DIR_ROOT:-/opt/GEOS} HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} +BUILDX_CPUS=${BUILDX_CPUS:-} +BUILDX_MEMORY=${BUILDX_MEMORY:-} keep_artifacts=0 dry_run=0 @@ -63,6 +65,8 @@ Build settings (each also readable from the environment): --base-sha SHA DOCKER_BASE_IMAGE_SHA [${DOCKER_BASE_IMAGE_SHA}] --install-dir DIR INSTALL_DIR_ROOT [${INSTALL_DIR_ROOT}] --host-config F HOST_CONFIG [${HOST_CONFIG}] + --cpus N BUILDX_CPUS [${BUILDX_CPUS:-unset}] + --memory SIZE BUILDX_MEMORY [${BUILDX_MEMORY:-unset}] Other: --keep Keep the image and buildx builder after a successful build @@ -128,6 +132,8 @@ while (( $# )); do --base-sha) DOCKER_BASE_IMAGE_SHA=$2; shift 2 ;; --install-dir) INSTALL_DIR_ROOT=$2; shift 2 ;; --host-config) HOST_CONFIG=$2; shift 2 ;; + --cpus) BUILDX_CPUS=$2; shift 2 ;; + --memory) BUILDX_MEMORY=$2; shift 2 ;; --keep) keep_artifacts=1; shift ;; --dry-run) dry_run=1; shift ;; -h|--help) usage; exit 0 ;; @@ -188,10 +194,20 @@ build() { printf ' commit %s\n' "${COMMIT}" printf ' install dir %s\n' "${INSTALL_DIR_ROOT}" printf ' host config %s\n' "${HOST_CONFIG}" + [[ ${BUILDX_CPUS} ]] && printf ' cpus %s\n' "${BUILDX_CPUS}" + [[ ${BUILDX_MEMORY} ]] && printf ' memory %s\n' "${BUILDX_MEMORY}" return fi - if ! docker buildx create --name "${builder}" --driver docker-container --bootstrap >/dev/null; then + create_args=(--name "${builder}" --driver docker-container) + if [[ ${BUILDX_CPUS} ]]; then + # docker-container has no "cpus" option; CFS quota 100000us * N == N CPUs. + create_args+=(--driver-opt "cpu-period=100000") + create_args+=(--driver-opt "cpu-quota=$((BUILDX_CPUS * 100000))") + create_args+=(--driver-opt "cpuset-cpus=0-$((BUILDX_CPUS - 1))") + fi + [[ ${BUILDX_MEMORY} ]] && create_args+=(--driver-opt "memory=${BUILDX_MEMORY}") + if ! docker buildx create "${create_args[@]}" --bootstrap >/dev/null; then printf 'FAILED TO CREATE BUILDER: %s\n' "${name}" >&2 failed=1 return diff --git a/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch b/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch new file mode 100644 index 00000000..1fd49630 --- /dev/null +++ b/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch @@ -0,0 +1,17 @@ +# nvcc with a Clang host compiler cannot parse Clang's _BitInt extension. +# Upstream: https://github.com/fmtlib/fmt/pull/4564 +--- a/include/fmt/base.h ++++ b/include/fmt/base.h +@@ -420,7 +420,11 @@ inline auto map(uint128_opt) -> monostate { return {}; } + #endif + + #ifndef FMT_USE_BITINT +-# define FMT_USE_BITINT (FMT_CLANG_VERSION >= 1500) ++# if FMT_CLANG_VERSION >= 1500 && !defined(__CUDACC__) ++# define FMT_USE_BITINT 1 ++# else ++# define FMT_USE_BITINT 0 ++# endif + #endif + + #if FMT_USE_BITINT diff --git a/scripts/spack_packages/packages/fmt/package.py b/scripts/spack_packages/packages/fmt/package.py new file mode 100644 index 00000000..ecf2570f --- /dev/null +++ b/scripts/spack_packages/packages/fmt/package.py @@ -0,0 +1,8 @@ +from spack.package import * +from spack_repo.builtin.packages.fmt.package import Fmt as BuiltinFmt + + +class Fmt(BuiltinFmt): + # nvcc + clang host compiler cannot parse Clang's _BitInt extension. + # https://github.com/fmtlib/fmt/pull/4564 (merged after fmt 11.2.0) + patch("nvcc-clang-bitint.patch", when="@11.1:11.2.0") From d351f7b71dbd19d3bd905609b2d7e8616342fa97 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 14:03:00 -0400 Subject: [PATCH 11/19] Add a logged --all sweep to reproduce_ci.sh --- scripts/reproduce_ci.sh | 66 +++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 5a4fb9d0..768f57b1 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -29,11 +29,17 @@ INSTALL_DIR_ROOT=${INSTALL_DIR_ROOT:-/opt/GEOS} HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} BUILDX_CPUS=${BUILDX_CPUS:-} BUILDX_MEMORY=${BUILDX_MEMORY:-} +LOG_DIR=${LOG_DIR:-} keep_artifacts=0 dry_run=0 +sweep_all=0 selected=() +job_slug() { + printf '%s' "$1" | tr -c 'A-Za-z0-9' '_' | tr -s '_' +} + job_name() { printf '%s' "${MATRIX[$1]%%|*}"; } list_jobs() { @@ -56,7 +62,9 @@ Job selection: --ci NAME Build only this job. Repeatable. Matched exactly, or by a unique case-insensitive substring (e.g. --ci "clang 22"). An index from --list also works (e.g. --ci 8). - --all Build every job in the matrix. + --all Build every job in the matrix, one after another. Writes + per-job logs and summary.log under --log-dir (default + /tmp/geos-tpl-ci-builds). Stops on the first failure. --list List the available jobs and exit. Build settings (each also readable from the environment): @@ -67,6 +75,7 @@ Build settings (each also readable from the environment): --host-config F HOST_CONFIG [${HOST_CONFIG}] --cpus N BUILDX_CPUS [${BUILDX_CPUS:-unset}] --memory SIZE BUILDX_MEMORY [${BUILDX_MEMORY:-unset}] + --log-dir DIR LOG_DIR [${LOG_DIR:-/tmp/geos-tpl-ci-builds with --all}] Other: --keep Keep the image and buildx builder after a successful build @@ -125,7 +134,7 @@ while (( $# )); do --ci) [[ ${2:-} ]] || { echo "--ci needs a job name" >&2; exit 2; } selected+=("$2"); shift 2 ;; --ci=*) selected+=("${1#*=}"); shift ;; - --all) selected=(__all__); shift ;; + --all) selected=(__all__); sweep_all=1; shift ;; --list) list_jobs; exit 0 ;; --tag) DOCKER_TAG=$2; shift 2 ;; --commit) COMMIT=$2; shift 2 ;; @@ -134,6 +143,7 @@ while (( $# )); do --host-config) HOST_CONFIG=$2; shift 2 ;; --cpus) BUILDX_CPUS=$2; shift 2 ;; --memory) BUILDX_MEMORY=$2; shift 2 ;; + --log-dir) LOG_DIR=$2; shift 2 ;; --keep) keep_artifacts=1; shift ;; --dry-run) dry_run=1; shift ;; -h|--help) usage; exit 0 ;; @@ -151,6 +161,7 @@ fi indices=() if [[ ${selected[0]} == __all__ ]]; then indices=("${!MATRIX[@]}") + [[ ${LOG_DIR} ]] || LOG_DIR=/tmp/geos-tpl-ci-builds else resolve_failed=0 for query in "${selected[@]}"; do @@ -210,7 +221,7 @@ build() { if ! docker buildx create "${create_args[@]}" --bootstrap >/dev/null; then printf 'FAILED TO CREATE BUILDER: %s\n' "${name}" >&2 failed=1 - return + return 1 fi if ! TPL_DOCKERFILE=${dockerfile} \ @@ -230,7 +241,7 @@ build() { printf 'FAILED: %s (builder retained: %s)\n' "${name}" "${builder}" >&2 failed=1 failed_builders+=("${builder}") - return + return 1 fi if (( keep_artifacts )); then @@ -252,24 +263,58 @@ build() { if (( cleanup_failed )); then failed=1 failed_builders+=("${builder}") - else - printf 'PASSED: %s (image and build cache removed)\n' "${name}" + return 1 fi + printf 'PASSED: %s (image and build cache removed)\n' "${name}" } printf 'Selected %d of %d job(s).\n' "${#indices[@]}" "${#MATRIX[@]}" +summary= +if [[ ${LOG_DIR} ]] && (( ! dry_run )); then + mkdir -p "${LOG_DIR}" + summary=${LOG_DIR}/summary.log + { + echo "Sequential TPL docker CI reproductions started $(date -Is)" + echo "Limits: ${BUILDX_CPUS:-all} CPUs, ${BUILDX_MEMORY:-unlimited} RAM, one image at a time" + echo + } | tee "${summary}" +fi + row=0 for idx in "${indices[@]}"; do row=$((row + 1)) IFS='|' read -r name dockerfile base_tag base_repository repository gcc clang spec \ <<<"${MATRIX[${idx}]}" - build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ - "${repository}" "${gcc}" "${clang}" "${spec}" + + slog= + if [[ ${summary} ]]; then + slog=${LOG_DIR}/$(job_slug "${name}").log + echo "==== START ${name} $(date -Is) ====" | tee -a "${summary}" | tee "${slog}" + if build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ + "${repository}" "${gcc}" "${clang}" "${spec}" > >(tee -a "${slog}") 2>&1; then + echo "==== PASSED ${name} $(date -Is) ====" | tee -a "${summary}" | tee -a "${slog}" + else + echo "==== FAILED ${name} $(date -Is) ====" | tee -a "${summary}" | tee -a "${slog}" + echo "See ${slog}" | tee -a "${summary}" + if (( sweep_all )); then + echo "One or more Docker CI builds failed." >&2 + if ((${#failed_builders[@]})); then + echo "Retained builders for investigation:" >&2 + printf ' %s\n' "${failed_builders[@]}" >&2 + fi + exit 1 + fi + fi + else + build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ + "${repository}" "${gcc}" "${clang}" "${spec}" + fi done if (( dry_run )); then - printf '\nDry run: nothing was built.\n' + [[ ${LOG_DIR} ]] && printf '\nLog directory: %s\n' "${LOG_DIR}" + printf 'Dry run: nothing was built.\n' exit 0 fi @@ -282,4 +327,7 @@ if (( failed )); then exit 1 fi +if [[ ${summary} ]]; then + echo "==== ALL PASSED $(date -Is) ====" | tee -a "${summary}" +fi echo "All selected Docker CI builds passed." From 255304e49dbb6d9102e5fa83823d7178bd93fb53 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 14:33:33 -0400 Subject: [PATCH 12/19] Keep the streak2 CA bundle after Ubuntu ca-certificates upgrades. --- .github/workflows/docker_build_tpls.yml | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index dabc2ff7..dda506f8 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -186,26 +186,30 @@ jobs: if grep -q 'RUN dnf clean all' "${dockerfile}"; then sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}" else - # Ubuntu CUDA jobs also run on streak2. Insert CA trust immediately - # after the first FROM so apt/git/pip can talk through the MITM proxy. + # Ubuntu CUDA jobs also run on streak2. Keep the LLNL bundle in a + # dedicated file: apt-get install ca-certificates regenerates + # /etc/ssl/certs/ca-certificates.crt and would drop a concatenated + # MITM cert. Do not place the multi-cert bundle under + # /usr/local/share/ca-certificates (update-ca-certificates skips or + # fails those files). awk ' BEGIN { inserted=0 } { print if (!inserted && $0 ~ /^FROM /) { - print "COPY ca-bundle.crt /usr/local/share/ca-certificates/llnl-ca-bundle.crt" - print "RUN mkdir -p /etc/ssl/certs && \\" + print "COPY ca-bundle.crt /etc/ssl/certs/llnl-ca-bundle.crt" + print "RUN mkdir -p /etc/apt/apt.conf.d && \\" print " if [ -f /etc/ssl/certs/ca-certificates.crt ]; then \\" - print " cat /usr/local/share/ca-certificates/llnl-ca-bundle.crt >> /etc/ssl/certs/ca-certificates.crt; \\" + print " cat /etc/ssl/certs/llnl-ca-bundle.crt >> /etc/ssl/certs/ca-certificates.crt; \\" print " else \\" - print " cp /usr/local/share/ca-certificates/llnl-ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; \\" + print " cp /etc/ssl/certs/llnl-ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; \\" print " fi && \\" - print " (command -v update-ca-certificates >/dev/null 2>&1 && update-ca-certificates || true)" - print "ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \\" - print " REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \\" - print " CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \\" - print " GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \\" - print " PIP_CERT=/etc/ssl/certs/ca-certificates.crt" + print " echo '\''Acquire::https::CaInfo \"/etc/ssl/certs/llnl-ca-bundle.crt\";'\'' > /etc/apt/apt.conf.d/99-llnl-ca" + print "ENV SSL_CERT_FILE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " REQUESTS_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " CURL_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " GIT_SSL_CAINFO=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " PIP_CERT=/etc/ssl/certs/llnl-ca-bundle.crt" inserted=1 } } From bb7eab3dbafe087630cf246ac86b197343c9b761 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 14:55:24 -0400 Subject: [PATCH 13/19] Fix Ubuntu CUDA streak2 builds through the MITM proxy. --- .github/workflows/docker_build_tpls.yml | 24 +++++++++++------------- docker/tpl-ubuntu.Dockerfile | 13 ++++++++++++- scripts/docker-build.sh | 8 ++++++++ 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index dda506f8..771a6078 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -167,8 +167,15 @@ jobs: lfs: true - name: Set up Docker Buildx + if: matrix.RUNS_ON != 'streak2' uses: docker/setup-buildx-action@v4 + - name: Set up Docker Buildx + if: matrix.RUNS_ON == 'streak2' + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host + - name: Print environment run: printenv @@ -186,25 +193,15 @@ jobs: if grep -q 'RUN dnf clean all' "${dockerfile}"; then sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}" else - # Ubuntu CUDA jobs also run on streak2. Keep the LLNL bundle in a - # dedicated file: apt-get install ca-certificates regenerates - # /etc/ssl/certs/ca-certificates.crt and would drop a concatenated - # MITM cert. Do not place the multi-cert bundle under - # /usr/local/share/ca-certificates (update-ca-certificates skips or - # fails those files). + # Ubuntu CUDA jobs also run on streak2. Only COPY the host bundle; + # tpl-ubuntu.Dockerfile installs it after ca-certificates so the + # package postinst cannot wipe it. ENV covers pip/curl/git. awk ' BEGIN { inserted=0 } { print if (!inserted && $0 ~ /^FROM /) { print "COPY ca-bundle.crt /etc/ssl/certs/llnl-ca-bundle.crt" - print "RUN mkdir -p /etc/apt/apt.conf.d && \\" - print " if [ -f /etc/ssl/certs/ca-certificates.crt ]; then \\" - print " cat /etc/ssl/certs/llnl-ca-bundle.crt >> /etc/ssl/certs/ca-certificates.crt; \\" - print " else \\" - print " cp /etc/ssl/certs/llnl-ca-bundle.crt /etc/ssl/certs/ca-certificates.crt; \\" - print " fi && \\" - print " echo '\''Acquire::https::CaInfo \"/etc/ssl/certs/llnl-ca-bundle.crt\";'\'' > /etc/apt/apt.conf.d/99-llnl-ca" print "ENV SSL_CERT_FILE=/etc/ssl/certs/llnl-ca-bundle.crt \\" print " REQUESTS_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" print " CURL_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" @@ -235,6 +232,7 @@ jobs: COMMIT: ${{ github.sha }} BUILD_DIR: ${{ github.workspace }} DOCKER_TAG: ${{ github.event.number }}-${{ github.run_number }} + DOCKER_NETWORK: ${{ matrix.RUNS_ON == 'streak2' && 'host' || '' }} run: bash -x ./scripts/docker-build.sh - name: Login to DockerHub diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 367092a3..5e67f0ac 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -51,11 +51,22 @@ RUN apt-get update && \ lbzip2 \ bzip2 \ gnupg && \ + if [ -f /etc/ssl/certs/llnl-ca-bundle.crt ]; then \ + mkdir -p /usr/local/share/ca-certificates && \ + awk 'BEGIN {n=0} /-----BEGIN/ {n++; f=sprintf("/usr/local/share/ca-certificates/llnl-%03d.crt", n)} n>0 {print > f}' \ + /etc/ssl/certs/llnl-ca-bundle.crt && \ + update-ca-certificates ; \ + fi && \ apt-get clean && rm -rf /var/lib/apt/lists/* # Install clingo for Spack. Do not upgrade Ubuntu's Debian-managed pip in # place; Ubuntu 24.04's pip package cannot be uninstalled by pip. -RUN python3 -m pip install --break-system-packages clingo +# --trusted-host covers streak2 MITM when the system store is incomplete. +RUN python3 -m pip install --break-system-packages \ + --trusted-host pypi.org \ + --trusted-host files.pythonhosted.org \ + --trusted-host pypi.python.org \ + clingo # MPI environment. CC/CXX/FC come from the base image. ENV MPICC=/usr/bin/mpicc \ diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 93afb5fc..553453cd 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -39,6 +39,14 @@ if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION BUILDER_ARGS=() if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi if [ "${DOCKER_LOAD:-0}" = 1 ]; then BUILDER_ARGS+=(--load); fi +if [ -n "${DOCKER_NETWORK:-}" ]; then BUILDER_ARGS+=(--network "${DOCKER_NETWORK}"); fi + +# Forward proxy settings into RUN steps (BuildKit special-cases these args). +for v in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy; do + if [ -n "${!v:-}" ]; then + EXTRA_BUILD_ARGS+=(--build-arg "${v}=${!v}") + fi +done docker build --progress=plain \ "${BUILDER_ARGS[@]}" \ From b1e3f5b820a9c88a615ed3aae3c16d6f07655611 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 15:38:43 -0400 Subject: [PATCH 14/19] Fix Ubuntu streak2 CA bootstrap --- .github/workflows/docker_build_tpls.yml | 5 +++-- docker/tpl-ubuntu.Dockerfile | 18 +++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 771a6078..9be42e6c 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -194,8 +194,9 @@ jobs: sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}" else # Ubuntu CUDA jobs also run on streak2. Only COPY the host bundle; - # tpl-ubuntu.Dockerfile installs it after ca-certificates so the - # package postinst cannot wipe it. ENV covers pip/curl/git. + # tpl-ubuntu.Dockerfile configures APT before its first update and + # installs the bundle after ca-certificates so the package postinst + # cannot wipe it. ENV covers pip/curl/git. awk ' BEGIN { inserted=0 } { diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 5e67f0ac..aeb99b24 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -30,7 +30,16 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR # Packages needed both for the TPL build and for the downstream GEOS build. # We avoid reinstalling anything already present in the base image (compiler, # cmake, doxygen, blas/lapack-dev when included by base PACKAGES, etc.). -RUN apt-get update && \ +# The streak2 workflow injects the LLNL CA bundle before this stage. Configure +# APT to use that bundle before the first update, since the base image does not +# yet trust the runner's MITM certificate and ca-certificates is installed below. +RUN if [ -f /etc/ssl/certs/llnl-ca-bundle.crt ]; then \ + mkdir -p /etc/apt/apt.conf.d && \ + printf '%s\n' \ + 'Acquire::https::CaInfo "/etc/ssl/certs/llnl-ca-bundle.crt";' \ + > /etc/apt/apt.conf.d/99-llnl-ca; \ + fi && \ + apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ apt-get install -y --no-install-recommends \ ca-certificates \ @@ -61,12 +70,7 @@ RUN apt-get update && \ # Install clingo for Spack. Do not upgrade Ubuntu's Debian-managed pip in # place; Ubuntu 24.04's pip package cannot be uninstalled by pip. -# --trusted-host covers streak2 MITM when the system store is incomplete. -RUN python3 -m pip install --break-system-packages \ - --trusted-host pypi.org \ - --trusted-host files.pythonhosted.org \ - --trusted-host pypi.python.org \ - clingo +RUN python3 -m pip install --break-system-packages clingo # MPI environment. CC/CXX/FC come from the base image. ENV MPICC=/usr/bin/mpicc \ From 424df6507e0efb7ca03c396d2a70be679909e630 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 17:04:52 -0400 Subject: [PATCH 15/19] Limit Spack build parallelism --- docker/tpl-rockylinux.Dockerfile | 5 +++++ docker/tpl-ubuntu.Dockerfile | 4 ++++ scripts/docker-build.sh | 1 + 3 files changed, 10 insertions(+) diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index ac39daf7..856435c1 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -90,6 +90,9 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR ARG SPEC +# Keep Spack's package build parallelism bounded on self-hosted runners. The +# value remains overridable with --build-arg SPACK_BUILD_JOBS=... . +ARG SPACK_BUILD_JOBS=4 RUN dnf -y install \ tbb-devel \ @@ -147,6 +150,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k " ; \ else \ ./scripts/uberenv/uberenv.py \ @@ -154,6 +158,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k ; \ fi && \ rm -f lvarray* && \ diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index aeb99b24..6fe77e07 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -116,6 +116,9 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR ARG SPEC +# Keep Spack's package build parallelism bounded on self-hosted runners. The +# value remains overridable with --build-arg SPACK_BUILD_JOBS=... . +ARG SPACK_BUILD_JOBS=4 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ @@ -153,6 +156,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k && \ rm -f lvarray* && \ cp *.cmake /spack-generated.cmake && \ diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 553453cd..a8d3cddb 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -35,6 +35,7 @@ echo "Docker base image is ${DOCKER_BASE_IMAGE}" EXTRA_BUILD_ARGS=() if [ -n "${GCC_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi +if [ -n "${SPACK_BUILD_JOBS:-}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS}"); fi BUILDER_ARGS=() if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi From ace0478b7b1d410f6c352eb8424a5a09e3efeb19 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 17:17:01 -0400 Subject: [PATCH 16/19] Use Ubuntu CA bundle --- docker/tpl-ubuntu.Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 6fe77e07..58625f8e 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -68,6 +68,17 @@ RUN if [ -f /etc/ssl/certs/llnl-ca-bundle.crt ]; then \ fi && \ apt-get clean && rm -rf /var/lib/apt/lists/* +# APT needs the runner bundle for its first HTTPS transaction. Once +# ca-certificates has rebuilt Ubuntu's trust store, use that merged bundle for +# all subsequent Python, curl, Git, and pip operations. Keeping the runner +# bundle selected globally can omit Ubuntu's public roots on some streak2 +# images. +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \ + PIP_CERT=/etc/ssl/certs/ca-certificates.crt + # Install clingo for Spack. Do not upgrade Ubuntu's Debian-managed pip in # place; Ubuntu 24.04's pip package cannot be uninstalled by pip. RUN python3 -m pip install --break-system-packages clingo From 942f42a317ee2a66d51cad74bae168025210e2d8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 17:33:54 -0400 Subject: [PATCH 17/19] Handle Ubuntu FIPS on streak2 --- docker/openssl-non-fips.cnf | 14 ++++++++++++++ docker/tpl-ubuntu.Dockerfile | 7 +++++++ 2 files changed, 21 insertions(+) create mode 100644 docker/openssl-non-fips.cnf diff --git a/docker/openssl-non-fips.cnf b/docker/openssl-non-fips.cnf new file mode 100644 index 00000000..f7362003 --- /dev/null +++ b/docker/openssl-non-fips.cnf @@ -0,0 +1,14 @@ +openssl_conf = openssl_init + +[openssl_init] +providers = provider_sect +alg_section = algorithm_sect + +[provider_sect] +default = default_sect + +[default_sect] +activate = 1 + +[algorithm_sect] +default_properties = fips=no diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 58625f8e..eff95832 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -22,6 +22,13 @@ FROM ${DOCKER_BASE_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR ARG CLANG_VERSION +# streak2 hosts can enable kernel FIPS mode even though this Ubuntu image has +# no FIPS provider. Use OpenSSL's default provider for the image's package +# downloads and source builds, matching the GEOS streak2 container setup. +COPY docker/openssl-non-fips.cnf /etc/ssl/openssl-non-fips.cnf +ENV OPENSSL_FORCE_FIPS_MODE=0 \ + OPENSSL_CONF=/etc/ssl/openssl-non-fips.cnf + # Install directory provided as a docker build argument; forwarded via ENV # (GEOSX_TPL_DIR is part of the image contract consumed by GEOS). ARG INSTALL_DIR From 79a5d4c5b8ffd6327a02445089c6e6954a30cf0b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 15 Aug 2026 18:48:14 -0400 Subject: [PATCH 18/19] Remove CI reproduction logging --- scripts/reproduce_ci.sh | 55 +++-------------------------------------- 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 768f57b1..301ccfdd 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -29,17 +29,11 @@ INSTALL_DIR_ROOT=${INSTALL_DIR_ROOT:-/opt/GEOS} HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} BUILDX_CPUS=${BUILDX_CPUS:-} BUILDX_MEMORY=${BUILDX_MEMORY:-} -LOG_DIR=${LOG_DIR:-} keep_artifacts=0 dry_run=0 -sweep_all=0 selected=() -job_slug() { - printf '%s' "$1" | tr -c 'A-Za-z0-9' '_' | tr -s '_' -} - job_name() { printf '%s' "${MATRIX[$1]%%|*}"; } list_jobs() { @@ -62,9 +56,7 @@ Job selection: --ci NAME Build only this job. Repeatable. Matched exactly, or by a unique case-insensitive substring (e.g. --ci "clang 22"). An index from --list also works (e.g. --ci 8). - --all Build every job in the matrix, one after another. Writes - per-job logs and summary.log under --log-dir (default - /tmp/geos-tpl-ci-builds). Stops on the first failure. + --all Build every job in the matrix. --list List the available jobs and exit. Build settings (each also readable from the environment): @@ -75,7 +67,6 @@ Build settings (each also readable from the environment): --host-config F HOST_CONFIG [${HOST_CONFIG}] --cpus N BUILDX_CPUS [${BUILDX_CPUS:-unset}] --memory SIZE BUILDX_MEMORY [${BUILDX_MEMORY:-unset}] - --log-dir DIR LOG_DIR [${LOG_DIR:-/tmp/geos-tpl-ci-builds with --all}] Other: --keep Keep the image and buildx builder after a successful build @@ -134,7 +125,7 @@ while (( $# )); do --ci) [[ ${2:-} ]] || { echo "--ci needs a job name" >&2; exit 2; } selected+=("$2"); shift 2 ;; --ci=*) selected+=("${1#*=}"); shift ;; - --all) selected=(__all__); sweep_all=1; shift ;; + --all) selected=(__all__); shift ;; --list) list_jobs; exit 0 ;; --tag) DOCKER_TAG=$2; shift 2 ;; --commit) COMMIT=$2; shift 2 ;; @@ -143,7 +134,6 @@ while (( $# )); do --host-config) HOST_CONFIG=$2; shift 2 ;; --cpus) BUILDX_CPUS=$2; shift 2 ;; --memory) BUILDX_MEMORY=$2; shift 2 ;; - --log-dir) LOG_DIR=$2; shift 2 ;; --keep) keep_artifacts=1; shift ;; --dry-run) dry_run=1; shift ;; -h|--help) usage; exit 0 ;; @@ -161,7 +151,6 @@ fi indices=() if [[ ${selected[0]} == __all__ ]]; then indices=("${!MATRIX[@]}") - [[ ${LOG_DIR} ]] || LOG_DIR=/tmp/geos-tpl-ci-builds else resolve_failed=0 for query in "${selected[@]}"; do @@ -270,50 +259,17 @@ build() { printf 'Selected %d of %d job(s).\n' "${#indices[@]}" "${#MATRIX[@]}" -summary= -if [[ ${LOG_DIR} ]] && (( ! dry_run )); then - mkdir -p "${LOG_DIR}" - summary=${LOG_DIR}/summary.log - { - echo "Sequential TPL docker CI reproductions started $(date -Is)" - echo "Limits: ${BUILDX_CPUS:-all} CPUs, ${BUILDX_MEMORY:-unlimited} RAM, one image at a time" - echo - } | tee "${summary}" -fi - row=0 for idx in "${indices[@]}"; do row=$((row + 1)) IFS='|' read -r name dockerfile base_tag base_repository repository gcc clang spec \ <<<"${MATRIX[${idx}]}" - slog= - if [[ ${summary} ]]; then - slog=${LOG_DIR}/$(job_slug "${name}").log - echo "==== START ${name} $(date -Is) ====" | tee -a "${summary}" | tee "${slog}" - if build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ - "${repository}" "${gcc}" "${clang}" "${spec}" > >(tee -a "${slog}") 2>&1; then - echo "==== PASSED ${name} $(date -Is) ====" | tee -a "${summary}" | tee -a "${slog}" - else - echo "==== FAILED ${name} $(date -Is) ====" | tee -a "${summary}" | tee -a "${slog}" - echo "See ${slog}" | tee -a "${summary}" - if (( sweep_all )); then - echo "One or more Docker CI builds failed." >&2 - if ((${#failed_builders[@]})); then - echo "Retained builders for investigation:" >&2 - printf ' %s\n' "${failed_builders[@]}" >&2 - fi - exit 1 - fi - fi - else - build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ - "${repository}" "${gcc}" "${clang}" "${spec}" - fi + build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ + "${repository}" "${gcc}" "${clang}" "${spec}" done if (( dry_run )); then - [[ ${LOG_DIR} ]] && printf '\nLog directory: %s\n' "${LOG_DIR}" printf 'Dry run: nothing was built.\n' exit 0 fi @@ -327,7 +283,4 @@ if (( failed )); then exit 1 fi -if [[ ${summary} ]]; then - echo "==== ALL PASSED $(date -Is) ====" | tee -a "${summary}" -fi echo "All selected Docker CI builds passed." From b0921cd3e22ad5b49e687f8201df85ef7083b70f Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Mon, 17 Aug 2026 16:11:36 -0700 Subject: [PATCH 19/19] fixes for the new spack builds on LC --- scripts/setupLC-TPL-uberenv-helper.bash | 12 ++++++------ scripts/setupLC-TPL-uberenv.bash | 17 ++++++++++++++--- .../toss_4_x86_64_ib_cray/spack.yaml | 4 ++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 6ee12ca2..ff57bf7c 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -7,7 +7,7 @@ # By default, we will set permissions and reuse a previous build if available SET_PERMISSIONS=true CLEAN=false -: ${USER:=$(whoami)} +: "${USER:=$(whoami)}" # --- Argument Parsing --- INSTALL_DIR=$1 @@ -49,7 +49,7 @@ fi echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." # Execute the command to be run on the compute node via ssh -ssh -t "${USER}@${MACHINE}.llnl.gov" " +ssh -T -o ServerAliveInterval=60 -o ServerAliveCountMax=10 "${USER}@${MACHINE}.llnl.gov" " source /etc/profile && cd '${PWD}' && ${GET_A_NODE} bash -c ' @@ -59,14 +59,14 @@ ssh -t "${USER}@${MACHINE}.llnl.gov" " --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" \ --spack-env-name \"${CONFIG}_env\" \ \"\$@\" + uberenv_status=\$? echo \"End time: \$(date)\" + exit \${uberenv_status} ' " > "${LOG_FILE}" 2>&1 +UBERENV_STATUS=$? -## Check the last ten lines of the log file. -## A successful install should show up on one of the final lines. -tail -10 "$LOG_FILE" | grep -E "Successfully installed geos" > /dev/null -if [ $? -eq 0 ]; then +if [ "$UBERENV_STATUS" -eq 0 ]; then echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/." rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env" rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/.spack-db" diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index e884717b..dfee259e 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -88,7 +88,7 @@ function launch_jobs() { # Note: The max time allowed on the debug queue is 1h. If we need more, switch to pbatch case "$machine" in dane) - ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex" + ALLOC_CMD="srun -N 1 --exclusive -p pdebug -t 60 -A vortex" "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %%gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %%gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-14 "+docs %%clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" & @@ -96,7 +96,7 @@ function launch_jobs() { ;; matrix) - ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex" + ALLOC_CMD="srun -N 1 --exclusive -p pdebug -t 60 -A vortex" "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda ~uncrustify cuda_arch=90 %%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-14-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & @@ -131,7 +131,13 @@ done echo "All jobs launched. Waiting for completion..." # Note: Estimated completion time is ~90 minutes. # Check log files for unreported completion of jobs. -wait +BUILD_FAILED=false +mapfile -t JOB_PIDS < <(jobs -p) +for pid in "${JOB_PIDS[@]}"; do + if ! wait "$pid"; then + BUILD_FAILED=true + fi +done # --- Conditionally Set Final Permissions --- if [ "$SET_PERMISSIONS" = true ]; then @@ -144,4 +150,9 @@ else echo "Skipping final permission updates as requested." fi +if [ "$BUILD_FAILED" = true ]; then + echo "One or more builds failed. Check the corresponding log files." >&2 + exit 1 +fi + echo "Complete." diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 48af6ee7..a1cf268f 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -89,7 +89,7 @@ spack: - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - /opt/cray/pe/mpich/9.0.1/gtl/lib modules: - - PrgEnv-cray/8.6.0 + - PrgEnv-cray/8.7.0 - craype-x86-genoa - craype/2.7.35 - craype-network-ofi @@ -125,7 +125,7 @@ spack: - /opt/cray/pe/mpich/9.0.1/gtl/lib modules: - rocm/6.4.3 - - PrgEnv-cray/8.6.0 + - PrgEnv-cray/8.7.0 - craype-x86-genoa - craype/2.7.35 - craype-network-ofi