diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d2a12691 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +build +build-* +install +install-* +*_tpls +.rocm-ci-local.* +*.cmake +*.log +__pycache__ +vtk-9.6.2-local/ +tplMirror/*.tar.gz +tplMirror/chai-develop-060425/ diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 22333d22..9be42e6c 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -167,8 +167,15 @@ jobs: lfs: true - name: Set up Docker Buildx + if: matrix.RUNS_ON != 'streak2' uses: docker/setup-buildx-action@v4 + - name: Set up Docker Buildx + if: matrix.RUNS_ON == 'streak2' + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host + - name: Print environment run: printenv @@ -180,8 +187,38 @@ jobs: - name: Inject CA certificate into Docker build if: matrix.RUNS_ON == 'streak2' run: | + set -euo pipefail cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt - sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }} + dockerfile="${{ matrix.TPL_DOCKERFILE }}" + if grep -q 'RUN dnf clean all' "${dockerfile}"; then + sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}" + else + # Ubuntu CUDA jobs also run on streak2. Only COPY the host bundle; + # tpl-ubuntu.Dockerfile configures APT before its first update and + # installs the bundle after ca-certificates so the package postinst + # cannot wipe it. ENV covers pip/curl/git. + awk ' + BEGIN { inserted=0 } + { + print + if (!inserted && $0 ~ /^FROM /) { + print "COPY ca-bundle.crt /etc/ssl/certs/llnl-ca-bundle.crt" + print "ENV SSL_CERT_FILE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " REQUESTS_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " CURL_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " GIT_SSL_CAINFO=/etc/ssl/certs/llnl-ca-bundle.crt \\" + print " PIP_CERT=/etc/ssl/certs/llnl-ca-bundle.crt" + inserted=1 + } + } + END { + if (!inserted) { + print "ERROR: no FROM line in Dockerfile" > "/dev/stderr" + exit 1 + } + } + ' "${dockerfile}" > "${dockerfile}.ca" && mv "${dockerfile}.ca" "${dockerfile}" + fi - name: Run the docker build docker script env: @@ -196,6 +233,7 @@ jobs: COMMIT: ${{ github.sha }} BUILD_DIR: ${{ github.workspace }} DOCKER_TAG: ${{ github.event.number }}-${{ github.run_number }} + DOCKER_NETWORK: ${{ matrix.RUNS_ON == 'streak2' && 'host' || '' }} run: bash -x ./scripts/docker-build.sh - name: Login to DockerHub diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..11d6a5ee 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026" } diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,10 +47,12 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -589,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) @@ -737,7 +809,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +856,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +887,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +920,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +964,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +989,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/cmake/blt b/cmake/blt index e783e30f..f08f95e0 160000 --- a/cmake/blt +++ b/cmake/blt @@ -1 +1 @@ -Subproject commit e783e30f2823ee1a208f7f90741b41c1f5a08063 +Subproject commit f08f95e0f49b1d53db8be2dd577d1cdbfd9e9f90 diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/docker/openssl-non-fips.cnf b/docker/openssl-non-fips.cnf new file mode 100644 index 00000000..f7362003 --- /dev/null +++ b/docker/openssl-non-fips.cnf @@ -0,0 +1,14 @@ +openssl_conf = openssl_init + +[openssl_init] +providers = provider_sect +alg_section = algorithm_sect + +[provider_sect] +default = default_sect + +[default_sect] +activate = 1 + +[algorithm_sect] +default_properties = fips=no diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..856435c1 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf @@ -89,6 +90,9 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR ARG SPEC +# Keep Spack's package build parallelism bounded on self-hosted runners. The +# value remains overridable with --build-arg SPACK_BUILD_JOBS=... . +ARG SPACK_BUILD_JOBS=4 RUN dnf -y install \ tbb-devel \ @@ -146,6 +150,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k " ; \ else \ ./scripts/uberenv/uberenv.py \ @@ -153,6 +158,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k ; \ fi && \ rm -f lvarray* && \ diff --git a/docker/tpl-ubuntu.Dockerfile b/docker/tpl-ubuntu.Dockerfile index 367092a3..eff95832 100644 --- a/docker/tpl-ubuntu.Dockerfile +++ b/docker/tpl-ubuntu.Dockerfile @@ -22,6 +22,13 @@ FROM ${DOCKER_BASE_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain ARG SRC_DIR ARG CLANG_VERSION +# streak2 hosts can enable kernel FIPS mode even though this Ubuntu image has +# no FIPS provider. Use OpenSSL's default provider for the image's package +# downloads and source builds, matching the GEOS streak2 container setup. +COPY docker/openssl-non-fips.cnf /etc/ssl/openssl-non-fips.cnf +ENV OPENSSL_FORCE_FIPS_MODE=0 \ + OPENSSL_CONF=/etc/ssl/openssl-non-fips.cnf + # Install directory provided as a docker build argument; forwarded via ENV # (GEOSX_TPL_DIR is part of the image contract consumed by GEOS). ARG INSTALL_DIR @@ -30,7 +37,16 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR # Packages needed both for the TPL build and for the downstream GEOS build. # We avoid reinstalling anything already present in the base image (compiler, # cmake, doxygen, blas/lapack-dev when included by base PACKAGES, etc.). -RUN apt-get update && \ +# The streak2 workflow injects the LLNL CA bundle before this stage. Configure +# APT to use that bundle before the first update, since the base image does not +# yet trust the runner's MITM certificate and ca-certificates is installed below. +RUN if [ -f /etc/ssl/certs/llnl-ca-bundle.crt ]; then \ + mkdir -p /etc/apt/apt.conf.d && \ + printf '%s\n' \ + 'Acquire::https::CaInfo "/etc/ssl/certs/llnl-ca-bundle.crt";' \ + > /etc/apt/apt.conf.d/99-llnl-ca; \ + fi && \ + apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ apt-get install -y --no-install-recommends \ ca-certificates \ @@ -51,8 +67,25 @@ RUN apt-get update && \ lbzip2 \ bzip2 \ gnupg && \ + if [ -f /etc/ssl/certs/llnl-ca-bundle.crt ]; then \ + mkdir -p /usr/local/share/ca-certificates && \ + awk 'BEGIN {n=0} /-----BEGIN/ {n++; f=sprintf("/usr/local/share/ca-certificates/llnl-%03d.crt", n)} n>0 {print > f}' \ + /etc/ssl/certs/llnl-ca-bundle.crt && \ + update-ca-certificates ; \ + fi && \ apt-get clean && rm -rf /var/lib/apt/lists/* +# APT needs the runner bundle for its first HTTPS transaction. Once +# ca-certificates has rebuilt Ubuntu's trust store, use that merged bundle for +# all subsequent Python, curl, Git, and pip operations. Keeping the runner +# bundle selected globally can omit Ubuntu's public roots on some streak2 +# images. +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ + REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \ + GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt \ + PIP_CERT=/etc/ssl/certs/ca-certificates.crt + # Install clingo for Spack. Do not upgrade Ubuntu's Debian-managed pip in # place; Ubuntu 24.04's pip package cannot be uninstalled by pip. RUN python3 -m pip install --break-system-packages clingo @@ -101,6 +134,9 @@ FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR ARG SPEC +# Keep Spack's package build parallelism bounded on self-hosted runners. The +# value remains overridable with --build-arg SPACK_BUILD_JOBS=... . +ARG SPACK_BUILD_JOBS=4 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ @@ -138,6 +174,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ --spack-env-file=${GEOSX_SPACK_ENV_FILE} \ --project-json=${SRC_DIR}/.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k && \ rm -f lvarray* && \ cp *.cmake /spack-generated.cmake && \ diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 59218646..a8d3cddb 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,8 +1,13 @@ #!/bin/bash env -# We save memory for the docker context -echo .git > .dockerignore +# Keep .git out of the build context. Do not clobber an existing +# .dockerignore; local checkouts often ignore large untracked trees. +if [ -f .dockerignore ]; then + grep -qxF '.git' .dockerignore || echo '.git' >> .dockerignore +else + echo .git > .dockerignore +fi # Get uberenv submodule git submodule update --init scripts/uberenv @@ -30,10 +35,19 @@ echo "Docker base image is ${DOCKER_BASE_IMAGE}" EXTRA_BUILD_ARGS=() if [ -n "${GCC_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi +if [ -n "${SPACK_BUILD_JOBS:-}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS}"); fi BUILDER_ARGS=() if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi if [ "${DOCKER_LOAD:-0}" = 1 ]; then BUILDER_ARGS+=(--load); fi +if [ -n "${DOCKER_NETWORK:-}" ]; then BUILDER_ARGS+=(--network "${DOCKER_NETWORK}"); fi + +# Forward proxy settings into RUN steps (BuildKit special-cases these args). +for v in HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy; do + if [ -n "${!v:-}" ]; then + EXTRA_BUILD_ARGS+=(--build-arg "${v}=${!v}") + fi +done docker build --progress=plain \ "${BUILDER_ARGS[@]}" \ diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 35ca6c48..301ccfdd 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -27,6 +27,8 @@ DOCKER_TAG=${DOCKER_TAG:-local} COMMIT=${COMMIT:-$(git rev-parse HEAD 2>/dev/null || echo HEAD)} INSTALL_DIR_ROOT=${INSTALL_DIR_ROOT:-/opt/GEOS} HOST_CONFIG=${HOST_CONFIG:-host-configs/environment.cmake} +BUILDX_CPUS=${BUILDX_CPUS:-} +BUILDX_MEMORY=${BUILDX_MEMORY:-} keep_artifacts=0 dry_run=0 @@ -63,6 +65,8 @@ Build settings (each also readable from the environment): --base-sha SHA DOCKER_BASE_IMAGE_SHA [${DOCKER_BASE_IMAGE_SHA}] --install-dir DIR INSTALL_DIR_ROOT [${INSTALL_DIR_ROOT}] --host-config F HOST_CONFIG [${HOST_CONFIG}] + --cpus N BUILDX_CPUS [${BUILDX_CPUS:-unset}] + --memory SIZE BUILDX_MEMORY [${BUILDX_MEMORY:-unset}] Other: --keep Keep the image and buildx builder after a successful build @@ -128,6 +132,8 @@ while (( $# )); do --base-sha) DOCKER_BASE_IMAGE_SHA=$2; shift 2 ;; --install-dir) INSTALL_DIR_ROOT=$2; shift 2 ;; --host-config) HOST_CONFIG=$2; shift 2 ;; + --cpus) BUILDX_CPUS=$2; shift 2 ;; + --memory) BUILDX_MEMORY=$2; shift 2 ;; --keep) keep_artifacts=1; shift ;; --dry-run) dry_run=1; shift ;; -h|--help) usage; exit 0 ;; @@ -188,13 +194,23 @@ build() { printf ' commit %s\n' "${COMMIT}" printf ' install dir %s\n' "${INSTALL_DIR_ROOT}" printf ' host config %s\n' "${HOST_CONFIG}" + [[ ${BUILDX_CPUS} ]] && printf ' cpus %s\n' "${BUILDX_CPUS}" + [[ ${BUILDX_MEMORY} ]] && printf ' memory %s\n' "${BUILDX_MEMORY}" return fi - if ! docker buildx create --name "${builder}" --driver docker-container --bootstrap >/dev/null; then + create_args=(--name "${builder}" --driver docker-container) + if [[ ${BUILDX_CPUS} ]]; then + # docker-container has no "cpus" option; CFS quota 100000us * N == N CPUs. + create_args+=(--driver-opt "cpu-period=100000") + create_args+=(--driver-opt "cpu-quota=$((BUILDX_CPUS * 100000))") + create_args+=(--driver-opt "cpuset-cpus=0-$((BUILDX_CPUS - 1))") + fi + [[ ${BUILDX_MEMORY} ]] && create_args+=(--driver-opt "memory=${BUILDX_MEMORY}") + if ! docker buildx create "${create_args[@]}" --bootstrap >/dev/null; then printf 'FAILED TO CREATE BUILDER: %s\n' "${name}" >&2 failed=1 - return + return 1 fi if ! TPL_DOCKERFILE=${dockerfile} \ @@ -214,7 +230,7 @@ build() { printf 'FAILED: %s (builder retained: %s)\n' "${name}" "${builder}" >&2 failed=1 failed_builders+=("${builder}") - return + return 1 fi if (( keep_artifacts )); then @@ -236,9 +252,9 @@ build() { if (( cleanup_failed )); then failed=1 failed_builders+=("${builder}") - else - printf 'PASSED: %s (image and build cache removed)\n' "${name}" + return 1 fi + printf 'PASSED: %s (image and build cache removed)\n' "${name}" } printf 'Selected %d of %d job(s).\n' "${#indices[@]}" "${#MATRIX[@]}" @@ -248,12 +264,13 @@ for idx in "${indices[@]}"; do row=$((row + 1)) IFS='|' read -r name dockerfile base_tag base_repository repository gcc clang spec \ <<<"${MATRIX[${idx}]}" + build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ "${repository}" "${gcc}" "${clang}" "${spec}" done if (( dry_run )); then - printf '\nDry run: nothing was built.\n' + printf 'Dry run: nothing was built.\n' exit 0 fi diff --git a/scripts/setupLC-TPL-uberenv-helper.bash b/scripts/setupLC-TPL-uberenv-helper.bash index 6ee12ca2..ff57bf7c 100755 --- a/scripts/setupLC-TPL-uberenv-helper.bash +++ b/scripts/setupLC-TPL-uberenv-helper.bash @@ -7,7 +7,7 @@ # By default, we will set permissions and reuse a previous build if available SET_PERMISSIONS=true CLEAN=false -: ${USER:=$(whoami)} +: "${USER:=$(whoami)}" # --- Argument Parsing --- INSTALL_DIR=$1 @@ -49,7 +49,7 @@ fi echo "Building the TPLs on $MACHINE for $COMPILER to be installed at $INSTALL_DIR. Progress will be written to $LOG_FILE." # Execute the command to be run on the compute node via ssh -ssh -t "${USER}@${MACHINE}.llnl.gov" " +ssh -T -o ServerAliveInterval=60 -o ServerAliveCountMax=10 "${USER}@${MACHINE}.llnl.gov" " source /etc/profile && cd '${PWD}' && ${GET_A_NODE} bash -c ' @@ -59,14 +59,14 @@ ssh -t "${USER}@${MACHINE}.llnl.gov" " --prefix \"${INSTALL_DIR}/${CONFIG}_tpls\" \ --spack-env-name \"${CONFIG}_env\" \ \"\$@\" + uberenv_status=\$? echo \"End time: \$(date)\" + exit \${uberenv_status} ' " > "${LOG_FILE}" 2>&1 +UBERENV_STATUS=$? -## Check the last ten lines of the log file. -## A successful install should show up on one of the final lines. -tail -10 "$LOG_FILE" | grep -E "Successfully installed geos" > /dev/null -if [ $? -eq 0 ]; then +if [ "$UBERENV_STATUS" -eq 0 ]; then echo "Cleanup extra build files at ${INSTALL_DIR}/${CONFIG}_tpls/." rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/${CONFIG}_env" rm -rf "${INSTALL_DIR}/${CONFIG}_tpls/.spack-db" diff --git a/scripts/setupLC-TPL-uberenv.bash b/scripts/setupLC-TPL-uberenv.bash index e884717b..dfee259e 100755 --- a/scripts/setupLC-TPL-uberenv.bash +++ b/scripts/setupLC-TPL-uberenv.bash @@ -88,7 +88,7 @@ function launch_jobs() { # Note: The max time allowed on the debug queue is 1h. If we need more, switch to pbatch case "$machine" in dane) - ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex" + ALLOC_CMD="srun -N 1 --exclusive -p pdebug -t 60 -A vortex" "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-12 "+docs %%gcc-12 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane gcc-13 "+docs %%gcc-13 ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" dane llvm-14 "+docs %%clang-14 ${COMMON}" "${ALLOC_CMD}" "$@" & @@ -96,7 +96,7 @@ function launch_jobs() { ;; matrix) - ALLOC_CMD="srun -N 1 --exclusive -t 60 -A vortex" + ALLOC_CMD="srun -N 1 --exclusive -p pdebug -t 60 -A vortex" "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-12-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%gcc-12 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix gcc-13-cuda-12.9 "+cuda ~uncrustify cuda_arch=90 %%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & "${UBERENV_HELPER}" "$INSTALL_DIR" matrix llvm-14-cuda-12.6 "+cuda ~uncrustify cuda_arch=90 %%clang-14 ^cuda@12.6.0+allow-unsupported-compilers ${COMMON}" "${ALLOC_CMD}" "$@" & @@ -131,7 +131,13 @@ done echo "All jobs launched. Waiting for completion..." # Note: Estimated completion time is ~90 minutes. # Check log files for unreported completion of jobs. -wait +BUILD_FAILED=false +mapfile -t JOB_PIDS < <(jobs -p) +for pid in "${JOB_PIDS[@]}"; do + if ! wait "$pid"; then + BUILD_FAILED=true + fi +done # --- Conditionally Set Final Permissions --- if [ "$SET_PERMISSIONS" = true ]; then @@ -144,4 +150,9 @@ else echo "Skipping final permission updates as requested." fi +if [ "$BUILD_FAILED" = true ]; then + echo "One or more builds failed. Check the corresponding log files." >&2 + exit 1 +fi + echo "Complete." diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false @@ -231,6 +233,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..a1cf268f 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false @@ -87,7 +89,7 @@ spack: - /opt/cray/pe/mpich/9.0.1/ofi/crayclang/20.0/lib - /opt/cray/pe/mpich/9.0.1/gtl/lib modules: - - PrgEnv-cray/8.6.0 + - PrgEnv-cray/8.7.0 - craype-x86-genoa - craype/2.7.35 - craype-network-ofi @@ -123,7 +125,7 @@ spack: - /opt/cray/pe/mpich/9.0.1/gtl/lib modules: - rocm/6.4.3 - - PrgEnv-cray/8.6.0 + - PrgEnv-cray/8.7.0 - craype-x86-genoa - craype/2.7.35 - craype-network-ofi diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch b/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch new file mode 100644 index 00000000..1fd49630 --- /dev/null +++ b/scripts/spack_packages/packages/fmt/nvcc-clang-bitint.patch @@ -0,0 +1,17 @@ +# nvcc with a Clang host compiler cannot parse Clang's _BitInt extension. +# Upstream: https://github.com/fmtlib/fmt/pull/4564 +--- a/include/fmt/base.h ++++ b/include/fmt/base.h +@@ -420,7 +420,11 @@ inline auto map(uint128_opt) -> monostate { return {}; } + #endif + + #ifndef FMT_USE_BITINT +-# define FMT_USE_BITINT (FMT_CLANG_VERSION >= 1500) ++# if FMT_CLANG_VERSION >= 1500 && !defined(__CUDACC__) ++# define FMT_USE_BITINT 1 ++# else ++# define FMT_USE_BITINT 0 ++# endif + #endif + + #if FMT_USE_BITINT diff --git a/scripts/spack_packages/packages/fmt/package.py b/scripts/spack_packages/packages/fmt/package.py new file mode 100644 index 00000000..ecf2570f --- /dev/null +++ b/scripts/spack_packages/packages/fmt/package.py @@ -0,0 +1,8 @@ +from spack.package import * +from spack_repo.builtin.packages.fmt.package import Fmt as BuiltinFmt + + +class Fmt(BuiltinFmt): + # nvcc + clang host compiler cannot parse Clang's _BitInt extension. + # https://github.com/fmtlib/fmt/pull/4564 (merged after fmt 11.2.0) + patch("nvcc-clang-bitint.patch", when="@11.1:11.2.0") diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..5bd2a7ae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -64,7 +64,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') @@ -356,9 +356,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -375,16 +372,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): + if sys_type in ('toss_4_x86_64_ib_cray'): cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) else: @@ -470,10 +458,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -568,12 +552,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_MKL', True)) cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -689,9 +667,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -769,9 +744,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80))