diff --git a/CMakeLists.txt b/CMakeLists.txt index ab7fe24a..c75ff239 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -259,8 +259,9 @@ ExternalProject_Add( hdf5 URL_HASH SHA256=${HDF5_URL_HASH} PREFIX ${PROJECT_BINARY_DIR}/hdf5 INSTALL_DIR ${HDF5_DIR} - BUILD_COMMAND make -j ${NUM_PROC} - INSTALL_COMMAND make install + BUILD_COMMAND ${TPL_BUILD_COMMAND} + INSTALL_COMMAND ${TPL_INSTALL_COMMAND} + CMAKE_GENERATOR ${TPL_GENERATOR} CMAKE_ARGS -D CMAKE_C_COMPILER:STRING=${HDF5_C_COMPILER} -D CMAKE_CXX_COMPILER:STRING=${HDF5_CXX_COMPILER} -D CMAKE_C_FLAGS:STRING=${C_FLAGS_NO_WARNINGS} @@ -396,6 +397,19 @@ set(RAJA_URL_HASH "2774a569f19a7d8e37d1e166d197c828c6b614e7f8ef6d16ccbfb144b8284 message(STATUS "Building RAJA found at ${RAJA_URL}") +# RDNA GPUs use 32-lane wavefronts, while the CDNA GPUs targeted by the +# default RAJA configuration use 64-lane wavefronts. Keep this override +# configurable so callers can select the wave size explicitly when needed. +if( ENABLE_HIP ) + set( RAJA_HIP_WAVESIZE_DEFAULT 64 ) + if( CMAKE_HIP_ARCHITECTURES MATCHES "(^|;)gfx(10|11)" ) + set( RAJA_HIP_WAVESIZE_DEFAULT 32 ) + endif() + set( RAJA_HIP_WAVESIZE "${RAJA_HIP_WAVESIZE_DEFAULT}" CACHE STRING + "RAJA HIP wavefront size (32 for RDNA, 64 for CDNA)" ) + message( STATUS "RAJA HIP wavefront size = ${RAJA_HIP_WAVESIZE}" ) +endif() + # RAJA's vectorization layer does not compile with the ROCm toolchain. if( ENABLE_HIP ) set( RAJA_ENABLE_VECTORIZATION OFF ) @@ -421,6 +435,7 @@ ExternalProject_Add( raja -D RAJA_ENABLE_NV_TOOLS_EXT:BOOL=${ENABLE_CUDA_NVTOOLSEXT} -D RAJA_ENABLE_ROCTX:BOOL=${ENABLE_ROCTX} -D CMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER} + -D CMAKE_CUDA_HOST_COMPILER:FILEPATH=${CMAKE_CUDA_HOST_COMPILER} -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CUB_DIR:PATH=${CUB_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} @@ -432,6 +447,7 @@ ExternalProject_Add( raja -D hip_DIR=${HIP_ROOT}/lib/cmake/hip -D ROCPRIM_DIR=${HIP_ROOT} -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} + -D RAJA_HIP_WAVESIZE:STRING=${RAJA_HIP_WAVESIZE} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} @@ -439,6 +455,7 @@ ExternalProject_Add( raja -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} + -D RAJA_ENABLE_EXERCISES:BOOL=OFF ) list(APPEND build_list raja ) @@ -470,6 +487,7 @@ ExternalProject_Add( chai -D ENABLE_CUDA:BOOL=${ENABLE_CUDA} -D CHAI_ENABLE_CUDA:BOOL=${ENABLE_CUDA} -D CMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER} + -D CMAKE_CUDA_HOST_COMPILER:FILEPATH=${CMAKE_CUDA_HOST_COMPILER} -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} @@ -851,7 +869,7 @@ list(APPEND build_list metis ) if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "https://github.com/xiaoyeli/superlu_dist/archive/0f6efc3.tar.gz") - set(SUPERLU_URL_HASH "49cd47da22fe72a37011e8dfaf94bd63ad8d61cc8144eac7a571bcbd0a2edf58a") + set(SUPERLU_URL_HASH "49cd47da22fe72a37011e8dfaf94bd63ad8d61cc8144eac7a571bcbd0a2edf58") if ( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 16.0 ) set(SUPERLU_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS}") @@ -962,6 +980,23 @@ if( ENABLE_HYPRE ) --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + if( NOT "${CMAKE_CUDA_FLAGS}" STREQUAL "" ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-extra-CUFLAGS=\"${CMAKE_CUDA_FLAGS}\"" ) + endif() + + if( ENABLE_MPI ) + if( MPI_C_INCLUDE_DIRS ) + list( GET MPI_C_INCLUDE_DIRS 0 HYPRE_MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) + get_filename_component( HYPRE_MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( HYPRE_MPI_PREFIX_DIR ${HYPRE_MPI_BIN_DIR} DIRECTORY ) + set( HYPRE_MPI_INCLUDE_DIR "${HYPRE_MPI_PREFIX_DIR}/include" ) + endif() + if( HYPRE_MPI_INCLUDE_DIR ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --with-MPI-include=${HYPRE_MPI_INCLUDE_DIR}" ) + endif() + endif() + if( ENABLE_HYPRE_GPU_AWARE_MPI ) set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) endif() @@ -1251,9 +1286,10 @@ if( ENABLE_SCOTCH ) URL ${SCOTCH_URL} URL_HASH SHA256=${SCOTCH_URL_HASH} INSTALL_DIR ${SCOTCH_DIR} - BUILD_COMMAND ${TPL_BUILD_COMMAND} - INSTALL_COMMAND ${TPL_INSTALL_COMMAND} - CMAKE_GENERATOR ${TPL_GENERATOR} + BINARY_DIR ${PROJECT_BINARY_DIR}/scotch/src/scotch-make-build + BUILD_COMMAND make -j ${NUM_PROC} + INSTALL_COMMAND make install + CMAKE_GENERATOR "Unix Makefiles" CMAKE_ARGS -D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -D BUILD_PTSCOTCH:BOOL=ON -D BUILD_LIBSCOTCHMETIS:BOOL=OFF diff --git a/scripts/spack_configs/maple/spack.yaml b/scripts/spack_configs/maple/spack.yaml deleted file mode 100644 index 82594316..00000000 --- a/scripts/spack_configs/maple/spack.yaml +++ /dev/null @@ -1,161 +0,0 @@ -# This is a Spack Environment file for Maple. -# -# It describes a set of packages to be installed, along with -# configuration settings. -# -# Run command from the top-level of the repository: -# ./scripts/uberenv/uberenv.py \ -# --spec "%clang@17.0.4 +cuda~pygeosx~docs cuda_arch=90 ^cuda@12.4.131+allow-unsupported-compilers" \ -# --spack-env-file=scripts/spack_configs/maple/spack.yaml \ -# --project-json=.uberenv_config.json \ -# --prefix ${GEOS_TPL_DIR} -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage: - - $spack/../build_stage - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - ############# - # COMPILERS # - ############# - compilers: - - compiler: - spec: clang@17.0.4 - paths: - cc: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/llvm-17.0.4-24xg7dvliazr7dqlcimhsgpog3nvdiie/bin/clang - cxx: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/llvm-17.0.4-24xg7dvliazr7dqlcimhsgpog3nvdiie/bin/clang++ - f77: /usr/bin/gfortran - fc: /usr/bin/gfortran - flags: - cflags: - cxxflags: - operating_system: rocky9 - target: aarch64 - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] - - ############# - # PACKAGES # - ############# - packages: - all: - target: [neoverse_n1] - compiler: [clang] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] - - #### - # make sure spack doesn't rebuild mpi and mathpresso libs - mpi: - buildable: false - - mathpresso: - buildable: false - - #### - # spec of spack packages to reuse - bison: - externals: - - spec: bison@3.8.2 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/bison-3.8.2-mkim26leceonxvdvlpukc4y65kkaa4j3/ - buildable: false - gmp: - externals: - - spec: gmp@6.2.1 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/gmp-6.2.1-h7dxik46ba5ebtbvpdtfqm7kanbxokzm - buildable: false - mpfr: - externals: - - spec: mpfr@4.2.0 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/mpfr-4.2.0-i6bud7m6k6lmduamd6h5gw62xdf2wytf - buildable: false - libiconv: - externals: - - spec: libiconv@1.17 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/libiconv-1.17-m3y2ti2b4r73c2w2d4tbln6bdlozblq6 - buildable: false - perl: - externals: - - spec: perl@5.38.0 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/perl-5.38.0-vjg3pfiwld5tk2vu22ji2f2fgrgxrekn - buildable: false - berkeley-db: - externals: - - spec: berkeley-db@18.1.40 - prefix: /hrtc/apps/devtools/spack/GHOPPER/linux-rocky9-neoverse_n1/gcc-11.4.1/berkeley-db-18.1.40-nqcmpq7kkd5nqo7x6dd7tvcahv7fk5nd - buildable: false - cmake: - externals: - - spec: cmake@3.29.6 - prefix: /hrtc/apps/devtools/spack/MAPLE/linux-rocky9-neoverse_v2/gcc-12.2.0/cmake-3.29.6-yl7bm5nfishcx3d6dlkqfj7pc2muvowe - buildable: false - - #### - # spec of system packages to reuse - # bin - flex: - externals: - - spec: flex@2.6.4+lex - prefix: /usr - buildable: false - git: - externals: - - spec: git@2.39.3~tcltk - prefix: /usr - buildable: false - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - buildable: false - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - buildable: false - python: - buildable: False - externals: - - spec: python@3.9.18 - prefix: /usr - #lib - cuda: - externals: - - spec: cuda@12.4.131 - prefix: /hrtc/apps/cuda/12.4.131/aarch64/rocky9/ - buildable: false - openmpi: - externals: - - spec: openmpi@5.0.5 - prefix: /hrtc/apps/mpi/openmpi/aarch64/rocky9/5.0.5/llvm/17.0.4/cuda.12.4 - buildable: false - pkgconf: - externals: - - spec: pkgconf@3.0.0 - prefix: /usr - buildable: false - readline: - externals: - - spec: readline@8.1 - prefix: /usr - buildable: false - - diff --git a/scripts/spack_configs/perlmutter/spack.yaml b/scripts/spack_configs/perlmutter/spack.yaml new file mode 100644 index 00000000..2b0d7b0b --- /dev/null +++ b/scripts/spack_configs/perlmutter/spack.yaml @@ -0,0 +1,396 @@ +#------------------------------------------------------------------------------------------------------------ +# SPDX-License-Identifier: LGPL-2.1-only +# +# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC +# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University +# Copyright (c) 2018-2020 TotalEnergies +# Copyright (c) 2019- GEOSX Contributors +# All rights reserved +# +# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. +#------------------------------------------------------------------------------------------------------------ +# +# NERSC Perlmutter GPU (A100 / cuda_arch=80, Milan / znver3). +# Uses the default CPE 26.03 stack: gcc-native/14 or nvidia/26.5, cray-mpich/9.1.0, +# cray-libsci/26.03.0, and cudatoolkit/13.2. +# +# CFS install (private dir under project mXXXX, no nested quota): +# mkdir -p "$CFS/mXXXX/${USER}/geos-tpl" +# chmod 700 "$CFS/mXXXX/${USER}" +# +# Before uberenv, on a GPU allocation (-C gpu -A mXXXX_g): +# module unload darshan +# module load python/3.12-26.1.0 # /usr/bin/python3 is 3.6; too old for Spack +# # Stage on a FS that supports flock. CFS is DVS-mounted on compute nodes +# # (error 524 / autom4te). Prefer node-local NVMe; PSCRATCH is second choice. +# unset TMPDIR +# export TMPDIR=/tmp +# mkdir -p "$TMPDIR" +# +# GNU (recommended): +# module load PrgEnv-gnu/8.7.0 gcc-native/14 craype-accel-nvidia80 cudatoolkit/13.2 cmake/3.30.2 +# python3 ./scripts/uberenv/uberenv.py \ +# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%gcc-14 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja" \ +# --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ +# --prefix "$CFS/mXXXX/${USER}/geos-tpl" +# +# NVHPC (do not load cudatoolkit; it conflicts with the nvidia module). +# Keep "%%" so C++ TPLs (vtk, raja, hypre) stay nvc. %% is a preference; +# packages.yaml require %gcc is a hard exception for GNU C that nvc 26.5 ICEs. +# Use a separate env name so uberenv does not delete the gcc spack_env. +# module load PrgEnv-nvidia/8.7.0 nvidia/26.5 craype-accel-nvidia80 cmake/3.30.2 +# python3 ./scripts/uberenv/uberenv.py \ +# --spec="+cuda~uncrustify~pygeosx~docs~trilinos~petsc cuda_arch=80 %%nvhpc-26 ^cuda@13.2.0+allow-unsupported-compilers ^vtk generator=ninja ^python%gcc-14" \ +# --spack-env-file=scripts/spack_configs/perlmutter/spack.yaml \ +# --spack-env-name=spack_env_nvhpc \ +# --prefix "$CFS/mXXXX/${USER}/geos-tpl" +# +# NOTE: Use "%%" to propagate the compiler choice to dependency libraries. +# +# If the NVHPC cray-libsci prefix is wrong, run: +# module load PrgEnv-nvidia/8.7.0 nvidia/26.5 +# echo "$CRAY_LIBSCI_PREFIX_DIR" +# +# Resume a live env after gmp/superlu-dist failures (do not copy this file +# over spack_env/spack.yaml; only patch config:build_stage, then concretize): +# export TMPDIR=/tmp +# ENV="$CFS/mXXXX/${USER}/geos/tpl-install/spack_env" +# SPACK="$CFS/mXXXX/${USER}/geos/tpl-install/spack/bin/spack" +# "$SPACK" -D "$ENV" config add 'config:build_stage:[$tempdir/$user/spack-stage, $spack/../build_stage]' +# "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' +# "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig +# +# NVHPC env. Copy overlays into the geos-tpls checkout (zstd/libmd are not +# on GitHub until committed). Same hashes as a previous fail mean the lockfile +# was not re-concretized — config add on a live env does not restage vtk. +# export TMPDIR=/tmp +# ENV="$CFS/mXXXX/${USER}/geos/tpl-install/spack_env_nvhpc" +# SPACK="$CFS/mXXXX/${USER}/geos/tpl-install/spack/bin/spack" +# "$SPACK" -D "$ENV" config add 'packages:blas:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:lapack:require:[cray-libsci]' +# "$SPACK" -D "$ENV" config add 'packages:python:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libffi:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libiconv:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:libmd:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:zstd:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:gettext:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:expat:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:gdbm:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:ncurses:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:readline:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:sqlite:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:util-linux-uuid:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'packages:texinfo:require:["%gcc"]' +# "$SPACK" -D "$ENV" config add 'concretizer:unify:when_possible' +# # Do not use --fresh on this env (Spack 1.2.2). If vtk stays on the old +# # lockfile hash, back up and drop spack.lock, then concretize -f. +# "$SPACK" -D "$ENV" concretize -f +# "$SPACK" -D "$ENV" spec | grep -E 'python|libffi|libiconv|libmd |zstd|vtk' +# # python/libffi/libiconv/libmd/zstd must be %gcc@14.3.0 under gcc-14.3.0/ +# # (new hashes). vtk stays %nvhpc@26.5 with a NEW hash (surface-nets patch). +# "$SPACK" -D "$ENV" uninstall --yes /fnqnbm7 /4e3655z /auvwsmg /jdknojp /xqymogj || true +# "$SPACK" -D "$ENV" install --keep-stage -u lvarray_hostconfig +# + +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + # Do not stage on CFS: DVS has no flock, so gmp/mpfr autoreconf fails. + # $tempdir follows TMPDIR (/tmp on the GPU node, or $PSCRATCH if you set it). + build_stage: + - $tempdir/$user/spack-stage + - $spack/../build_stage + build_jobs: 32 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new + + # Allow python%gcc next to %%nvhpc-26. %% is a preference; require wins. + concretizer: + unify: when_possible + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + toolchains: + gcc-14: + - spec: '%c=gcc@14.3.0' + when: '%c' + - spec: '%cxx=gcc@14.3.0' + when: '%cxx' + - spec: '%fortran=gcc@14.3.0' + when: '%fortran' + - spec: '%cray-mpich@9.1.0_gnu' + when: '%mpi' + nvhpc-26: + - spec: '%c=nvhpc@26.5' + when: '%c' + - spec: '%cxx=nvhpc@26.5' + when: '%cxx' + - spec: '%fortran=nvhpc@26.5' + when: '%fortran' + - spec: '%cray-mpich@9.1.0_nvhpc' + when: '%mpi' + + packages: + all: + target: [znver3] + providers: + blas: [cray-libsci] + lapack: [cray-libsci] + mpi: [cray-mpich] + zlib-api: [zlib] + pkgconfig: [pkg-config] + + # nvhpc also provides blas/lapack (default +blas+lapack) and the concretizer + # will pick it over cray-libsci because the compiler is already in the DAG. + # Force LibSci; otherwise TPL_BLAS_LIBRARIES and SuiteSparse BLAS= are empty. + blas: + require: + - cray-libsci + lapack: + require: + - cray-libsci + + # nvc 26.5 ICEs GNU C (libffi .S, libiconv error.h, libmd aliases, zstd -MF, + # gettext nvhpc-export-symbols.patch vs 1.0). Hard exception to %%nvhpc-26. + python: + require: + - "%gcc" + libffi: + require: + - "%gcc" + libiconv: + require: + - "%gcc" + libmd: + require: + - "%gcc" + zstd: + require: + - "%gcc" + gettext: + require: + - "%gcc" + expat: + require: + - "%gcc" + gdbm: + require: + - "%gcc" + ncurses: + require: + - "%gcc" + readline: + require: + - "%gcc" + sqlite: + require: + - "%gcc" + util-linux-uuid: + require: + - "%gcc" + texinfo: + require: + - "%gcc" + + gcc: + buildable: false + externals: + - spec: gcc@14.3.0 languages:=c,c++,fortran + prefix: /opt/cray/pe/gcc-native/14 + extra_attributes: + compilers: + c: /opt/cray/pe/gcc-native/14/bin/gcc + cxx: /opt/cray/pe/gcc-native/14/bin/g++ + fortran: /opt/cray/pe/gcc-native/14/bin/gfortran + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.1.0/ofi/gnu/12.3/lib + - /opt/cray/pe/mpich/9.1.0/gtl/lib + - /opt/cray/pe/libsci/26.03.0/GNU/12/x86_64/lib + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64 + modules: + - PrgEnv-gnu/8.7.0 + - gcc-native/14 + - craype-x86-milan + - craype/2.7.36 + - craype-network-ofi + - craype-accel-nvidia80 + - cray-mpich/9.1.0 + - cray-libsci/26.03.0 + - cudatoolkit/13.2 + - cray-dsmml/0.3.1 + - libfabric/1.22.0 + - xpmem/2.11.5-1.3_g73ade43320bc + - cpe/26.03 + - cmake/3.30.2 + - perftools-base/26.03.0 + + nvhpc: + buildable: false + externals: + - spec: nvhpc@26.5 ~blas ~lapack + prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5 + extra_attributes: + compilers: + c: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvc + cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvc++ + fortran: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/bin/nvfortran + flags: {} + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: + - /opt/cray/pe/mpich/9.1.0/ofi/nvidia/23.3/lib + - /opt/cray/pe/mpich/9.1.0/gtl/lib + - /opt/cray/pe/libsci/26.03.0/NVIDIA/23/x86_64/lib + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64 + - /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/compilers/lib + modules: + - PrgEnv-nvidia/8.7.0 + - nvidia/26.5 + - craype-x86-milan + - craype/2.7.36 + - craype-network-ofi + - craype-accel-nvidia80 + - cray-mpich/9.1.0 + - cray-libsci/26.03.0 + - cray-dsmml/0.3.1 + - libfabric/1.22.0 + - xpmem/2.11.5-1.3_g73ade43320bc + - cpe/26.03 + - cmake/3.30.2 + - perftools-base/26.03.0 + + cray-mpich: + buildable: false + externals: + - spec: cray-mpich@9.1.0_gnu %gcc@14.3.0 + prefix: /opt/cray/pe/mpich/9.1.0/ofi/gnu/12.3 + - spec: cray-mpich@9.1.0_nvhpc %nvhpc@26.5 + prefix: /opt/cray/pe/mpich/9.1.0/ofi/nvidia/23.3 + + cray-libsci: + buildable: false + externals: + - spec: cray-libsci@26.03.0 %gcc@14.3.0 + prefix: /opt/cray/pe/libsci/26.03.0/GNU/12/x86_64 + # ABI dir matches cray-mpich nvidia/23.3. Confirm with $CRAY_LIBSCI_PREFIX_DIR. + - spec: cray-libsci@26.03.0 %nvhpc@26.5 + prefix: /opt/cray/pe/libsci/26.03.0/NVIDIA/23/x86_64 + + cuda: + buildable: false + externals: + - spec: cuda@13.2.0 +allow-unsupported-compilers + prefix: /opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2 + + cmake: + buildable: false + externals: + - spec: cmake@3.30.2 + prefix: /global/common/software/nersc9/cmake/3.30.2 + modules: + - cmake/3.30.2 + + ninja: + buildable: false + externals: + - spec: ninja@1.10.0 + prefix: /usr + + glibc: + buildable: false + externals: + - spec: glibc@2.38 + prefix: /usr + + autoconf: + buildable: false + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + buildable: false + externals: + - spec: automake@1.15.1 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + buildable: false + externals: + - spec: perl@5.26.1 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + gmake: + buildable: false + externals: + - spec: gmake@4.2.1 + prefix: /usr + git: + buildable: false + externals: + - spec: git@2.51.0 ~tcltk + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.34 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + curl: + buildable: false + externals: + - spec: curl@8.14.1 + prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.2.13 + prefix: /usr + binutils: + buildable: false + externals: + - spec: binutils@2.45 + prefix: /usr + # System bison is 3.0.4 (too old for scotch). Let Spack build bison. + # Do not mark flex external; scotch has issues with some system flex. diff --git a/scripts/spack_configs/pine/spack.yaml b/scripts/spack_configs/pine/spack.yaml deleted file mode 100644 index 6141aec7..00000000 --- a/scripts/spack_configs/pine/spack.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# This is a Spack Environment file for Pine (Spack 1.0.2) -# Run command from the top-level of the repository: -# ./scripts/uberenv/uberenv.py \ -# --spec "~openmp~pygeosx~docs %gcc-11" \ -# --spack-env-file=scripts/spack_configs/pine/spack.yaml \ -# --project-json=.uberenv_config.json \ -# --prefix ${GEOS_TPL_DIR} - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage: - - $spack/../build_stage - build_jobs: 16 - - view: false - - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - gcc-11: - - spec: '%c=gcc@11.4.1' - when: '%c' - - spec: '%cxx=gcc@11.4.1' - when: '%cxx' - - spec: '%fortran=gcc@11.4.1' - when: '%fortran' - - spec: '%openmpi@5.0.5' - when: '%mpi' - - packages: - all: - target: [zen4] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] - zlib-api: [zlib] - pkgconfig: [pkgconf] - - # Force Spack to build openblas, not use system version - openblas: - buildable: true - require: "@0.3.27 threads=none" - - # Disable unwanted compilers - nvhpc: - buildable: false - intel: - buildable: false - oneapi: - buildable: false - cce: - buildable: false - llvm-amdgpu: - buildable: false - - gcc: - externals: - - spec: gcc@=11.4.1 - prefix: /usr - extra_attributes: - compilers: - c: /usr/bin/gcc - cxx: /usr/bin/g++ - fortran: /usr/bin/gfortran - flags: - cflags: -march=native -mtune=native - cxxflags: -march=native -mtune=native - environment: {} - extra_rpaths: [] - buildable: false - - # libgcc_s.so.1 is missing in /usr/lib/gcc/x86_64-redhat-linux/11 so removing it for now - # gcc-runtime: - # buildable: false - # externals: - # - spec: gcc-runtime@11.4.1 - # prefix: /usr/lib/gcc/x86_64-redhat-linux/11 - - gmp: - externals: - - spec: gmp@6.3.0 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/gmp-6.3.0-q4glkxa77fd4rejkrrsuydut2c222fkm - buildable: false - - mpfr: - externals: - - spec: mpfr@4.2.0 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/mpfr-4.2.1-eaxjflobjpilpwfe4aalusexhcxhptrf - buildable: false - - libiconv: - externals: - - spec: libiconv@1.17 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/libiconv-1.17-crgtppb7hj54qwkl3hfc6ejyu34gsa4d - buildable: false - - perl: - externals: - - spec: perl@5.38.2 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/perl-5.38.2-wxq42supt6zs3f63ajjtp6423q6teyot - buildable: false - - berkeley-db: - externals: - - spec: berkeley-db@18.1.40 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/berkeley-db-18.1.40-nixwddyxlb523z3aly4b7bje4rfft3me - buildable: false - - cmake: - externals: - - spec: cmake@3.29.6 - prefix: /hrtc/apps/devtools/spack/PINE/linux-rocky9-zen4/gcc-11.4.1/cmake-3.29.6-i23jd7kq7lyqvrpsguhznfhoy4mso6up - buildable: false - - git: - externals: - - spec: git@2.39.3~tcltk - prefix: /usr - buildable: false - - gmake: - externals: - - spec: gmake@4.3 - prefix: /usr - buildable: false - - m4: - externals: - - spec: m4@1.4.19 - prefix: /usr - buildable: false - - python: - buildable: false - externals: - - spec: python@3.9.18 - prefix: /usr - - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - - ninja: - buildable: false - externals: - - spec: ninja@1.10.2 - prefix: /usr - - openmpi: - externals: - - spec: openmpi@5.0.5 - prefix: /hrtc/apps/mpi/openmpi/x86_64/rocky9/5.0.5/gcc/11.4.1/nocuda/ - buildable: false - - pkgconf: - externals: - - spec: pkgconf@3.0.0 - prefix: /usr - buildable: false - - readline: - externals: - - spec: readline@8.1 - prefix: /usr - buildable: false - - zlib: - buildable: false - externals: - - spec: zlib@1.2.11 - prefix: /usr - - concretizer: - unify: true - reuse: false diff --git a/scripts/spack_packages/packages/cray-libsci/package.py b/scripts/spack_packages/packages/cray-libsci/package.py new file mode 100644 index 00000000..7081cad0 --- /dev/null +++ b/scripts/spack_packages/packages/cray-libsci/package.py @@ -0,0 +1,73 @@ +from glob import glob +import os +import re + +from spack.package import * +from spack_repo.builtin.packages.cray_libsci.package import CrayLibsci as BuiltinCrayLibsci + + +class CrayLibsci(BuiltinCrayLibsci): + # Builtin recipe only lists versions through 23.02 and looks for unversioned + # names (libsci_gnu.so). CPE 26.03 on Perlmutter ships compiler-ABI names + # (libsci_gnu_123.so, libsci_nvidia_*.so) and no unversioned symlink, so + # spec["blas"].libs is empty and CMake consumers pass -l-l / a space. + version("26.03.0") + + @property + def blas_libs(self): + try: + libs = super().blas_libs + except RuntimeError: + libs = None + if libs: + return libs + + candidates = [name for name in self.canonical_names.values() if name in self.prefix] + if len(candidates) != 1: + return libs if libs is not None else find_libraries( + ["libsci_missing"], root=str(self.prefix), shared=True, recursive=False + ) + tag = candidates[0].lower() + + if self.spec.satisfies("+openmp") and self.spec.satisfies("+mpi"): + suffixes = ["_mpi_mp", "_mp"] + elif self.spec.satisfies("+openmp"): + suffixes = ["_mp"] + elif self.spec.satisfies("+mpi"): + suffixes = ["_mpi", ""] + else: + suffixes = [""] + + shared = "+shared" in self.spec + lib_dirs = [p for p in (self.prefix.lib, self.prefix.lib64) if os.path.isdir(p)] + ext = "so" if shared else "a" + + def ver_key(path): + base = os.path.basename(path) + match = re.search(r"_(\d+)\.(?:so|a)", base) + compiler_abi = int(match.group(1)) if match else 0 + # Prefer the unversioned .so/.a symlink over libfoo.so.6. + is_devlink = 1 if re.search(r"\.(so|a)$", base) else 0 + return (compiler_abi, is_devlink) + + for suffix in suffixes: + rx = re.compile( + r"^libsci_{tag}(_\d+)?{suffix}\.(so|a)(\.\d+)*$".format( + tag=re.escape(tag), suffix=re.escape(suffix) + ) + ) + matches = [] + for lib_dir in lib_dirs: + for path in glob(os.path.join(lib_dir, f"libsci_{tag}*{suffix}.{ext}*")): + if rx.match(os.path.basename(path)): + matches.append(path) + if matches: + best = sorted(set(matches), key=ver_key)[-1] + libname = os.path.basename(best).split(".")[0] + return find_libraries( + [libname], root=os.path.dirname(best), shared=shared, recursive=False + ) + + return libs if libs is not None else find_libraries( + ["libsci_missing"], root=str(self.prefix), shared=True, recursive=False + ) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 2c095bb8..7f57cbfc 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -208,7 +208,10 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Python # - depends_on('python') + # Hostconfig always needs an interpreter. Keep python off the link/run DAG + # unless +pygeosx; otherwise %%nvhpc prefers nvc for libffi/libiconv/libmd. + depends_on('python', type='build', when='~pygeosx') + depends_on('python', type=('build', 'link', 'run'), when='+pygeosx') # @@ -482,7 +485,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) - cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + hip_compiler = pjoin(str(spec['hip'].prefix), 'bin', 'amdclang++') + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', hip_compiler)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) @@ -784,7 +788,7 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) + cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) diff --git a/scripts/spack_packages/packages/gettext/package.py b/scripts/spack_packages/packages/gettext/package.py new file mode 100644 index 00000000..d833aa54 --- /dev/null +++ b/scripts/spack_packages/packages/gettext/package.py @@ -0,0 +1,35 @@ +from spack.package import * +from spack_repo.builtin.packages.gettext.package import Gettext as BuiltinGettext + + +class Gettext(BuiltinGettext): + # Builtin nvhpc-export-symbols.patch targets gettext-tools/intl (0.21 layout). + # 1.0 moved those files; the hunk fails. nvc is not used for gettext on + # Perlmutter (packages.yaml require %gcc). + pass + + +def _drop_nvhpc_export_symbols_patch(cls): + # Spack stores patches as {when_spec: [patch, ...]}. Replacing that dict + # with a list makes concretize raise "'list' object has no attribute 'items'". + patches = getattr(cls, "patches", None) + if not isinstance(patches, dict): + return + kept = {} + for when, plist in patches.items(): + filtered = [] + for patch in plist: + path = ( + getattr(patch, "relative_path", None) + or getattr(patch, "path", None) + or str(patch) + ) + if "nvhpc-export-symbols" in str(path): + continue + filtered.append(patch) + if filtered: + kept[when] = filtered + cls.patches = kept + + +_drop_nvhpc_export_symbols_patch(Gettext) diff --git a/scripts/spack_packages/packages/libmd/package.py b/scripts/spack_packages/packages/libmd/package.py new file mode 100644 index 00000000..f65815c3 --- /dev/null +++ b/scripts/spack_packages/packages/libmd/package.py @@ -0,0 +1,10 @@ +from spack.package import * +from spack_repo.builtin.packages.libmd.package import Libmd as BuiltinLibmd + + +class Libmd(BuiltinLibmd): + # Builtin nvhpc-aliases.patch matches libmd_alias(); 1.1.0 renamed that + # to libmd_strong_alias(), so the hunk fails. nvc 26 supports aliases. + # Must be a dict (when_spec -> [patch]); a list breaks Spack 1.2 concretize + # with "'list' object has no attribute 'items'". + patches = {} diff --git a/scripts/spack_packages/packages/superlu-dist/package.py b/scripts/spack_packages/packages/superlu-dist/package.py new file mode 100644 index 00000000..e6c4ff94 --- /dev/null +++ b/scripts/spack_packages/packages/superlu-dist/package.py @@ -0,0 +1,30 @@ +from spack.package import * +from spack_repo.builtin.packages.superlu_dist.package import SuperluDist as BuiltinSuperluDist + + +class SuperluDist(BuiltinSuperluDist): + # SuperLU_DIST's CMake wants absolute library paths in TPL_*_LIBRARIES. + # The builtin recipe passes ld_flags (-L/-l). When cray-libsci.libs is + # empty that becomes a blank, FindBLAS on Cray fills in "-l", and ld + # fails with "cannot find -l-l". + def cmake_args(self): + args = [ + a + for a in super().cmake_args() + if "TPL_BLAS_LIBRARIES" not in a and "TPL_LAPACK_LIBRARIES" not in a + ] + + def abs_libs(virtual): + libs = self.spec[virtual].libs + if not libs: + raise InstallError( + "{0} libraries were empty for {1}; " + "expected cray-libsci (not nvhpc) as the {0} provider".format( + virtual, self.spec[virtual] + ) + ) + return libs.joined(";") + + args.append(self.define("TPL_BLAS_LIBRARIES", abs_libs("blas"))) + args.append(self.define("TPL_LAPACK_LIBRARIES", abs_libs("lapack"))) + return args diff --git a/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch new file mode 100644 index 00000000..4ef6436f --- /dev/null +++ b/scripts/spack_packages/packages/vtk/nvhpc-surface-nets-ice.patch @@ -0,0 +1,45 @@ +--- a/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx ++++ b/Filters/Core/vtkSurfaceNets3DNonManifoldCases.cxx +@@ -8,6 +8,12 @@ + #include + #include + ++#ifdef __NVCOMPILER ++// nvc++ 26.5 ICE in extract_value_from_constant even at -O0 while evaluating ++// these constexpr tables. Drop constexpr so initialization is runtime. ++#define constexpr ++#endif ++ + // enable when creating the metadata table + // #define VTK_SURFACE_NETS_3D_NON_MANIFOLD_CASES_CREATE_TABLE + #ifdef VTK_SURFACE_NETS_3D_NON_MANIFOLD_CASES_CREATE_TABLE +@@ -6016,10 +6022,17 @@ + // clang-format on + return { offsets, metadata }; + } ++#ifdef __NVCOMPILER ++static const auto Result = CreateNonManifoldMetaDataPerEdgeCase(); ++static const auto& Offsets = Result.first; ++static const auto& Metadata = Result.second; ++#else + constexpr auto Result = CreateNonManifoldMetaDataPerEdgeCase(); + constexpr auto& Offsets = Result.first; + constexpr auto& Metadata = Result.second; + #endif ++#endif + VTK_ABI_NAMESPACE_END + } + +--- a/Filters/Core/CMakeLists.txt ++++ b/Filters/Core/CMakeLists.txt +@@ -152,6 +152,11 @@ + + vtk_add_test_mangling(VTK::FiltersCore) + ++if(CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC") ++ set_source_files_properties(vtkSurfaceNets3DNonManifoldCases.cxx PROPERTIES COMPILE_OPTIONS "-O0") ++endif() ++ + # Add precompiled header to improve build performance + if(VTK_USE_PCH) + vtk_module_precompile_headers(VTK::FiltersCore PRIVATE "$<$:${CMAKE_CURRENT_SOURCE_DIR}/vtkFiltersCorePCH.h>") diff --git a/scripts/spack_packages/packages/vtk/package.py b/scripts/spack_packages/packages/vtk/package.py index 2a75d931..45d9fcc7 100644 --- a/scripts/spack_packages/packages/vtk/package.py +++ b/scripts/spack_packages/packages/vtk/package.py @@ -71,6 +71,15 @@ class Vtk(CMakePackage): # not promote its expected condition to a compiler warning. patch('9.7.0-patch/diy-fortify-macro.patch', when='@9.7.0') + # nvc++ 26.5 ICE: interpret.cpp extract_value_from_constant on the + # constexpr SurfaceNets tables. sha256 must match the file; changing the + # patch without --fresh leaves the lockfile on the old hash (7671f62). + patch( + "nvhpc-surface-nets-ice.patch", + sha256="f56d3979fbd1b5b344c4dd7daf660ba75071f4503d447d12548a2c09102b54cc", + when="@9.7.0 %nvhpc", + ) + def cmake_args(self): spec = self.spec diff --git a/scripts/spack_packages/packages/zstd/package.py b/scripts/spack_packages/packages/zstd/package.py new file mode 100644 index 00000000..ab00fb63 --- /dev/null +++ b/scripts/spack_packages/packages/zstd/package.py @@ -0,0 +1,36 @@ +from spack.package import * +from spack_repo.builtin.packages.zstd.package import MakefileBuilder as BuiltinZstdMakefileBuilder +from spack_repo.builtin.packages.zstd.package import Zstd as BuiltinZstd + + +class Zstd(BuiltinZstd): + pass + + +class MakefileBuilder(BuiltinZstdMakefileBuilder): + def install(self, pkg, spec, prefix): + args = ["VERBOSE=1", "PREFIX=" + prefix] + + # Builtin uses DEPFLAGS=-MT $@ -MMD -MF for %nvhpc (no -MP). nvc 26.x + # still requires -MF ; the makefile then passes -o, which nvc + # treats as a missing -MF argument. + if spec.satisfies("%nvhpc"): + args.append("DEPFLAGS=") + + lib_args = ["-C", "lib"] + args + ["install-pc", "install-includes"] + if "libs=shared" in spec: + lib_args.append("install-shared") + if "libs=static" in spec: + lib_args.append("install-static") + make(*lib_args) + + if "+programs" in spec: + programs_args = ["-C", "programs"] + args + if "compression=zlib" not in spec: + programs_args.append("HAVE_ZLIB=0") + if "compression=lzma" not in spec: + programs_args.append("HAVE_LZMA=0") + if "compression=lz4" not in spec: + programs_args.append("HAVE_LZ4=0") + programs_args.append("install") + make(*programs_args)