diff --git a/mache/cime_machine_config/config_machines.xml b/mache/cime_machine_config/config_machines.xml index acd10f74..a85d5c0a 100644 --- a/mache/cime_machine_config/config_machines.xml +++ b/mache/cime_machine_config/config_machines.xml @@ -266,7 +266,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} @@ -584,7 +583,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} @@ -903,7 +901,6 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} @@ -1072,7 +1069,7 @@ /usr/share/lmod/lmod/libexec/lmod python - Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1085,16 +1082,22 @@ - Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 + + + - Core/24.07 + Core Core/24.00 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1103,12 +1106,13 @@ rocm/5.4.0 - cray-python/3.11.5 + cray-python/3.9.13.1 cray-libsci - cmake/3.27.9 + cmake/3.21.3 subversion git zlib + libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 @@ -1142,9 +1146,6 @@ spread threads - @@ -1218,10 +1218,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss + Core/24.00 cpe/22.12 craype-accel-amd-gfx90a rocm/5.4.0 - libunwind/1.6.2 + libunwind/1.5.0 cce/15.0.1 libfabric/1.15.2.0 craype/2.7.20 @@ -1690,6 +1691,49 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss + + Huge Linux workstation for Sandia climate scientists + ^[a-fA-F0-9]{12}$ + LINUX + proxy.sandia.gov:80 + gnu + openmpi + /projects/e3sm/scratch + /projects/e3sm/inputdata + /projects/e3sm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /projects/e3sm/baselines/ghci-snl-cpu/$COMPILER + /projects/e3sm/cprnc/cprnc + 32 + e3sm_developer + none + lbertag at sandia dot gov + 32 + 32 + + mpirun + + --bind-to core + -np {{ total_tasks }} + + + + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + 0.1 + 0 + + $ENV{NETCDF_C_ROOT} + $ENV{NETCDF_FORTRAN_ROOT} + $ENV{PARALLEL_NETCDF_ROOT} + 64M + spread + threads + Generic + 4000MB + + + Sandia GPU testbed weaver @@ -1990,7 +2034,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /nfs/gce/projects/climate/software/perl5/lib/perl5 - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.9.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.10.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/c-blosc2/2.15.1/gcc-12.1.0; else echo "$BLOSC2_ROOT"; fi} @@ -2333,6 +2378,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} /lcrc/group/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} + self,sm,ud + 16384 0 @@ -2813,7 +2860,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} - ^lockedfile + ^lockedfile,individual 128M @@ -3193,10 +3240,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss pbspro e3sm 208 - 208 - 104 + 96 104 - 104 48 FALSE @@ -3221,10 +3266,10 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cmake - oneapi/eng-compiler/2024.04.15.002 + oneapi/eng-compiler/2024.07.30.002 - kokkos/git.7ff87a5-omp-sycl + kokkos/4.4.01-omp-sycl spack cmake @@ -3234,10 +3279,10 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf - /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf - /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/lib:$ENV{LD_LIBRARY_PATH} - /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf/bin:$ENV{PATH} + /lus/gila/projects/CSC249ADSE15_CNDA/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002 + /lus/gila/projects/CSC249ADSE15_CNDA/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002 + /lus/gila/projects/CSC249ADSE15_CNDA/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/gila/projects/CSC249ADSE15_CNDA/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH} + /lus/gila/projects/CSC249ADSE15_CNDA/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/gila/projects/CSC249ADSE15_CNDA/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH} 1 @@ -3256,7 +3301,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss 20 $ENV{KOKKOS_ROOT} 1 - 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 + 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4 0 @@ -3285,22 +3330,22 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss LINUX oneapi-ifx,oneapi-ifxgpu,gnu mpich - CSC249ADSE15_CNDA - /lus/flare/projects/CSC249ADSE15_CNDA/performance_archive + E3SM_Dec + /lus/flare/projects/E3SM_Dec/performance_archive .* - /lus/flare/projects/CSC249ADSE15_CNDA/$USER/scratch - /lus/flare/projects/CSC249ADSE15_CNDA/inputdata - /lus/flare/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 + /lus/flare/projects/$PROJECT/$USER/scratch + /lus/flare/projects/E3SM_Dec/inputdata + /lus/flare/projects/E3SM_Dec/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE - /lus/flare/projects/CSC249ADSE15_CNDA/baselines/$COMPILER - /lus/flare/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc + /lus/flare/projects/E3SM_Dec/baselines/$COMPILER + /lus/flare/projects/E3SM_Dec/tools/cprnc/cprnc 16 e3sm_developer 4 pbspro e3sm 208 - 104 + 96 104 48 FALSE @@ -3316,20 +3361,20 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /lus/flare/projects/CSC249ADSE15_CNDA/modules/lmod.sh - /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/csh - /soft/sunspot_migrate/soft/packaging/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/env_modules_python.py module module - /soft/sunspot_migrate/soft/packaging/lmod/lmod/libexec/lmod python + /usr/share/lmod/lmod/libexec/lmod python cmake - oneapi/eng-compiler/2024.04.15.002 + oneapi/eng-compiler/2024.07.30.002 - kokkos/git.7ff87a5-omp-sycl + kokkos/4.4.01-omp-sycl spack-pe-gcc cmake @@ -3339,11 +3384,10 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002 - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/lib:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/lib:$ENV{LD_LIBRARY_PATH} - /lus/flare/projects/CSC249ADSE15_CNDA/software/pnetcdf/1.12.3/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-fortran/4.6.1/oneapi.eng.2024.04.15.002/bin:/lus/flare/projects/CSC249ADSE15_CNDA/software/netcdf-c/4.9.2/oneapi.eng.2024.04.15.002/bin:$ENV{PATH} + /lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002 + /lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002 + /lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/lib:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/lib:$ENV{LD_LIBRARY_PATH} + /lus/flare/projects/E3SM_Dec/soft/pnetcdf/1.14.0/oneapi.eng.2024.07.30.002/bin:/lus/flare/projects/E3SM_Dec/soft/netcdf/4.9.2c-4.6.1f/oneapi.eng.2024.07.30.002/bin:$ENV{PATH} 1 @@ -3362,7 +3406,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss 20 $ENV{KOKKOS_ROOT} 1 - 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4 + 0:4,1:4,2:4,3:4:4:4,5:4,6:4,7:4,8:4,9:4,10:4,11:4 0 @@ -3983,7 +4027,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4003,11 +4047,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4019,39 +4063,42 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc + gcc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 + cray-libsci/23.12.5 - PrgEnv-nvidia/8.4.0 - nvidia/22.7 - cray-libsci/23.05.1.4 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 + cray-libsci/23.12.5 - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4062,13 +4109,12 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - libfabric/1.15.2.0 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4090,14 +4136,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} + -1 - Chicoma GPU nodes at LANL IC. Each GPU node has single -AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' + Chicoma GPU nodes at LANL IC. Each GPU node has single AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' ch-fe* Linux gnugpu,gnu,nvidiagpu,nvidia @@ -4107,7 +4155,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4131,11 +4179,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4147,32 +4195,35 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/11.2.0 + PrgEnv-gnu/8.5.0 + gcc/12.2.0 + cray-libsci/23.05.1.4 PrgEnv-nvidia/8.4.0 nvidia/22.7 + cray-libsci/23.05.1.4 @@ -4195,14 +4246,13 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - cray-libsci/23.05.1.4 + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + cmake/3.27.7 @@ -4225,6 +4275,9 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' $ENV{CRAY_PARALLEL_NETCDF_PREFIX} /usr/projects/e3sm/cudatoolkit:$ENV{PKG_CONFIG_PATH} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 diff --git a/mache/spack/chicoma-cpu_gnu_mpich.csh b/mache/spack/chicoma-cpu_gnu_mpich.csh index 434a23cd..c8445524 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.csh +++ b/mache/spack/chicoma-cpu_gnu_mpich.csh @@ -56,7 +56,6 @@ setenv MPICH_VERSION_DISPLAY 1 setenv HDF5_USE_FILE_LOCKING FALSE setenv PERL5LIB /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -setenv FI_CXI_RX_MATCH_MODE software setenv MPICH_COLL_SYNC MPI_Bcast # for standalone MPAS builds diff --git a/mache/spack/chicoma-cpu_gnu_mpich.sh b/mache/spack/chicoma-cpu_gnu_mpich.sh index d3ec206e..5cd56340 100644 --- a/mache/spack/chicoma-cpu_gnu_mpich.sh +++ b/mache/spack/chicoma-cpu_gnu_mpich.sh @@ -56,7 +56,6 @@ export MPICH_VERSION_DISPLAY=1 export HDF5_USE_FILE_LOCKING=FALSE export PERL5LIB=/usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" -export FI_CXI_RX_MATCH_MODE=software export MPICH_COLL_SYNC=MPI_Bcast export LD_LIBRARY_PATH="/usr/lib64/gcc/x86_64-suse-linux/12:${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}" diff --git a/mache/spack/frontier_amdclang_mpich.csh b/mache/spack/frontier_amdclang_mpich.csh index 31ca9c28..fe95a120 100644 --- a/mache/spack/frontier_amdclang_mpich.csh +++ b/mache/spack/frontier_amdclang_mpich.csh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null module switch amd amd/5.4.0 >& /dev/null diff --git a/mache/spack/frontier_amdclang_mpich.sh b/mache/spack/frontier_amdclang_mpich.sh index aee1dd35..8f1d3d31 100644 --- a/mache/spack/frontier_amdclang_mpich.sh +++ b/mache/spack/frontier_amdclang_mpich.sh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null module switch amd amd/5.4.0 >& /dev/null diff --git a/mache/spack/frontier_amdclang_mpich.yaml b/mache/spack/frontier_amdclang_mpich.yaml index 47a0d240..e1747f42 100644 --- a/mache/spack/frontier_amdclang_mpich.yaml +++ b/mache/spack/frontier_amdclang_mpich.yaml @@ -156,7 +156,7 @@ spack: operating_system: sles15 target: any modules: - - Core/24.07 + - Core/24.00 - PrgEnv-amd/8.3.3 - amd/5.4.0 - libfabric/1.15.2.0 diff --git a/mache/spack/frontier_amdclanggpu_mpich.csh b/mache/spack/frontier_amdclanggpu_mpich.csh index 279c55eb..0f40d699 100644 --- a/mache/spack/frontier_amdclanggpu_mpich.csh +++ b/mache/spack/frontier_amdclanggpu_mpich.csh @@ -1,8 +1,9 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null module switch amd amd/5.4.0 >& /dev/null -module load craype-accel-amd-gfx90a >& /dev/null +# see https://github.com/E3SM-Project/E3SM/issues/6755 +# module load craype-accel-amd-gfx90a >& /dev/null {% if e3sm_lapack %} module load cray-libsci/22.12.1.1 diff --git a/mache/spack/frontier_amdclanggpu_mpich.sh b/mache/spack/frontier_amdclanggpu_mpich.sh index a9ee68e2..b55bb6cd 100644 --- a/mache/spack/frontier_amdclanggpu_mpich.sh +++ b/mache/spack/frontier_amdclanggpu_mpich.sh @@ -1,8 +1,9 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-amd/8.3.3 >& /dev/null module switch amd amd/5.4.0 >& /dev/null -module load craype-accel-amd-gfx90a >& /dev/null +# see https://github.com/E3SM-Project/E3SM/issues/6755 +# module load craype-accel-amd-gfx90a >& /dev/null {% if e3sm_lapack %} module load cray-libsci/22.12.1.1 diff --git a/mache/spack/frontier_amdclanggpu_mpich.yaml b/mache/spack/frontier_amdclanggpu_mpich.yaml index 53462add..9fd2b466 100644 --- a/mache/spack/frontier_amdclanggpu_mpich.yaml +++ b/mache/spack/frontier_amdclanggpu_mpich.yaml @@ -156,10 +156,11 @@ spack: operating_system: sles15 target: any modules: - - Core/24.07 + - Core/24.00 - PrgEnv-amd/8.3.3 - amd/5.4.0 - - craype-accel-amd-gfx90a + # see https://github.com/E3SM-Project/E3SM/issues/6755 + # - craype-accel-amd-gfx90a - libfabric/1.15.2.0 environment: set: diff --git a/mache/spack/frontier_crayclang_mpich.csh b/mache/spack/frontier_crayclang_mpich.csh index a65a8bda..65726273 100644 --- a/mache/spack/frontier_crayclang_mpich.csh +++ b/mache/spack/frontier_crayclang_mpich.csh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null module switch cce cce/15.0.1 >& /dev/null module switch craype craype/2.7.20 >& /dev/null diff --git a/mache/spack/frontier_crayclang_mpich.sh b/mache/spack/frontier_crayclang_mpich.sh index cb5fbe76..3297fc3a 100644 --- a/mache/spack/frontier_crayclang_mpich.sh +++ b/mache/spack/frontier_crayclang_mpich.sh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null module switch cce cce/15.0.1 >& /dev/null module switch craype craype/2.7.20 >& /dev/null diff --git a/mache/spack/frontier_crayclang_mpich.yaml b/mache/spack/frontier_crayclang_mpich.yaml index 97d45454..a54fd296 100644 --- a/mache/spack/frontier_crayclang_mpich.yaml +++ b/mache/spack/frontier_crayclang_mpich.yaml @@ -156,7 +156,7 @@ spack: operating_system: sles15 target: any modules: - - Core/24.07 + - Core/24.00 - PrgEnv-cray/8.3.3 - cce/15.0.1 - craype/2.7.20 diff --git a/mache/spack/frontier_crayclanggpu_mpich.csh b/mache/spack/frontier_crayclanggpu_mpich.csh index d2722bc9..67992212 100644 --- a/mache/spack/frontier_crayclanggpu_mpich.csh +++ b/mache/spack/frontier_crayclanggpu_mpich.csh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null module switch cce cce/15.0.1 >& /dev/null module switch craype craype/2.7.20 >& /dev/null diff --git a/mache/spack/frontier_crayclanggpu_mpich.sh b/mache/spack/frontier_crayclanggpu_mpich.sh index c0e4a68d..903cc9b3 100644 --- a/mache/spack/frontier_crayclanggpu_mpich.sh +++ b/mache/spack/frontier_crayclanggpu_mpich.sh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null module switch cce cce/15.0.1 >& /dev/null module switch craype craype/2.7.20 >& /dev/null diff --git a/mache/spack/frontier_crayclanggpu_mpich.yaml b/mache/spack/frontier_crayclanggpu_mpich.yaml index 6b4640a4..8f559f7b 100644 --- a/mache/spack/frontier_crayclanggpu_mpich.yaml +++ b/mache/spack/frontier_crayclanggpu_mpich.yaml @@ -156,7 +156,7 @@ spack: operating_system: sles15 target: any modules: - - Core/24.07 + - Core/24.00 - PrgEnv-cray/8.3.3 - cce/15.0.1 - craype/2.7.20 diff --git a/mache/spack/frontier_gnu_mpich.csh b/mache/spack/frontier_gnu_mpich.csh index 50113051..6317f814 100644 --- a/mache/spack/frontier_gnu_mpich.csh +++ b/mache/spack/frontier_gnu_mpich.csh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null module switch gcc gcc/12.2.0 >& /dev/null diff --git a/mache/spack/frontier_gnu_mpich.sh b/mache/spack/frontier_gnu_mpich.sh index a06a0b7b..85e27a08 100644 --- a/mache/spack/frontier_gnu_mpich.sh +++ b/mache/spack/frontier_gnu_mpich.sh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null module switch gcc gcc/12.2.0 >& /dev/null diff --git a/mache/spack/frontier_gnu_mpich.yaml b/mache/spack/frontier_gnu_mpich.yaml index e4163c89..2a201b1a 100644 --- a/mache/spack/frontier_gnu_mpich.yaml +++ b/mache/spack/frontier_gnu_mpich.yaml @@ -171,7 +171,7 @@ spack: operating_system: sles15 target: x86_64 modules: - - Core/24.07 + - Core/24.00 - PrgEnv-gnu/8.3.3 - gcc/12.2.0 - craype/2.7.19 diff --git a/mache/spack/frontier_gnugpu_mpich.csh b/mache/spack/frontier_gnugpu_mpich.csh index ef9dcb2d..abd9f15b 100644 --- a/mache/spack/frontier_gnugpu_mpich.csh +++ b/mache/spack/frontier_gnugpu_mpich.csh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null module switch gcc gcc/11.2.0 >& /dev/null module load craype-accel-amd-gfx90a diff --git a/mache/spack/frontier_gnugpu_mpich.sh b/mache/spack/frontier_gnugpu_mpich.sh index eda5278a..3ccd4c35 100644 --- a/mache/spack/frontier_gnugpu_mpich.sh +++ b/mache/spack/frontier_gnugpu_mpich.sh @@ -1,5 +1,5 @@ module reset >& /dev/null -module switch Core/24.07 >& /dev/null +module switch Core Core/24.00 >& /dev/null module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null module switch gcc gcc/11.2.0 >& /dev/null module load craype-accel-amd-gfx90a diff --git a/mache/spack/frontier_gnugpu_mpich.yaml b/mache/spack/frontier_gnugpu_mpich.yaml index b08f429a..dd1d1b6d 100644 --- a/mache/spack/frontier_gnugpu_mpich.yaml +++ b/mache/spack/frontier_gnugpu_mpich.yaml @@ -172,7 +172,7 @@ spack: operating_system: sles15 target: x86_64 modules: - - Core/24.07 + - Core/24.00 - PrgEnv-gnu/8.3.3 - gcc/11.2.0 - craype/2.7.19 diff --git a/mache/spack/pm-cpu_gnu_mpich.csh b/mache/spack/pm-cpu_gnu_mpich.csh index 174ef24f..25a550bc 100644 --- a/mache/spack/pm-cpu_gnu_mpich.csh +++ b/mache/spack/pm-cpu_gnu_mpich.csh @@ -51,7 +51,6 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1 setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv FI_CXI_RX_MATCH_MODE software setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB diff --git a/mache/spack/pm-cpu_gnu_mpich.sh b/mache/spack/pm-cpu_gnu_mpich.sh index b824d419..19314006 100644 --- a/mache/spack/pm-cpu_gnu_mpich.sh +++ b/mache/spack/pm-cpu_gnu_mpich.sh @@ -51,7 +51,6 @@ export MPICH_MPIIO_DVS_MAXNODES=1 export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export FI_CXI_RX_MATCH_MODE=software export FI_MR_CACHE_MONITOR=kdreg2 if [ -z "${NERSC_HOST:-}" ]; then diff --git a/mache/spack/pm-cpu_intel_mpich.csh b/mache/spack/pm-cpu_intel_mpich.csh index 52de4034..4d8e1942 100644 --- a/mache/spack/pm-cpu_intel_mpich.csh +++ b/mache/spack/pm-cpu_intel_mpich.csh @@ -49,7 +49,6 @@ setenv MPICH_MPIIO_DVS_MAXNODES 1 setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -setenv FI_CXI_RX_MATCH_MODE software setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB diff --git a/mache/spack/pm-cpu_intel_mpich.sh b/mache/spack/pm-cpu_intel_mpich.sh index 5d4e3b85..577ab3cc 100644 --- a/mache/spack/pm-cpu_intel_mpich.sh +++ b/mache/spack/pm-cpu_intel_mpich.sh @@ -49,7 +49,6 @@ export MPICH_MPIIO_DVS_MAXNODES=1 export HDF5_USE_FILE_LOCKING=FALSE ## Not needed # export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch -export FI_CXI_RX_MATCH_MODE=software export FI_MR_CACHE_MONITOR=kdreg2 if [ -z "${NERSC_HOST:-}" ]; then diff --git a/mache/spack/pm-cpu_nvidia_mpich.csh b/mache/spack/pm-cpu_nvidia_mpich.csh index 6a0f0732..27064d2c 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.csh +++ b/mache/spack/pm-cpu_nvidia_mpich.csh @@ -52,7 +52,6 @@ setenv HDF5_USE_FILE_LOCKING FALSE ## Not needed # setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch setenv MPICH_GPU_SUPPORT_ENABLED 1 -setenv FI_CXI_RX_MATCH_MODE software setenv FI_MR_CACHE_MONITOR kdreg2 setenv MPICH_COLL_SYNC MPI_Bcast setenv GATOR_INITIAL_MB 4000MB diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh index 90f1d967..121bef12 100644 --- a/mache/spack/pm-cpu_nvidia_mpich.sh +++ b/mache/spack/pm-cpu_nvidia_mpich.sh @@ -57,7 +57,6 @@ if [ -z "${NERSC_HOST:-}" ]; then # happens when building spack environment export NERSC_HOST="perlmutter" fi -export FI_CXI_RX_MATCH_MODE=software export FI_MR_CACHE_MONITOR=kdreg2 export MPICH_COLL_SYNC=MPI_Bcast export GATOR_INITIAL_MB=4000MB