From a774e83d1254ac67b22d20d5c216d228cf2d19b9 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 08:34:32 -0700 Subject: [PATCH 01/15] Add files for Ubuntu container, decoupled from machine --- configs/perlmuttercontainer/compilers.yaml | 13 ++++++++ configs/perlmuttercontainer/config.yaml | 2 ++ configs/perlmuttercontainer/packages.yaml | 31 +++++++++++++++++++ .../exawind_perlmuttercontainer.yaml | 11 +++++++ 4 files changed, 57 insertions(+) create mode 100644 configs/perlmuttercontainer/compilers.yaml create mode 100644 configs/perlmuttercontainer/config.yaml create mode 100644 configs/perlmuttercontainer/packages.yaml create mode 100644 env-templates/exawind_perlmuttercontainer.yaml diff --git a/configs/perlmuttercontainer/compilers.yaml b/configs/perlmuttercontainer/compilers.yaml new file mode 100644 index 00000000..2b553b6f --- /dev/null +++ b/configs/perlmuttercontainer/compilers.yaml @@ -0,0 +1,13 @@ +compilers: +- compiler: + spec: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: any + modules: [] + extra_rpaths: [] diff --git a/configs/perlmuttercontainer/config.yaml b/configs/perlmuttercontainer/config.yaml new file mode 100644 index 00000000..b7797366 --- /dev/null +++ b/configs/perlmuttercontainer/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 20 diff --git a/configs/perlmuttercontainer/packages.yaml b/configs/perlmuttercontainer/packages.yaml new file mode 100644 index 00000000..b0e11aac --- /dev/null +++ b/configs/perlmuttercontainer/packages.yaml @@ -0,0 +1,31 @@ +packages: + mpich: + version: [] + buildable: + externals: + - spec: + modules: [] + cuda: + version: [] + buildable: false + externals: + - spec: + prefix: + modules: + cmake: + version: [] + buildable: + externals: + - spec: + prefix: + modules: + trilinos: + require: + - any_of: ["@13.4.0", "@develop"] + all: + compiler: + providers: + mpi: [] + blas: [] + lapack: [] + variants: build_type=Release cuda_arch=80 diff --git a/env-templates/exawind_perlmuttercontainer.yaml b/env-templates/exawind_perlmuttercontainer.yaml new file mode 100644 index 00000000..09f2fc05 --- /dev/null +++ b/env-templates/exawind_perlmuttercontainer.yaml @@ -0,0 +1,11 @@ +spack: + include: + - include.yaml + concretizer: + unify: false + reuse: false + view: false + specs: + - 'exawind+hypre+amr_wind_gpu+nalu_wind_gpu+cuda cuda_arch=80' + - 'exawind+hypre~amr_wind_gpu~nalu_wind_gpu~cuda' + #- 'exawind~hypre+amr_wind_gpu~nalu_wind_gpu+cuda cuda_arch=80 ^nalu-wind+hypre ^amr-wind~hypre' From 4f3bec1f20bd892979ff72343579e8ef652a0e69 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 09:06:46 -0700 Subject: [PATCH 02/15] change OS from sles15 to ubuntu22.04 --- configs/perlmutter/compilers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/perlmutter/compilers.yaml b/configs/perlmutter/compilers.yaml index 1d807f01..a4fe7aa7 100644 --- a/configs/perlmutter/compilers.yaml +++ b/configs/perlmutter/compilers.yaml @@ -7,7 +7,7 @@ compilers: f77: ftn fc: ftn flags: {} - operating_system: sles15 + operating_system: ubuntu22.04 target: any modules: - PrgEnv-gnu/8.3.3 From e259b548c3c9accda8837ddc6ec2d579097eedab Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 10:44:54 -0700 Subject: [PATCH 03/15] build exawind for CPU with ubuntu22.04 base image --- .../compilers.yaml | 2 +- .../config.yaml | 0 configs/containercpu/packages.yaml | 79 +++++++++++++++++++ configs/perlmutter/compilers.yaml | 2 +- configs/perlmuttercontainer/packages.yaml | 31 -------- scripts/create-exawind-snapshot.sh | 2 + 6 files changed, 83 insertions(+), 33 deletions(-) rename configs/{perlmuttercontainer => containercpu}/compilers.yaml (91%) rename configs/{perlmuttercontainer => containercpu}/config.yaml (100%) create mode 100644 configs/containercpu/packages.yaml delete mode 100644 configs/perlmuttercontainer/packages.yaml diff --git a/configs/perlmuttercontainer/compilers.yaml b/configs/containercpu/compilers.yaml similarity index 91% rename from configs/perlmuttercontainer/compilers.yaml rename to configs/containercpu/compilers.yaml index 2b553b6f..10686fe8 100644 --- a/configs/perlmuttercontainer/compilers.yaml +++ b/configs/containercpu/compilers.yaml @@ -1,6 +1,6 @@ compilers: - compiler: - spec: + spec: gcc@11.4.0 paths: cc: /usr/bin/gcc cxx: /usr/bin/g++ diff --git a/configs/perlmuttercontainer/config.yaml b/configs/containercpu/config.yaml similarity index 100% rename from configs/perlmuttercontainer/config.yaml rename to configs/containercpu/config.yaml diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml new file mode 100644 index 00000000..d2153507 --- /dev/null +++ b/configs/containercpu/packages.yaml @@ -0,0 +1,79 @@ +packages: +# Global settings + all: + compiler: + - gcc@11.4.0 + providers: + mpi: [mpich@4.0] + blas: [netlib-lapack] + lapack: [netlib-lapack] + variants: build_type=Release +# External packages from Ubuntu repos + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + gmake: + externals: + - spec: gmake@4.3 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + git: + externals: + - spec: git@2.34.1~tcltk + prefix: /usr + cmake: + externals: + - spec: cmake@3.22.1 + prefix: /usr + mpich: + externals: + - spec: mpich@4.0+hydra device=ch4 netmod=ofi + prefix: /usr +# Package preferences to be built by Spack for correct Exawind + ascent: + variants: ~fortran~openmp + amr-wind: + variants: +tiny_profile + conduit: + variants: ~fortran~hdf5_compat + boost: + version: [1.78.0] + variants: cxxstd=17 + hdf5: + version: [1.10.7] + variants: +cxx+hl + masa: + variants: ~fortran~python + netcdf-c: + require: '@4.7.4' + variants: +parallel-netcdf maxdims=65536 maxvars=524288 + openfast: + version: [master] + variants: +cxx + parallel-netcdf: + version: [1.12.2] + variants: ~fortran + perl: + require: '@5.34.1' + tioga: + version: [develop] + hypre: + require: '@develop' + variants: ~fortran + hypre2: + require: '@develop' + variants: ~fortran + yaml-cpp: + version: [0.6.3] diff --git a/configs/perlmutter/compilers.yaml b/configs/perlmutter/compilers.yaml index a4fe7aa7..1d807f01 100644 --- a/configs/perlmutter/compilers.yaml +++ b/configs/perlmutter/compilers.yaml @@ -7,7 +7,7 @@ compilers: f77: ftn fc: ftn flags: {} - operating_system: ubuntu22.04 + operating_system: sles15 target: any modules: - PrgEnv-gnu/8.3.3 diff --git a/configs/perlmuttercontainer/packages.yaml b/configs/perlmuttercontainer/packages.yaml deleted file mode 100644 index b0e11aac..00000000 --- a/configs/perlmuttercontainer/packages.yaml +++ /dev/null @@ -1,31 +0,0 @@ -packages: - mpich: - version: [] - buildable: - externals: - - spec: - modules: [] - cuda: - version: [] - buildable: false - externals: - - spec: - prefix: - modules: - cmake: - version: [] - buildable: - externals: - - spec: - prefix: - modules: - trilinos: - require: - - any_of: ["@13.4.0", "@develop"] - all: - compiler: - providers: - mpi: [] - blas: [] - lapack: [] - variants: build_type=Release cuda_arch=80 diff --git a/scripts/create-exawind-snapshot.sh b/scripts/create-exawind-snapshot.sh index 2dcef826..d93069c9 100755 --- a/scripts/create-exawind-snapshot.sh +++ b/scripts/create-exawind-snapshot.sh @@ -55,6 +55,8 @@ elif [[ "${SPACK_MANAGER_MACHINE}" == "perlmutter" ]]; then elif [[ "${SPACK_MANAGER_MACHINE}" == "snl-hpc" ]]; then # TODO we should probably launch the install through slurm and exit on this one cmd "nice -n19 spack manager snapshot -s exawind+hypre+openfast amr-wind+hypre+openfast" +elif [[ "${SPACK_MANAGER_MACHINE}" == "containercpu" ]]; then + cmd "nice -n19 spack manager snapshot -m -s exawind%gcc+hypre~cuda" else cmd "nice -n19 spack manager snapshot -s exawind+hypre+openfast" fi From 1d1a3ab4f034f7f0660b7f267d9dae4398d5662f Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 11:18:15 -0700 Subject: [PATCH 04/15] exawind_containercpu.yaml, find_machine.py --- ...wind_perlmuttercontainer.yaml => exawind_containercpu.yaml} | 1 - spack-scripting/scripting/cmd/manager_cmds/find_machine.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) rename env-templates/{exawind_perlmuttercontainer.yaml => exawind_containercpu.yaml} (79%) diff --git a/env-templates/exawind_perlmuttercontainer.yaml b/env-templates/exawind_containercpu.yaml similarity index 79% rename from env-templates/exawind_perlmuttercontainer.yaml rename to env-templates/exawind_containercpu.yaml index 09f2fc05..2dfe68df 100644 --- a/env-templates/exawind_perlmuttercontainer.yaml +++ b/env-templates/exawind_containercpu.yaml @@ -6,6 +6,5 @@ spack: reuse: false view: false specs: - - 'exawind+hypre+amr_wind_gpu+nalu_wind_gpu+cuda cuda_arch=80' - 'exawind+hypre~amr_wind_gpu~nalu_wind_gpu~cuda' #- 'exawind~hypre+amr_wind_gpu~nalu_wind_gpu+cuda cuda_arch=80 ^nalu-wind+hypre ^amr-wind~hypre' diff --git a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py index 34cc495e..a80e2643 100644 --- a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py +++ b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py @@ -99,6 +99,9 @@ def is_e4s(): ), # General "darwin": MachineData(lambda: sys.platform == "darwin", "darwin.nodomain.gov"), + "containercpu": MachineData( + lambda: os.environ["SPACK_MACHINE_MANAGER"] == "containercpu", "containcpu.nodomain.gov" + ), } From 6c071b6d04d5de2b72a30c36efe65d7322feb67f Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 11:28:46 -0700 Subject: [PATCH 05/15] find_machine.py: add CONTAINER_BUILD=cpu --- spack-scripting/scripting/cmd/manager_cmds/find_machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py index a80e2643..fa3bc805 100644 --- a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py +++ b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py @@ -100,7 +100,7 @@ def is_e4s(): # General "darwin": MachineData(lambda: sys.platform == "darwin", "darwin.nodomain.gov"), "containercpu": MachineData( - lambda: os.environ["SPACK_MACHINE_MANAGER"] == "containercpu", "containcpu.nodomain.gov" + lambda: os.environ["CONTAINER_BUILD"] == "cpu", "containcpu.nodomain.gov" ), } From 0a73323a8a4b885eb10ec48be63ea555bfbeef78 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 12:09:15 -0700 Subject: [PATCH 06/15] buildable: false for external (ubuntu) packages --- configs/containercpu/packages.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index d2153507..359b52f6 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -10,34 +10,42 @@ packages: variants: build_type=Release # External packages from Ubuntu repos m4: + buildable: false externals: - spec: m4@1.4.18 prefix: /usr gmake: + buildable: false externals: - spec: gmake@4.3 prefix: /usr automake: + buildable: false externals: - spec: automake@1.16.5 prefix: /usr autoconf: + buildable: false externals: - spec: autoconf@2.71 prefix: /usr libtool: + buildable: false externals: - spec: libtool@2.4.6 prefix: /usr git: + buildable: false externals: - spec: git@2.34.1~tcltk prefix: /usr cmake: + buildable: false externals: - spec: cmake@3.22.1 prefix: /usr mpich: + buildable: false externals: - spec: mpich@4.0+hydra device=ch4 netmod=ofi prefix: /usr From 9a8e4805acef1445b723f89a1f7c54de3f248ea3 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 12:57:38 -0700 Subject: [PATCH 07/15] CMake: Build with Spack for >= 3.23.x --- configs/containercpu/packages.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 359b52f6..01ffa598 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -39,11 +39,6 @@ packages: externals: - spec: git@2.34.1~tcltk prefix: /usr - cmake: - buildable: false - externals: - - spec: cmake@3.22.1 - prefix: /usr mpich: buildable: false externals: @@ -59,6 +54,9 @@ packages: boost: version: [1.78.0] variants: cxxstd=17 + cmake: + version: [3.26.3] + variants: build_type=Release hdf5: version: [1.10.7] variants: +cxx+hl From 735dd506cf061aebb4b71a0a9c7c6a2ab29a728c Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Tue, 26 Sep 2023 08:09:39 -0700 Subject: [PATCH 08/15] libtool: use Spack version 2.4.7 for correct linking --- configs/containercpu/packages.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 01ffa598..8d079e72 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -29,11 +29,6 @@ packages: externals: - spec: autoconf@2.71 prefix: /usr - libtool: - buildable: false - externals: - - spec: libtool@2.4.6 - prefix: /usr git: buildable: false externals: @@ -45,6 +40,7 @@ packages: - spec: mpich@4.0+hydra device=ch4 netmod=ofi prefix: /usr # Package preferences to be built by Spack for correct Exawind +# Nota bene: use libtool from Spack for correct linking ascent: variants: ~fortran~openmp amr-wind: @@ -60,6 +56,8 @@ packages: hdf5: version: [1.10.7] variants: +cxx+hl + libtool: + version: [2.4.7] masa: variants: ~fortran~python netcdf-c: From 04c46c90e298a54db4cbea67e64afdaabbea73a5 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Tue, 26 Sep 2023 11:11:58 -0700 Subject: [PATCH 09/15] mpich: require --- configs/containercpu/packages.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 8d079e72..2b882069 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -4,7 +4,7 @@ packages: compiler: - gcc@11.4.0 providers: - mpi: [mpich@4.0] + mpi: [mpich] blas: [netlib-lapack] lapack: [netlib-lapack] variants: build_type=Release @@ -39,6 +39,9 @@ packages: externals: - spec: mpich@4.0+hydra device=ch4 netmod=ofi prefix: /usr + mpi: + require: [mpich] + # Package preferences to be built by Spack for correct Exawind # Nota bene: use libtool from Spack for correct linking ascent: From 223321bd93ae616c3d799a3a437bad90cecc55aa Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Tue, 26 Sep 2023 12:45:51 -0700 Subject: [PATCH 10/15] add openmpi for hdf5 build --- configs/containercpu/packages.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 2b882069..53acd2b1 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -4,7 +4,7 @@ packages: compiler: - gcc@11.4.0 providers: - mpi: [mpich] + mpi: [mpich, openmpi] blas: [netlib-lapack] lapack: [netlib-lapack] variants: build_type=Release @@ -39,8 +39,6 @@ packages: externals: - spec: mpich@4.0+hydra device=ch4 netmod=ofi prefix: /usr - mpi: - require: [mpich] # Package preferences to be built by Spack for correct Exawind # Nota bene: use libtool from Spack for correct linking From 7db3d1946c90b125c00fcc5209cf77c0cddeeac5 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Tue, 26 Sep 2023 13:33:06 -0700 Subject: [PATCH 11/15] mpich: require Spack build (vs. ubuntu) --- configs/containercpu/packages.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 53acd2b1..67c2d026 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -4,7 +4,7 @@ packages: compiler: - gcc@11.4.0 providers: - mpi: [mpich, openmpi] + mpi: [mpich] blas: [netlib-lapack] lapack: [netlib-lapack] variants: build_type=Release @@ -34,12 +34,8 @@ packages: externals: - spec: git@2.34.1~tcltk prefix: /usr - mpich: - buildable: false - externals: - - spec: mpich@4.0+hydra device=ch4 netmod=ofi - prefix: /usr - + mpi: + require: mpich # Package preferences to be built by Spack for correct Exawind # Nota bene: use libtool from Spack for correct linking ascent: From 1c10480705a84c4806fff6a458886fa7a50ee2fe Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Mon, 25 Sep 2023 08:34:32 -0700 Subject: [PATCH 12/15] Add files for Ubuntu container, decoupled from machine change OS from sles15 to ubuntu22.04 build exawind for CPU with ubuntu22.04 base image exawind_containercpu.yaml, find_machine.py find_machine.py: add CONTAINER_BUILD=cpu buildable: false for external (ubuntu) packages CMake: Build with Spack for >= 3.23.x libtool: use Spack version 2.4.7 for correct linking mpich: require add openmpi for hdf5 build mpich: require Spack build (vs. ubuntu) rm: hard coded settings and use spack external find --- configs/containercpu/compilers.yaml | 13 +++++ configs/containercpu/config.yaml | 2 + configs/containercpu/packages.yaml | 55 +++++++++++++++++++ env-templates/exawind_containercpu.yaml | 10 ++++ scripts/create-exawind-snapshot.sh | 2 + .../cmd/manager_cmds/find_machine.py | 3 + 6 files changed, 85 insertions(+) create mode 100644 configs/containercpu/compilers.yaml create mode 100644 configs/containercpu/config.yaml create mode 100644 configs/containercpu/packages.yaml create mode 100644 env-templates/exawind_containercpu.yaml diff --git a/configs/containercpu/compilers.yaml b/configs/containercpu/compilers.yaml new file mode 100644 index 00000000..10686fe8 --- /dev/null +++ b/configs/containercpu/compilers.yaml @@ -0,0 +1,13 @@ +compilers: +- compiler: + spec: gcc@11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: any + modules: [] + extra_rpaths: [] diff --git a/configs/containercpu/config.yaml b/configs/containercpu/config.yaml new file mode 100644 index 00000000..b7797366 --- /dev/null +++ b/configs/containercpu/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 20 diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml new file mode 100644 index 00000000..9f3d3dfd --- /dev/null +++ b/configs/containercpu/packages.yaml @@ -0,0 +1,55 @@ +packages: +# Global settings + all: + compiler: + - gcc@11.4.0 + providers: + mpi: [mpich] + blas: [netlib-lapack] + lapack: [netlib-lapack] + variants: build_type=Release +# External packages from Ubuntu repos + mpi: + require: mpich +# Package preferences to be built by Spack for correct Exawind +# Nota bene: use libtool from Spack for correct linking + ascent: + variants: ~fortran~openmp + amr-wind: + variants: +tiny_profile + conduit: + variants: ~fortran~hdf5_compat + boost: + version: [1.78.0] + variants: cxxstd=17 + cmake: + version: [3.26.3] + variants: build_type=Release + hdf5: + version: [1.10.7] + variants: +cxx+hl + libtool: + version: [2.4.7] + masa: + variants: ~fortran~python + netcdf-c: + require: '@4.7.4' + variants: +parallel-netcdf maxdims=65536 maxvars=524288 + openfast: + version: [master] + variants: +cxx + parallel-netcdf: + version: [1.12.2] + variants: ~fortran + perl: + require: '@5.34.1' + tioga: + version: [develop] + hypre: + require: '@develop' + variants: ~fortran + hypre2: + require: '@develop' + variants: ~fortran + yaml-cpp: + version: [0.6.3] diff --git a/env-templates/exawind_containercpu.yaml b/env-templates/exawind_containercpu.yaml new file mode 100644 index 00000000..2dfe68df --- /dev/null +++ b/env-templates/exawind_containercpu.yaml @@ -0,0 +1,10 @@ +spack: + include: + - include.yaml + concretizer: + unify: false + reuse: false + view: false + specs: + - 'exawind+hypre~amr_wind_gpu~nalu_wind_gpu~cuda' + #- 'exawind~hypre+amr_wind_gpu~nalu_wind_gpu+cuda cuda_arch=80 ^nalu-wind+hypre ^amr-wind~hypre' diff --git a/scripts/create-exawind-snapshot.sh b/scripts/create-exawind-snapshot.sh index 2dcef826..d93069c9 100755 --- a/scripts/create-exawind-snapshot.sh +++ b/scripts/create-exawind-snapshot.sh @@ -55,6 +55,8 @@ elif [[ "${SPACK_MANAGER_MACHINE}" == "perlmutter" ]]; then elif [[ "${SPACK_MANAGER_MACHINE}" == "snl-hpc" ]]; then # TODO we should probably launch the install through slurm and exit on this one cmd "nice -n19 spack manager snapshot -s exawind+hypre+openfast amr-wind+hypre+openfast" +elif [[ "${SPACK_MANAGER_MACHINE}" == "containercpu" ]]; then + cmd "nice -n19 spack manager snapshot -m -s exawind%gcc+hypre~cuda" else cmd "nice -n19 spack manager snapshot -s exawind+hypre+openfast" fi diff --git a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py index 34cc495e..fa3bc805 100644 --- a/spack-scripting/scripting/cmd/manager_cmds/find_machine.py +++ b/spack-scripting/scripting/cmd/manager_cmds/find_machine.py @@ -99,6 +99,9 @@ def is_e4s(): ), # General "darwin": MachineData(lambda: sys.platform == "darwin", "darwin.nodomain.gov"), + "containercpu": MachineData( + lambda: os.environ["CONTAINER_BUILD"] == "cpu", "containcpu.nodomain.gov" + ), } From 67246ee0deafba7f748e17bb5581a15de86669ce Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Wed, 11 Oct 2023 09:43:41 -0700 Subject: [PATCH 13/15] set trilinoss@13.4.0, update configs --- configs/containercpu/config.yaml | 2 +- configs/containercpu/packages.yaml | 3 +++ env-templates/exawind_containercpu.yaml | 3 +-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/containercpu/config.yaml b/configs/containercpu/config.yaml index b7797366..874ad986 100644 --- a/configs/containercpu/config.yaml +++ b/configs/containercpu/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 20 + build_jobs: 32 diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index 9f3d3dfd..226609ba 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -45,6 +45,9 @@ packages: require: '@5.34.1' tioga: version: [develop] + trilinos: + require: + - any_of: ["@13.4.0", "@develop"] hypre: require: '@develop' variants: ~fortran diff --git a/env-templates/exawind_containercpu.yaml b/env-templates/exawind_containercpu.yaml index 2dfe68df..01fd9c68 100644 --- a/env-templates/exawind_containercpu.yaml +++ b/env-templates/exawind_containercpu.yaml @@ -6,5 +6,4 @@ spack: reuse: false view: false specs: - - 'exawind+hypre~amr_wind_gpu~nalu_wind_gpu~cuda' - #- 'exawind~hypre+amr_wind_gpu~nalu_wind_gpu+cuda cuda_arch=80 ^nalu-wind+hypre ^amr-wind~hypre' + - 'exawind+hypre~cuda' From 4bbf5d66cee681c1f6e961f7ca37dc3565733f91 Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Thu, 12 Oct 2023 09:00:24 -0700 Subject: [PATCH 14/15] packages.yaml: add Ubuntu libcurses-dev --- configs/containercpu/packages.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/containercpu/packages.yaml b/configs/containercpu/packages.yaml index cda3cc29..c1d18861 100644 --- a/configs/containercpu/packages.yaml +++ b/configs/containercpu/packages.yaml @@ -10,6 +10,11 @@ packages: variants: build_type=Release mpi: require: mpich +# Use Ubuntu libncurses-dev, b/c Spack version failes + ncurses: + externals: + - spec: ncurses@6.3 + prefix: /usr # Package preferences to be built by Spack for correct Exawind # Nota bene: use libtool from Spack for correct linking ascent: From aba5c2ab339b66c914c7fb8af340cbe839dece2f Mon Sep 17 00:00:00 2001 From: "Amy J. Powell" Date: Thu, 12 Oct 2023 10:58:15 -0700 Subject: [PATCH 15/15] Dockerfile-containercpu: For HPC CPU-only exawind --- .../Dockerfile-containercpu | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 hpc_containers/exawind_container_cpu/Dockerfile-containercpu diff --git a/hpc_containers/exawind_container_cpu/Dockerfile-containercpu b/hpc_containers/exawind_container_cpu/Dockerfile-containercpu new file mode 100644 index 00000000..c0066ec9 --- /dev/null +++ b/hpc_containers/exawind_container_cpu/Dockerfile-containercpu @@ -0,0 +1,92 @@ +MAINTAINER Philip Sakievich, Sandia National Laboratories +MAINTAINER Jon Rood, NREL + +ARG REGISTRY=docker.io/library +ARG IMAGE=ubuntu +ARG TAG=jammy + +FROM ${REGISTRY}/${IMAGE}:${TAG} + +# Make bash the default $SHELL +SHELL ["/bin/bash", "-c"] + +# Install Spack Prereqs: https://spack.readthedocs.io/en/latest/getting_started.html#system-prerequisites + +RUN apt-get update -yqq && \ + apt-get upgrade -yqq + +RUN apt-get install -yqq \ + ca-certificates \ + autoconf \ + automake \ + bzip2 \ + clangd \ + coreutils \ + curl \ + emacs-nox \ + file \ + flex \ + gcc \ + gcc-multilib \ + gcc-doc \ + g++ \ + gfortran \ + gfortran-multilib \ + gfortran-doc \ + git \ + git-doc \ + git-man \ + gnupg2 \ + libffi-dev \ + libfmt-dev \ + libgmp-dev \ + libjpeg-dev \ + libmpc-dev \ + libncurses-dev \ + libx11-dev \ + lsb-release \ + m4 \ + make \ + nano \ + openssl \ + patch \ + python3 \ + python3-distutils \ + python3-venv \ + unzip \ + vim \ + wget \ + wget2 \ + xz-utils \ + zip \ + zlib1g-dev + +RUN apt clean -yqq + +## Exawind CPU snapshot +WORKDIR /exawind-entry +##RUN git clone --recursive https://github.com/sandialabs/spack-manager + +## Pre-merge fork +RUN git clone --recursive https://github.com/ajpowelsnl/spack-manager +# +WORKDIR /exawind-entry/spack-manager +## Pre-merge branch from ajpowelsnl/spack-manager fork +RUN git checkout perlmutter/new_configs + +## Needed by "create-exawind-snapshot.sh" +ENV SPACK_MANAGER=/exawind-entry/spack-manager +ENV SPACK_MANAGER_MACHINE=containercpu +ENV CONTAINER_BUILD=cpu + +# Snapshot for CPU Exawind will be generated upon running container +RUN echo "export SPACK_MANAGER=$SPACK_MANAGER" >> /etc/bash.bashrc && \ + echo "source $SPACK_MANAGER/start.sh && spack-start" >> /etc/bash.bashrc && \ + echo "spack external find" >> /etc/bash.bashrc && \ + echo "$SPACK_MANAGER/scripts/create-exawind-snapshot.sh" >> /etc/bash.bashrc && \ + echo "spack clean -a" >> /etc/bash.bashrc && \ + echo "spack env activate -d snapshots/exawind/containercpu/$(date +%Y-%m-%d)" >> /etc/bash.bashrc && \ + echo "spack load exawind" >> /etc/bash.bashrc + + +CMD ["/usr/bin/bash"]