Skip to content

Commit

Permalink
Merge remote-tracking branch 'mbell/samurai/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
sjsprecious committed Jul 10, 2024
2 parents 330594b + dc6a2e0 commit f73d268
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 10 deletions.
2 changes: 1 addition & 1 deletion ncar_scripts/TDRP/hurricane_4panel.tdrp
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ projection = PROJ_TRANSVERSE_MERCATOR_EXACT;
//
// Type: string

data_directory = "/glade/campaign/cisl/asap/samurai/data/special/hurricane";
data_directory = "/glade/campaign/cisl/asap/samurai/data/special/hurricane_4panel";

///////////// output_directory ////////////////////////
//
Expand Down
3 changes: 3 additions & 0 deletions ncar_scripts/casper_a100_submit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ ID=`date '+%Y%m%d%H%M'`
##################
# Build the code #
##################
sed -i 's/cc70/cc80/g' CMakeLists.txt
sed -i 's/cc90/cc80/g' CMakeLists.txt


cd ncar_scripts
./ncar_build.sh gpu
Expand Down
39 changes: 39 additions & 0 deletions ncar_scripts/casper_h100_submit.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash -l
#PBS -N SAMURAI
#PBS -A NEOL0013
#PBS -l select=1:ncpus=36:ompthreads=1:mem=700GB:ngpus=1
#PBS -l gpu_type=h100
#PBS -q casper
#PBS -l walltime=05:00:00
#PBS -j oe
#PBS -k eod

cd $PBS_O_WORKDIR
cd ..
export SAMURAI_ROOT=$(pwd)

ID=`date '+%Y%m%d%H%M'`
##################
# Build the code #
##################
# Use cc90 for h100 GPU on Casper
sed -i 's/cc80/cc90/g' CMakeLists.txt
sed -i 's/cc70/cc90/g' CMakeLists.txt

cd ncar_scripts
./ncar_build.sh gpu

##############
# Run a case #
##############
suffix="casper_gpu"
for i in beltrami supercell hurricane typhoonChanthu2020 # hurricane_4panel

do
./ncar_run.sh $SAMURAI_ROOT/ncar_scripts/TDRP/${i}.tdrp >& log_${i}_$suffix.$ID
if [ ! -d ${i}_${suffix} ]; then
mkdir ${i}_${suffix}
fi
mv $SAMURAI_ROOT/run/timing.0 ${i}_${suffix}/timing.$ID
mv $SAMURAI_ROOT/run/samurai* log* ${i}_${suffix}
done
2 changes: 1 addition & 1 deletion ncar_scripts/casper_v100_submit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ cd $PBS_O_WORKDIR
cd ..
export SAMURAI_ROOT=$(pwd)

ID=`date '+%Y%m%d%H%M'
ID=`date '+%Y%m%d%H%M'`
##################
# Build the code #
##################
Expand Down
3 changes: 3 additions & 0 deletions ncar_scripts/derecho_a100_submit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ ID=`date '+%Y%m%d%H%M'`
##################
# Build the code #
##################
sed -i 's/cc70/cc80/g' CMakeLists.txt
sed -i 's/cc90/cc80/g' CMakeLists.txt

cd ncar_scripts
./ncar_build.sh gpu

Expand Down
17 changes: 10 additions & 7 deletions ncar_scripts/ncar_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@ else
module load ncarenv/23.09
fi
if [ "$MODE" == "GPU" ]; then
module load nvhpc/23.7
if [ "$NCAR_HOST" == "casper" ]; then
module load nvhpc/23.7
else
# Assume it is Derecho
module load nvhpc/24.3
fi
module load cuda/12.2.1
else
module load intel/2023.2.1
Expand All @@ -31,20 +36,18 @@ module load cmake/3.26.3
module load ncarcompilers/1.0.0

# Export the path to the pre-built LROSE library, which is currently pointed to Jian's directory
# Currently the available LROSE library is built by intel/2023.2.1 and nvhpc/23.7

if [ "$NCAR_HOST" == "casper" ]; then
if [ "$MODE" == "GPU" ]; then
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/casper/nvhpc23.7
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/casper/nvhpc/23.7/lrose-core-20240525
else
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/casper/intel2023.2.1
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/casper/intel/2023.2.1/lrose-core-20240525
fi
else
# Assume it is Derecho
if [ "$MODE" == "GPU" ]; then
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/derecho/nvhpc23.7
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/derecho/nvhpc/24.3/lrose-core-20240525
else
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/derecho/intel2023.2.1
export LROSE_INSTALL_DIR=/glade/work/sunjian/lrose/derecho/intel/2023.2.1/lrose-core-20240525
fi
fi

Expand Down
3 changes: 2 additions & 1 deletion ncar_scripts/ncar_run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

if [[ ! -v SAMURAI_ROOT ]]; then
echo "Please set the SAMURAI_ROOT environment variable first."
echo "Use export SAMURAI=/path_to_samurai_root_directory"
echo "Use export SAMURAI_ROOT=/path_to_samurai_root_directory"
exit 911
fi
cd $SAMURAI_ROOT
Expand Down Expand Up @@ -31,6 +31,7 @@ if [ "${GPU}" == "0" ]; then
export OMP_NUM_THREADS=128
fi
else
nvidia-smi
echo "GPU run; using 1 thread and 1 GPU"
export OMP_NUM_THREADS=1
fi
Expand Down

0 comments on commit f73d268

Please sign in to comment.