Merge pull request #316 from mabarnes/inbounds-loop-macro

@inbounds in loop macros

.github/workflows/parallel_test.yml

@@ -21,15 +21,15 @@ jobs:
       - uses: julia-actions/cache@v2
       - run: |
           touch Project.toml
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "PackageCompiler"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
-          julia --project -O3 --check-bounds=no -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")'
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")'
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()'
-          julia --project -O3 --check-bounds=no precompile.jl
+          julia --project -O3 -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "PackageCompiler"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
+          julia --project -O3 -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")'
+          julia --project -O3 -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")'
+          julia --project -O3 -e 'import Pkg; Pkg.precompile()'
+          julia --project -O3 precompile.jl
           # Need to use openmpi so that we can use `--oversubscribe` to allow using more MPI ranks than physical cores
-          ./mpiexecjl -np 3 --oversubscribe julia -J moment_kinetics.so --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --ci --debug 1
-          ./mpiexecjl -np 4 --oversubscribe julia -J moment_kinetics.so --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --ci --debug 1
-          ./mpiexecjl -np 2 --oversubscribe julia -J moment_kinetics.so --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --ci --debug 1 --long
+          ./mpiexecjl -np 3 --oversubscribe julia -J moment_kinetics.so --project -O3 moment_kinetics/test/runtests.jl --ci --debug 1
+          ./mpiexecjl -np 4 --oversubscribe julia -J moment_kinetics.so --project -O3 moment_kinetics/test/runtests.jl --ci --debug 1
+          ./mpiexecjl -np 2 --oversubscribe julia -J moment_kinetics.so --project -O3 moment_kinetics/test/runtests.jl --ci --debug 1 --long
           # Note: MPI.jl's default implementation is mpich, which has a similar option
           # `--with-device=ch3:sock`, but that needs to be set when compiling mpich.
         shell: bash
@@ -47,13 +47,13 @@ jobs:
       - uses: julia-actions/cache@v2
       - run: |
           touch Project.toml
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "PackageCompiler"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
-          julia --project -O3 --check-bounds=no -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")'
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")'
-          julia --project -O3 --check-bounds=no -e 'import Pkg; Pkg.precompile()'
-          julia --project -O3 --check-bounds=no precompile.jl
+          julia --project -O3 -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences", "PackageCompiler"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
+          julia --project -O3 -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")'
+          julia --project -O3 -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")'
+          julia --project -O3 -e 'import Pkg; Pkg.precompile()'
+          julia --project -O3 precompile.jl
           # Need to use openmpi so that we can use `--oversubscribe` to allow using more MPI ranks than physical cores
-          ./mpiexecjl -np 4 --oversubscribe julia -J moment_kinetics.so --project -O3 --check-bounds=no moment_kinetics/test/runtests.jl --ci --debug 1
+          ./mpiexecjl -np 4 --oversubscribe julia -J moment_kinetics.so --project -O3 moment_kinetics/test/runtests.jl --ci --debug 1
           # Note: MPI.jl's default implementation is mpich, which has a similar option
           # `--with-device=ch3:sock`, but that needs to be set when compiling mpich.
         shell: bash

docs/src/developing.md

@@ -231,6 +231,22 @@ communicator is `comm_block[]`).
 See also notes on debugging the 'anyv' parallelisation: [Collision operator and
 'anyv' region](@ref).
 
+## Bounds checking
+
+For best performance (i.e. 'production' runs), it is important that bounds
+checks not be included on array accesses. It should be possible to do this by
+running `julia` with the flag `--check-bounds=no`, but this flag has negative
+effects on the core Julia code and compiler, and works less well in Julia
+versions 1.10 and 1.11. As a workaround/alternative, the `@loop_*` macros
+described in the previous section wrap the contained code with an `@inbounds`
+macro (which disables bounds checks within the block, but the effect of
+`@inbounds` does not propagate down into functions called within the block). If
+performance-critical code that you write is within an `@loop`, then you do not
+need to do anything. However if it is not within an `@loop`, then you should
+add `@inbounds begin ... end` around any performance critical code. You can see
+examples of this being done in
+[`moment_kinetics.fokker_planck_calculus`](@ref).
+
 ## [Parallel I/O](@id parallel_io_section)
 
 The code provides an option to use parallel I/O, which allows all output to be

machines/archer/jobscript-precompile-no-run.template

@@ -22,6 +22,6 @@ export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile-no-run.jl
+bin/julia --project -O3 precompile-no-run.jl
 
 echo "finished! $(date)"

machines/archer/jobscript-precompile.template

@@ -22,6 +22,6 @@ export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile.jl
+bin/julia --project -O3 precompile.jl
 
 echo "finished! $(date)"

machines/archer/jobscript-restart.template

@@ -22,6 +22,6 @@ export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK
 
 echo "running INPUTFILE $(date)"
 
-srun --distribution=block:block --hint=nomultithread --ntasks=$SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
+srun --distribution=block:block --hint=nomultithread --ntasks=$SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
 
 echo "finished INPUTFILE $(date)"

machines/archer/jobscript-run.template

@@ -22,6 +22,6 @@ export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK
 
 echo "running INPUTFILE $(date)"
 
-srun --distribution=block:block --hint=nomultithread --ntasks=$SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl INPUTFILE
+srun --distribution=block:block --hint=nomultithread --ntasks=$SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl INPUTFILE
 
 echo "finished INPUTFILE $(date)"

machines/generic-batch-template/jobscript-precompile-no-run.template

@@ -15,6 +15,6 @@ source julia.env
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile-no-run.jl
+bin/julia --project -O3 precompile-no-run.jl
 
 echo "finished! $(date)"

machines/generic-batch-template/jobscript-precompile.template

@@ -15,6 +15,6 @@ source julia.env
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile.jl
+bin/julia --project -O3 precompile.jl
 
 echo "finished! $(date)"

machines/generic-batch-template/jobscript-restart.template

@@ -19,6 +19,6 @@ source julia.env
 echo "running INPUTFILE $(date)"
 
 # May need to change this if mpirun` is not what should be used on your system
-mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
+mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
 
 echo "finished INPUTFILE $(date)"

machines/generic-batch-template/jobscript-run.template

@@ -19,6 +19,6 @@ source julia.env
 echo "running INPUTFILE $(date)"
 
 # May need to change this if mpirun` is not what should be used on your system
-mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl INPUTFILE
+mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl INPUTFILE
 
 echo "finished INPUTFILE $(date)"

machines/machine_setup.sh

@@ -271,7 +271,7 @@ SEPARATE_POSTPROC_PROJECTS=$(bin/julia --project machines/shared/get_mk_preferen
 if [[ $BATCH_SYSTEM -eq 0 || $SEPARATE_POSTPROC_PROJECTS == "y" ]]; then
   # Batch systems can (conveniently) use different optimization flags for
   # running simulations and for post-processing.
-  OPTIMIZATION_FLAGS="-O3 --check-bounds=no"
+  OPTIMIZATION_FLAGS="-O3"
   POSTPROC_OPTIMIZATION_FLAGS="-O3"
 else
   # On interactive systems which use the same project for running simulations

machines/marconi/jobscript-precompile-no-run.template

@@ -15,6 +15,6 @@ source julia.env
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile-no-run.jl
+bin/julia --project -O3 precompile-no-run.jl
 
 echo "finished! $(date)"

machines/marconi/jobscript-precompile.template

@@ -15,6 +15,6 @@ source julia.env
 
 echo "precompiling $(date)"
 
-bin/julia --project -O3 --check-bounds=no precompile.jl
+bin/julia --project -O3 precompile.jl
 
 echo "finished! $(date)"

machines/marconi/jobscript-restart.template

@@ -18,6 +18,6 @@ source julia.env
 
 echo "running INPUTFILE $(date)"
 
-mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
+mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl --restart INPUTFILE RESTARTFROM
 
 echo "finished INPUTFILE $(date)"

machines/marconi/jobscript-run.template

@@ -18,6 +18,6 @@ source julia.env
 
 echo "running INPUTFILE $(date)"
 
-mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 --check-bounds=no run_moment_kinetics.jl INPUTFILE
+mpirun -np $SLURM_NTASKS bin/julia -Jmoment_kinetics.so --project -O3 run_moment_kinetics.jl INPUTFILE
 
 echo "finished INPUTFILE $(date)"

machines/shared/submit-restart.sh

@@ -123,7 +123,7 @@ sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s
 
 if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then
   # Check that source code has not been changed since moment_kinetics.so was created
-  bin/julia --project -O3 --check-bounds=no moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so
+  bin/julia --project -O3 moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so
 fi
 
 if [[ $SUBMIT -eq 0 ]]; then

machines/shared/submit-run.sh

@@ -118,7 +118,7 @@ sed -e "s|NODES|$NODES|" -e "s|RUNTIME|$RUNTIME|" -e "s|ACCOUNT|$ACCOUNT|" -e "s
 
 if [[ "$WARN_OLD_SYSIMAGE" -eq 0 ]]; then
   # Check that source code has not been changed since moment_kinetics.so was created
-  bin/julia --project -O3 --check-bounds=no moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so
+  bin/julia --project -O3 moment_kinetics/src/check_so_newer_than_code.jl moment_kinetics.so
 fi
 
 if [[ $SUBMIT -eq 0 ]]; then

makie_post_processing/makie_post_processing/src/chodura_condition.jl

@@ -1,3 +1,5 @@
+using moment_kinetics.analysis: check_Chodura_condition
+
 """
     Chodura_condition_plots(run_info::Tuple; plot_prefix)
     Chodura_condition_plots(run_info; plot_prefix=nothing, axes=nothing)

makie_post_processing/makie_post_processing/src/plots_for_variable.jl

@@ -53,9 +53,15 @@ function plots_for_variable(run_info, variable_name; plot_prefix, has_rdim=true,
     try
         variable = get_variable(run_info, variable_name)
     catch e
-        return makie_post_processing_error_handler(
-                   e,
-                   "plots_for_variable() failed for $variable_name - could not load data.")
+        if isa(e, KeyError)
+            println("Key $(e.key) not found when loading $variable_name - probably not "
+                    * "present in output")
+            return nothing
+        else
+            return makie_post_processing_error_handler(
+                       e,
+                       "plots_for_variable() failed for $variable_name - could not load data.")
+        end
     end
 
     if variable_name ∈ em_variables