diff --git a/dgx-2_scaling_benchmark.job b/dgx-2_scaling_benchmark.job index 458fa55..0cbb340 100644 --- a/dgx-2_scaling_benchmark.job +++ b/dgx-2_scaling_benchmark.job @@ -8,6 +8,20 @@ #SBATCH -e slurm.%N.%j.err # STDERR #SBATCH --reservation=martinls_6 + +# For Linux 64, Open MPI is built with CUDA awareness but this support is disabled by default. +# To enable it, please set the environment variable OMPI_MCA_opal_cuda_support=true before +# launching your MPI processes. Equivalently, you can set the MCA parameter in the command line: +# mpiexec --mca opal_cuda_support 1 ... +# +# In addition, the UCX support is also built but disabled by default. +# To enable it, first install UCX (conda install -c conda-forge ucx). Then, set the environment +# variables OMPI_MCA_pml="ucx" OMPI_MCA_osc="ucx" before launching your MPI processes. +# Equivalently, you can set the MCA parameters in the command line: +# mpiexec --mca pml ucx --mca osc ucx ... +# Note that you might also need to set UCX_MEMTYPE_CACHE=n for CUDA awareness via UCX. +# Please consult UCX's documentation for detail. + ulimit -s 10240 module load slurm/20.02.7 module load cuda11.2/toolkit/11.2.2 @@ -28,7 +42,7 @@ cd /work/$USER/$SLURM_JOB_ID/ShallowWaterGPU #mpirun --mca btl_openib_if_include mlx5_0 --mca btl_openib_warn_no_device_params_found 0 $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile #nsys profile -t nvtx,cuda mpirun -np $SLURM_NTASKS numactl --cpunodebind=0 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile #mpirun -np $SLURM_NTASKS numactl --cpunodebind=0 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile -mpirun -np $SLURM_NTASKS numactl --cpunodebind=1 $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile +mpirun -np $SLURM_NTASKS numactl --cpunodebind=1 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile cd $HOME/src/ShallowWaterGPU ## Copy files from work directory: