From a260230002ef62b2abbc63f51cbd63c7ce1f50eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Lilleeng=20S=C3=A6tra?= <martinls@met.no>
Date: Wed, 25 May 2022 14:52:44 +0200
Subject: [PATCH] Removed nvtx and MPI options for job script

---
 GPUSimulators/MPISimulator.py | 14 +++++++-------
 dgx-2_scaling_benchmark.job   |  4 +++-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py
index 4828a6c..c092c01 100644
--- a/GPUSimulators/MPISimulator.py
+++ b/GPUSimulators/MPISimulator.py
@@ -27,7 +27,7 @@ from mpi4py import MPI
 import time
 
 import pycuda.driver as cuda
-import nvtx
+#import nvtx
 
 
 
@@ -314,14 +314,14 @@ class MPISimulator(Simulator.BaseSimulator):
     
     def substep(self, dt, step_number):
         
-        nvtx.mark("substep start", color="yellow")
+        #nvtx.mark("substep start", color="yellow")
 
         self.profiling_data_mpi["start"]["t_mpi_step"] += time.time()
         
-        nvtx.mark("substep external", color="blue")
+        #nvtx.mark("substep external", color="blue")
         self.sim.substep(dt, step_number, external=True, internal=False) # only "internal ghost cells"
         
-        nvtx.mark("substep internal", color="red")
+        #nvtx.mark("substep internal", color="red")
         self.sim.substep(dt, step_number, internal=True, external=False) # "internal ghost cells" excluded
 
         #nvtx.mark("substep full", color="blue")
@@ -331,7 +331,7 @@ class MPISimulator(Simulator.BaseSimulator):
 
         self.profiling_data_mpi["end"]["t_mpi_step"] += time.time()
         
-        nvtx.mark("exchange", color="blue")
+        #nvtx.mark("exchange", color="blue")
         self.full_exchange()
 
         #nvtx.mark("download", color="blue")
@@ -343,10 +343,10 @@ class MPISimulator(Simulator.BaseSimulator):
         #nvtx.mark("upload", color="blue")
         #self.upload_for_exchange(self.sim.u0)
 
-        nvtx.mark("sync start", color="blue")
+        #nvtx.mark("sync start", color="blue")
         self.sim.stream.synchronize()
         self.sim.internal_stream.synchronize()
-        nvtx.mark("sync end", color="blue")
+        #nvtx.mark("sync end", color="blue")
         
         self.profiling_data_mpi["n_time_steps"] += 1
 
diff --git a/dgx-2_scaling_benchmark.job b/dgx-2_scaling_benchmark.job
index d607feb..03e9a4d 100644
--- a/dgx-2_scaling_benchmark.job
+++ b/dgx-2_scaling_benchmark.job
@@ -6,6 +6,7 @@
 #SBATCH -t 0-00:10                 # time (D-HH:MM)
 #SBATCH -o slurm.%N.%j.out  # STDOUT
 #SBATCH -e slurm.%N.%j.err  # STDERR
+#SBATCH --reservation=martinls_6
 
 ulimit -s 10240
 module load slurm/20.02.7
@@ -26,7 +27,8 @@ cp -r . /work/$USER/$SLURM_JOB_ID/ShallowWaterGPU
 cd /work/$USER/$SLURM_JOB_ID/ShallowWaterGPU
 #mpirun --mca btl_openib_if_include mlx5_0 --mca btl_openib_warn_no_device_params_found 0 $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile
 #nsys profile -t nvtx,cuda mpirun -np  $SLURM_NTASKS numactl --cpunodebind=0 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile
-mpirun -np  $SLURM_NTASKS numactl --cpunodebind=0 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile
+#mpirun -np  $SLURM_NTASKS numactl --cpunodebind=0 --localalloc $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile
+mpirun -np  $SLURM_NTASKS $HOME/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile
 cd $HOME/src/ShallowWaterGPU
 
 ## Copy files from work directory: