fix: missing bracket around multiple variables

(cherry picked from commit e954b5b181)

.gitignore

@@ -186,6 +186,7 @@ cython_debug/
 .pypirc
 
 # CUDA
+cuda_cache/
 
 # Taken from: https://github.com/github/gitignore/blob/main/CUDA.gitignore
 *.i

GPUSimulators/Common.py

@@ -35,8 +35,6 @@ import gc
 import netCDF4
 import json
 
-from tqdm import tqdm
-
 #import pycuda.compiler as cuda_compiler
 #import pycuda.gpuarray
 #import pycuda.driver as cuda
@@ -180,11 +178,11 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
         profiling_data_sim_runner["end"]["t_sim_init"] = time.time()
 
         #Start simulation loop
-        # progress_printer = ProgressPrinter(save_times[-1], print_every=10)
-        for k, t_step in tqdm(enumerate(t_steps), desc="Simulation Loop"):
+        progress_printer = ProgressPrinter(save_times[-1], print_every=10)
+        for k in range(len(save_times)):
             #Get target time and step size there
-            # t_step = t_steps[k]
-            # t_end = save_times[k]
+            t_step = t_steps[k]
+            t_end = save_times[k]
             
             #Sanity check simulator
             try:
@@ -196,7 +194,7 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
             profiling_data_sim_runner["start"]["t_full_step"] += time.time()
 
             #Simulate
-            if t_step > 0.0:
+            if (t_step > 0.0):
                 sim.simulate(t_step, dt)
 
             profiling_data_sim_runner["end"]["t_full_step"] += time.time()
@@ -213,11 +211,11 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
             profiling_data_sim_runner["end"]["t_nc_write"] += time.time()
 
             #Write progress to screen
-            # print_string = progress_printer.getPrintString(t_end)
-            # if (print_string):
-            #     logger.debug(print_string)
+            print_string = progress_printer.getPrintString(t_end)
+            if (print_string):
+                logger.debug(print_string)
                 
-        logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(save_times[-1], sim.simSteps(), sim.simTime() / sim.simSteps()))
+        logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps()))
         
     return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi
     #return outdata.filename
@@ -308,7 +306,7 @@ class IPEngine(object):
         import ipyparallel
         self.cluster = ipyparallel.Client()#profile='mpi')
         time.sleep(3)
-        while len(self.cluster.ids) != n_engines:
+        while(len(self.cluster.ids) != n_engines):
             time.sleep(0.5)
             self.logger.info("Waiting for cluster...")
             self.cluster = ipyparallel.Client()#profile='mpi')
@@ -435,58 +433,58 @@ class DataDumper(object):
         
 
 
-# class ProgressPrinter(object):
-#     """
-#     Small helper class for 
-#     """
-#     def __init__(self, total_steps, print_every=5):
-#         self.logger = logging.getLogger(__name__)
-#         self.start = time.time()
-#         self.total_steps = total_steps
-#         self.print_every = print_every
-#         self.next_print_time = self.print_every
-#         self.last_step = 0
-#         self.secs_per_iter = None
+class ProgressPrinter(object):
+    """
+    Small helper class for 
+    """
+    def __init__(self, total_steps, print_every=5):
+        self.logger = logging.getLogger(__name__)
+        self.start = time.time()
+        self.total_steps = total_steps
+        self.print_every = print_every
+        self.next_print_time = self.print_every
+        self.last_step = 0
+        self.secs_per_iter = None
         
-#     def getPrintString(self, step):
-#         elapsed =  time.time() - self.start
-#         if (elapsed > self.next_print_time):            
-#             dt = elapsed - (self.next_print_time - self.print_every)
-#             dsteps = step - self.last_step
-#             steps_remaining = self.total_steps - step
+    def getPrintString(self, step):
+        elapsed =  time.time() - self.start
+        if (elapsed > self.next_print_time):            
+            dt = elapsed - (self.next_print_time - self.print_every)
+            dsteps = step - self.last_step
+            steps_remaining = self.total_steps - step
                         
-#             if (dsteps == 0):
-#                 return
+            if (dsteps == 0):
+                return
                 
-#             self.last_step = step
-#             self.next_print_time = elapsed + self.print_every
+            self.last_step = step
+            self.next_print_time = elapsed + self.print_every
             
-#             if not self.secs_per_iter:
-#                 self.secs_per_iter = dt / dsteps
-#             self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps)
+            if not self.secs_per_iter:
+                self.secs_per_iter = dt / dsteps
+            self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps)
             
-#             remaining_time = steps_remaining * self.secs_per_iter
+            remaining_time = steps_remaining * self.secs_per_iter
 
-#             return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format(
-#                 ProgressPrinter.progressBar(step, self.total_steps), 
-#                 ProgressPrinter.timeString(elapsed + remaining_time), 
-#                 ProgressPrinter.timeString(elapsed), 
-#                 ProgressPrinter.timeString(remaining_time))
+            return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format(
+                ProgressPrinter.progressBar(step, self.total_steps), 
+                ProgressPrinter.timeString(elapsed + remaining_time), 
+                ProgressPrinter.timeString(elapsed), 
+                ProgressPrinter.timeString(remaining_time))
 
-#     def timeString(seconds):
-#         seconds = int(max(seconds, 1))
-#         minutes, seconds = divmod(seconds, 60)
-#         hours, minutes = divmod(minutes, 60)
-#         periods = [('h', hours), ('m', minutes), ('s', seconds)]
-#         time_string = ' '.join('{}{}'.format(value, name)
-#                                 for name, value in periods
-#                                 if value)
-#         return time_string
+    def timeString(seconds):
+        seconds = int(max(seconds, 1))
+        minutes, seconds = divmod(seconds, 60)
+        hours, minutes = divmod(minutes, 60)
+        periods = [('h', hours), ('m', minutes), ('s', seconds)]
+        time_string = ' '.join('{}{}'.format(value, name)
+                                for name, value in periods
+                                if value)
+        return time_string
 
-#     def progressBar(step, total_steps, width=30):
-#         progress = np.round(width * step / total_steps).astype(np.int32)
-#         progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%"
-#         return progressbar
+    def progressBar(step, total_steps, width=30):
+        progress = np.round(width * step / total_steps).astype(np.int32)
+        progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%"
+        return progressbar
 
 
 """

GPUSimulators/Simulator.py

@@ -25,7 +25,6 @@ import numpy as np
 import math
 import logging
 from enum import IntEnum
-from tqdm import tqdm
 
 #import pycuda.compiler as cuda_compiler
 #import pycuda.gpuarray
@@ -157,7 +156,7 @@ class BaseSimulator(object):
         self.num_substeps = num_substeps
         
         #Handle autotuning block size
-        if self.context.autotuner:
+        if (self.context.autotuner):
             peak_configuration = self.context.autotuner.get_peak_performance(self.__class__)
             block_width = int(peak_configuration["block_width"])
             block_height = int(peak_configuration["block_height"])
@@ -196,45 +195,42 @@ class BaseSimulator(object):
         Requires that the step() function is implemented in the subclasses
         """
 
-        # printer = Common.ProgressPrinter(t)
+        printer = Common.ProgressPrinter(t)
         
         t_start = self.simTime()
         t_end = t_start + t
         
         update_dt = True
-        if dt is not None:
+        if (dt is not None):
             update_dt = False
             self.dt = dt
         
-        for _ in tqdm(range(math.ceil((t_end - t_start) / self.dt)), desc="Simulation"):
+        while(self.simTime() < t_end):
             # Update dt every 100 timesteps and cross your fingers it works
             # for the next 100
-            # TODO this is probably broken now after fixing the "infinite" loop
-            if update_dt and (self.simSteps() % 100 == 0):
+            if (update_dt and (self.simSteps() % 100 == 0)):
                 self.dt = self.computeDt()*self.cfl_scale
-
+        
             # Compute timestep for "this" iteration (i.e., shorten last timestep)
             current_dt = np.float32(min(self.dt, t_end-self.simTime()))
 
             # Stop if end reached (should not happen)
-            if current_dt <= 0.0:
+            if (current_dt <= 0.0):
                 self.logger.warning("Timestep size {:d} is less than or equal to zero!".format(self.simSteps()))
                 break
-
+        
             # Step forward in time
             self.step(current_dt)
 
             #Print info
-            # print_string = printer.getPrintString(self.simTime() - t_start)
-            # if (print_string):
-            #     self.logger.info("%s: %s", self, print_string)
-            #     try:
-            #         self.check()
-            #     except AssertionError as e:
-            #         e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),)
-            #         raise
-
-        print("Done")
+            print_string = printer.getPrintString(self.simTime() - t_start)
+            if (print_string):
+                self.logger.info("%s: %s", self, print_string)
+                try:
+                    self.check()
+                except AssertionError as e:
+                    e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),)
+                    raise
 
 
     def step(self, dt):

Jobs/job_apptainer_lumi.slurm

@@ -1,26 +0,0 @@
-#!/bin/bash -l
-#SBATCH --job-name=lumi
-#SBATCH --account=project_4650000xx
-#SBATCH --time=00:10:00
-#SBATCH --partition=dev-g
-#SBATCH --nodes=1
-#SBATCH --ntasks-per-node=8
-#SBATCH --gpus-per-node=8
-#SBATCH --output=%x-%j.out
-#SBATCH --exclusive
-
-N=$SLURM_JOB_NUM_NODES
-echo "--nbr of nodes:", $N
-echo "--total nbr of gpus:", $SLURM_NTASKS
-
-MyDir=/project/project_4650000xx
-MyApplication=${MyDir}/FiniteVolumeGPU_HIP/mpiTesting.py
-Container=${MyDir}/FiniteVolumeGPU_HIP/my_container.sif
-
-CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
-
-export MPICH_GPU_SUPPORT_ENABLED=1
-
-srun --cpu-bind=${CPU_BIND} --mpi=pmi2 \
-     apptainer exec "${Container}" \
-     python ${MyApplication} -nx 1024 -ny 1024 --profile

Jobs/job_lumi.slrum

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+#SBATCH --job-name=lumi
+#SBATCH --account=project_4650000xx
+#SBATCH --time=00:10:00
+#SBATCH --partition=dev-g
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --gpus=8
+#SBATCH --gpus-per-node=8
+#SBATCH -o %x-%j.out
+#SBATCH --exclusive
+#
+
+N=$SLURM_JOB_NUM_NODES
+echo "--nbr of nodes:", $N
+echo "--total nbr of gpus:", $SLURM_NTASKS
+
+Mydir=/project/project_4650000xx
+Myapplication=${Mydir}/FiniteVolumeGPU_hip/mpiTesting.py
+
+#modules
+ml LUMI/24.03 partition/G
+ml lumi-container-wrapper
+ml cray-python/3.11.7
+ml rocm/5.4.6
+
+ml craype-accel-amd-gfx90a
+ml cray-mpich/8.1.29
+
+export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH"
+
+#missing library
+export LD_LIBRARY_PATH=/opt/cray/pe/mpich/8.1.29/ofi/cray/17.0/lib-abi-mpich:$LD_LIBRARY_PATH
+
+#Binding mask
+bind_mask="0x${fe}000000000000,0x${fe}00000000000000,0x${fe}0000,0x${fe}000000,0x${fe},0x${fe}00,0x${fe}00000000,0x${fe}0000000000"
+
+srun --cpu-bind=mask_cpu:$bind_mask \
+     python ${Myapplication} -nx 1024 -ny 1024 --profile 	

Jobs/job_lumi.slurm

@@ -1,27 +0,0 @@
-#!/bin/bash -l
-#SBATCH --job-name=lumi
-#SBATCH --account=project_4650000xx
-#SBATCH --time=00:10:00
-#SBATCH --partition=dev-g
-#SBATCH --nodes=1
-#SBATCH --ntasks-per-node=8
-#SBATCH --gpus-per-node=8
-#SBATCH --output=%x-%j.out
-#SBATCH --exclusive
-
-N=$SLURM_JOB_NUM_NODES
-echo "--nbr of nodes:", $N
-echo "--total nbr of gpus:", $SLURM_NTASKS
-
-MyDir=/project/project_4650000xx
-MyApplication=${MyDir}/FiniteVolumeGPU_HIP/mpiTesting.py
-CondaEnv=${MyDir}/FiniteVolumeGPU_HIP/MyCondaEnv/bin
-
-export PATH="${CondaEnv}:$PATH"
-
-CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
-
-export MPICH_GPU_SUPPORT_ENABLED=1
-
-srun --cpu-bind=${CPU_BIND} --mpi=pmi2 \
-     python ${MyApplication} -nx 1024 -ny 1024 --profile

README.md

@@ -5,53 +5,48 @@ This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/Fin
 ## Setup on LUMI-G
 Here is a step-by-step guide on installing packages on LUMI-G
 
-### Step 1: run conda-container
-Installation via conda can be done as:
-```shell
-ml LUMI/24.03 partition/G
-ml lumi-container-wrapper
+### Step 1: Install rocm-5.4.6 with Easybuild
+```
+export EBU_USER_PREFIX=/project/project_xxxxxx/EasyBuild
+ml LUMI/24.03 partition/G
+ml EasyBuild-user
+export PYTHONIOENCODING=utf-8
+eb rocm-5.4.6.eb -r
+```
+
+### Step 2: run conda-container
+Installation via conda can be done as:
+```
+ml LUMI/24.03 partition/G
+ml lumi-container-wrapper/0.3.3-cray-python-3.11.7
+```
 ```
-```shell
 conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml
 ```
 where the file `conda_environment_lumi.yml` contains packages to be installed.
 
-### Step 1 alternative: Convert to a singularity container with cotainr
-Load the required modules first
-```shell
-ml CrayEnv
-ml cotainr
+### Step 3: Set the env. variable to search for binaries
+```
+export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH"
+```
+### An alternative: Convert to a singularity container with cotainr
 ```
-
-Then build the Singularity/Apptainer container 
-```shell
 cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml
 ```
 
-### Step 2: Modify Slurm Job file
-Depending on your build method, update [`Jobs/job_lumi.slurm`](Jobs/job_lumi.slurm) if `conda-containerize` was used, or [`Jobs/job_apptainer_lumi.slurm`](Jobs/job_apptainer_lumi.slurm) if `containr` was used.
-
-In the job file, the required changes is to match your project allocation,
-and the directories of where the simulator and container is stored.
-
-### Step 3: Run the Slurm Job
-If `conda-containerize` was used for building:
-```shell
-sbatch Jobs/job_lumi.slurm
-```
-
-Otherwise, if `containr` was used for building:
-```shell
-sbatch Jobs/job_apptainer_lumi.slurm
-```
-
-### Troubleshooting
-
-#### Error when running MPI.
+### Error when running MPI.
 ```
 `MPI startup(): PMI server not found. Please set I_MPI_PMI_LIBRARY variable if it is not a singleton case.
 ```
 This can be resolved by exporting this:
 ```
-export I_MPI_PMI_LIBRARY=/opt/cray/pe/mpich/8.1.29/ofi/cray/17.0/lib/libmpi.so
-```
+export I_MPI_PMI_LIBRARY=/opt/cray/pe/mpich/8.1.27/ofi/cray/14.0/lib/libmpi.so
+```
+### Install hip-python
+```
+python -m pip install -i https://test.pypi.org/simple/ hip-python==5.4.3.470.16
+```
+
+The testing was done with this specific version `hip-python==5.4.3.470.16`
+
+ 

conda_environment_lumi.yml

@@ -13,9 +13,8 @@ dependencies:
 - pytools
 - netcdf4
 - scipy
-- tqdm
 - pip:
-  - hip-python==6.2.0.499.16
+  - hip-python==5.4.3.470.16
   - -i https://test.pypi.org/simple/ 
 
 

mpiTesting.py

@@ -70,7 +70,7 @@ def hip_check(call_result):
 
 args = parser.parse_args()
 
-if args.profile:
+if(args.profile):
     profiling_data = {}
     # profiling: total run time
     t_total_start = time.time()
@@ -79,8 +79,6 @@ if args.profile:
 
 # Get MPI COMM to use
 comm = MPI.COMM_WORLD
-size = comm.Get_size()
-rank = comm.Get_rank()
 
 
 ####
@@ -88,7 +86,7 @@ rank = comm.Get_rank()
 ####
 log_level_console = 20
 log_level_file = 10
-log_filename = 'mpi_' + str(rank) + '.log'
+log_filename = 'mpi_' + str(comm.rank) + '.log'
 logger = logging.getLogger('GPUSimulators')
 logger.setLevel(min(log_level_console, log_level_file))
 
@@ -112,7 +110,7 @@ logger.info("File logger using level %s to %s",
 # Initialize MPI grid etc
 ####
 logger.info("Creating MPI grid")
-grid = MPISimulator.MPIGrid(comm)
+grid = MPISimulator.MPIGrid(MPI.COMM_WORLD)
 
 """
 job_id = int(os.environ["SLURM_JOB_ID"])
@@ -154,7 +152,7 @@ gamma = 1.4
 #save_times = np.linspace(0, 0.000099, 11)
 #save_times = np.linspace(0, 0.000099, 2)
 save_times = np.linspace(0, 0.0000999, 2)
-outfile = "mpi_out_" + str(rank) + ".nc"
+outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc"
 save_var_names = ['rho', 'rho_u', 'rho_v', 'E']
 
 arguments = IC.genKelvinHelmholtz(nx, ny, gamma, grid=grid)
@@ -162,7 +160,7 @@ arguments['context'] = cuda_context
 arguments['theta'] = 1.2
 arguments['grid'] = grid
 
-if args.profile:
+if(args.profile):
     t_init_end = time.time()
     t_init = t_init_end - t_init_start
     profiling_data["t_init"] = t_init
@@ -183,14 +181,14 @@ def genSim(grid, **kwargs):
 (outfile, sim_runner_profiling_data, sim_profiling_data) = Common.runSimulation(
     genSim, arguments, outfile, save_times, save_var_names, dt)
 
-if args.profile:
+if(args.profile):
     t_total_end = time.time()
     t_total = t_total_end - t_total_start
     profiling_data["t_total"] = t_total
-    print("Total run time on rank " + str(rank) + " is " + str(t_total) + " s")
+    print("Total run time on rank " + str(MPI.COMM_WORLD.rank) + " is " + str(t_total) + " s")
 
 # write profiling to json file
-if args.profile and rank == 0:
+if(args.profile and MPI.COMM_WORLD.rank == 0):
     job_id = ""
     if "SLURM_JOB_ID" in os.environ:
         job_id = int(os.environ["SLURM_JOB_ID"])
@@ -201,7 +199,7 @@ if args.profile and rank == 0:
             str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json"
         profiling_data["outfile"] = outfile
     else:
-        profiling_file = "MPI_" + str(size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json"
+        profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json"
 
     for stage in sim_runner_profiling_data["start"].keys():
         profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage]
@@ -216,7 +214,7 @@ if args.profile and rank == 0:
 
     profiling_data["slurm_job_id"] = job_id
     profiling_data["n_cuda_devices"] = str(num_cuda_devices)
-    profiling_data["n_processes"] = str(size)
+    profiling_data["n_processes"] = str(MPI.COMM_WORLD.size)
     profiling_data["git_hash"] = Common.getGitHash()
     profiling_data["git_status"] = Common.getGitStatus()