Merge pull request #1 from HichamAgueny/implement-hip-python

Implement hip python

GPUSimulators/Autotuner.py

@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the different helper functions and 
+classes
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os
+import gc
+import numpy as np
+import logging
+from socket import gethostname
+
+#import pycuda.driver as cuda
+from hip import hip,hiprtc
+
+from GPUSimulators import Common, Simulator, CudaContext
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+
+class Autotuner:
+    def __init__(self, 
+                nx=2048, ny=2048, 
+                block_widths=range(8, 32, 1),
+                block_heights=range(8, 32, 1)):
+        logger = logging.getLogger(__name__)
+        self.filename = "autotuning_data_" + gethostname() + ".npz"
+        self.nx = nx
+        self.ny = ny
+        self.block_widths = block_widths
+        self.block_heights = block_heights
+        self.performance = {}
+
+
+    def benchmark(self, simulator, force=False):
+        logger = logging.getLogger(__name__)
+        
+        #Run through simulators and benchmark
+        key = str(simulator.__name__)
+        logger.info("Benchmarking %s to %s", key, self.filename)
+        
+        #If this simulator has been benchmarked already, skip it
+        if (force==False and os.path.isfile(self.filename)):
+            with np.load(self.filename) as data:
+                if key in data["simulators"]:
+                    logger.info("%s already benchmarked - skipping", key)
+                    return
+    
+        # Set arguments to send to the simulators during construction
+        context = CudaContext.CudaContext(autotuning=False)
+        g = 9.81
+        h0, hu0, hv0, dx, dy, dt = Autotuner.gen_test_data(nx=self.nx, ny=self.ny, g=g)
+        arguments = {
+            'context': context,
+            'h0': h0, 'hu0': hu0, 'hv0': hv0,
+            'nx': self.nx, 'ny': self.ny,
+            'dx': dx, 'dy': dy, 'dt': 0.9*dt,
+            'g': g
+        } 
+             
+        # Load existing data into memory
+        benchmark_data = {
+                "simulators": [],
+        }
+        if (os.path.isfile(self.filename)):
+            with np.load(self.filename) as data:
+                for k, v in data.items():
+                    benchmark_data[k] = v
+   
+        # Run benchmark
+        benchmark_data[key + "_megacells"] = Autotuner.benchmark_single_simulator(simulator, arguments, self.block_widths, self.block_heights)
+        benchmark_data[key + "_block_widths"] = self.block_widths
+        benchmark_data[key + "_block_heights"] = self.block_heights
+        benchmark_data[key + "_arguments"] = str(arguments)
+        
+        existing_sims = benchmark_data["simulators"]
+        if (isinstance(existing_sims, np.ndarray)):
+            existing_sims = existing_sims.tolist()
+        if (key not in existing_sims):
+            benchmark_data["simulators"] = existing_sims + [key]
+
+        # Save to file
+        np.savez_compressed(self.filename, **benchmark_data)
+
+
+            
+    """
+    Function which reads a numpy file with autotuning data
+    and reports the maximum performance and block size
+    """
+    def get_peak_performance(self, simulator):
+        logger = logging.getLogger(__name__)
+        
+        assert issubclass(simulator, Simulator.BaseSimulator)
+        key = simulator.__name__
+        
+        if (key in self.performance):
+            return self.performance[key]
+        else:
+            #Run simulation if required
+            if (not os.path.isfile(self.filename)):
+                logger.debug("Could not get autotuned peak performance for %s: benchmarking", key)
+                self.benchmark(simulator)
+            
+            with np.load(self.filename) as data:
+                if key not in data['simulators']:
+                    logger.debug("Could not get autotuned peak performance for %s: benchmarking", key)
+                    data.close()
+                    self.benchmark(simulator)
+                    data = np.load(self.filename)
+                
+                def find_max_index(megacells):
+                    max_index = np.nanargmax(megacells)
+                    return np.unravel_index(max_index, megacells.shape)
+                
+                megacells = data[key + '_megacells']
+                block_widths = data[key + '_block_widths']
+                block_heights = data[key + '_block_heights']
+                j, i = find_max_index(megacells)
+                
+                self.performance[key] = { "block_width": block_widths[i],
+                                         "block_height": block_heights[j],
+                                         "megacells": megacells[j, i] }
+                logger.debug("Returning %s as peak performance parameters", self.performance[key])
+                return self.performance[key]
+        
+            #This should never happen
+            raise "Something wrong: Could not get autotuning data!"
+            return None
+        
+        
+                
+    """
+    Runs a set of benchmarks for a single simulator
+    """
+    def benchmark_single_simulator(simulator, arguments, block_widths, block_heights):
+        logger = logging.getLogger(__name__)
+        
+        megacells = np.empty((len(block_heights), len(block_widths)))
+        megacells.fill(np.nan)
+
+        logger.debug("Running %d benchmarks with %s", len(block_heights)*len(block_widths), simulator.__name__)
+        
+        sim_arguments = arguments.copy()
+                    
+        with Common.Timer(simulator.__name__) as t:
+            for j, block_height in enumerate(block_heights):
+                sim_arguments.update({'block_height': block_height})
+                for i, block_width in enumerate(block_widths):
+                    sim_arguments.update({'block_width': block_width})
+                    megacells[j, i] = Autotuner.run_benchmark(simulator, sim_arguments)
+                        
+
+        logger.debug("Completed %s in %f seconds", simulator.__name__, t.secs)
+
+        return megacells
+            
+            
+    """
+    Runs a benchmark, and returns the number of megacells achieved
+    """
+    def run_benchmark(simulator, arguments, timesteps=10, warmup_timesteps=2):
+        logger = logging.getLogger(__name__)
+        
+        #Initialize simulator
+        try:
+            sim = simulator(**arguments)
+        except:
+            #An exception raised - not possible to continue
+            logger.debug("Failed creating %s with arguments %s", simulator.__name__, str(arguments))
+            return np.nan
+        
+        #Create timer events
+        #start = cuda.Event()
+        #end = cuda.Event()
+        stream = hip_check(hip.hipStreamCreate())
+
+        start = hip_check(hip.hipEventCreate())
+        end = hip_check(hip.hipEventCreate())
+
+        #Warmup
+        for i in range(warmup_timesteps):
+            sim.stepEuler(sim.dt)
+            
+        #Run simulation with timer        
+        #start.record(sim.stream)
+        #start recording
+        hip_check(hip.hipEventRecord(start, stream))
+        for i in range(timesteps):
+            sim.stepEuler(sim.dt)
+        #end.record(sim.stream)
+        #stop recording and synchronize
+        hip_check(hip.hipEventRecord(end, stream))
+
+        #Synchronize end event
+        #end.synchronize()
+        hip_check(hip.hipEventSynchronize(end))
+
+        #Compute megacells
+        #gpu_elapsed = end.time_since(start)*1.0e-3
+        gpu_elapsed = hip_check(hip.hipEventElapsedTime(start, end))
+
+        megacells = (sim.nx*sim.ny*timesteps / (1000*1000)) / gpu_elapsed
+
+        #Sanity check solution
+        h, hu, hv = sim.download()
+        sane = True
+        sane = sane and Autotuner.sanity_check(h, 0.3, 0.7)
+        sane = sane and Autotuner.sanity_check(hu, -0.2, 0.2)
+        sane = sane and Autotuner.sanity_check(hv, -0.2, 0.2)
+        
+        if (sane):
+            logger.debug("%s [%d x %d] succeeded: %f megacells, gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], megacells, gpu_elapsed)
+            return megacells
+        else:
+            logger.debug("%s [%d x %d] failed: gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], gpu_elapsed)
+            return np.nan
+        
+        
+        
+    """
+    Generates test dataset
+    """
+    def gen_test_data(nx, ny, g):
+        width = 100.0
+        height = 100.0
+        dx = width / float(nx)
+        dy = height / float(ny)
+
+        x_center = dx*nx/2.0
+        y_center = dy*ny/2.0
+
+        #Create a gaussian "dam break" that will not form shocks
+        size = width / 5.0
+        dt = 10**10
+        
+        h  = np.zeros((ny, nx), dtype=np.float32); 
+        hu = np.zeros((ny, nx), dtype=np.float32);
+        hv = np.zeros((ny, nx), dtype=np.float32);
+
+        extent = 1.0/np.sqrt(2.0)
+        x = (dx*(np.arange(0, nx, dtype=np.float32)+0.5) - x_center) / size
+        y = (dy*(np.arange(0, ny, dtype=np.float32)+0.5) - y_center) / size
+        xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')
+        r = np.minimum(1.0, np.sqrt(xv**2 + yv**2))
+        xv = None
+        yv = None
+        gc.collect()
+
+        #Generate highres
+        cos = np.cos(np.pi*r)
+        h = 0.5 + 0.1*0.5*(1.0 + cos)
+        hu = 0.1*0.5*(1.0 + cos)
+        hv = hu.copy()
+        
+        scale = 0.7
+        max_h_estimate = 0.6
+        max_u_estimate = 0.1*np.sqrt(2.0)
+        dx = width/nx
+        dy = height/ny
+        dt = scale * min(dx, dy) / (max_u_estimate + np.sqrt(g*max_h_estimate))
+        
+        return h, hu, hv, dx, dy, dt
+        
+    """
+    Checks that a variable is "sane"
+    """
+    def sanity_check(variable, bound_min, bound_max):
+        maxval = np.amax(variable)
+        minval = np.amin(variable)
+        if (np.isnan(maxval) 
+                or np.isnan(minval)
+                or maxval > bound_max
+                or minval < bound_min):
+            return False
+        else:
+            return True

GPUSimulators/Common.py

@@ -0,0 +1,958 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the different helper functions and 
+classes
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os
+
+import numpy as np
+import time
+import signal
+import subprocess
+import tempfile
+import re
+import io
+import hashlib
+import logging
+import gc
+import netCDF4
+import json
+
+#import pycuda.compiler as cuda_compiler
+#import pycuda.gpuarray
+#import pycuda.driver as cuda
+#from pycuda.tools import PageLockedMemoryPool
+
+from hip import hip, hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+        err = call_result[0]
+        result = call_result[1:]
+        if len(result) == 1:
+            result = result[0]
+        if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+            raise RuntimeError(str(err))
+        elif (
+            isinstance(err, hiprtc.hiprtcResult)
+            and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+        ):
+            raise RuntimeError(str(err))
+        return result
+
+def safeCall(cmd):
+    logger = logging.getLogger(__name__)
+    try:
+        #git rev-parse HEAD
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        params = dict()
+        params['stderr'] = subprocess.STDOUT
+        params['cwd'] = current_dir
+        params['universal_newlines'] = True #text=True in more recent python
+        params['shell'] = False
+        if os.name == 'nt':
+            params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+        stdout = subprocess.check_output(cmd, **params)
+    except subprocess.CalledProcessError as e:
+        output = e.output
+        logger.error("Git failed, \nReturn code: " + str(e.returncode) + "\nOutput: " + output)
+        raise e
+
+    return stdout
+
+def getGitHash():
+    return safeCall(["git", "rev-parse", "HEAD"])
+
+def getGitStatus():
+    return safeCall(["git", "status", "--porcelain", "-uno"])
+
+def toJson(in_dict, compressed=True):
+    """
+    Creates JSON string from a dictionary
+    """
+    logger = logging.getLogger(__name__)
+    out_dict = in_dict.copy()
+    for key in out_dict:
+        if isinstance(out_dict[key], np.ndarray):
+            out_dict[key] = out_dict[key].tolist()
+        else:
+            try:
+                json.dumps(out_dict[key])
+            except:
+                value = str(out_dict[key])
+                logger.warning("JSON: Converting {:s} to string ({:s})".format(key, value))
+                out_dict[key] = value
+    return json.dumps(out_dict)
+
+def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names=[], dt=None):
+    """
+    Runs a simulation, and stores output in netcdf file. Stores the times given in 
+    save_times, and saves all of the variables in list save_var_names. Elements in  
+    save_var_names can be set to None if you do not want to save them
+    """
+    profiling_data_sim_runner = { 'start': {}, 'end': {} }
+    profiling_data_sim_runner["start"]["t_sim_init"] = 0
+    profiling_data_sim_runner["end"]["t_sim_init"] = 0
+    profiling_data_sim_runner["start"]["t_nc_write"] = 0
+    profiling_data_sim_runner["end"]["t_nc_write"] = 0
+    profiling_data_sim_runner["start"]["t_full_step"] = 0
+    profiling_data_sim_runner["end"]["t_full_step"] = 0
+
+    profiling_data_sim_runner["start"]["t_sim_init"] = time.time()
+
+    logger = logging.getLogger(__name__)
+
+    assert len(save_times) > 0, "Need to specify which times to save"
+
+    with Timer("construct") as t:
+        sim = simulator(**simulator_args)
+    logger.info("Constructed in " + str(t.secs) + " seconds")
+
+    #Create netcdf file and simulate
+    with DataDumper(outfile, mode='w', clobber=False) as outdata:
+        
+        #Create attributes (metadata)
+        outdata.ncfile.created = time.ctime(time.time())
+        outdata.ncfile.git_hash = getGitHash()
+        outdata.ncfile.git_status = getGitStatus()
+        outdata.ncfile.simulator = str(simulator)
+        
+        # do not write fields to attributes (they are to large)
+        simulator_args_for_ncfile = simulator_args.copy()
+        del simulator_args_for_ncfile["rho"]
+        del simulator_args_for_ncfile["rho_u"]
+        del simulator_args_for_ncfile["rho_v"]
+        del simulator_args_for_ncfile["E"]
+        outdata.ncfile.sim_args = toJson(simulator_args_for_ncfile)
+        
+        #Create dimensions
+        outdata.ncfile.createDimension('time', len(save_times))
+        outdata.ncfile.createDimension('x', simulator_args['nx'])
+        outdata.ncfile.createDimension('y', simulator_args['ny'])
+
+        #Create variables for dimensions
+        ncvars = {}
+        ncvars['time'] = outdata.ncfile.createVariable('time', np.dtype('float32').char, 'time')
+        ncvars['x']    = outdata.ncfile.createVariable(   'x', np.dtype('float32').char,    'x')
+        ncvars['y']    = outdata.ncfile.createVariable(   'y', np.dtype('float32').char,    'y')
+        
+        #Fill variables with proper values
+        ncvars['time'][:] = save_times
+        extent = sim.getExtent()
+        ncvars['x'][:] = np.linspace(extent[0], extent[1], simulator_args['nx'])
+        ncvars['y'][:] = np.linspace(extent[2], extent[3], simulator_args['ny'])
+    
+        #Choose which variables to download (prune None from list, but keep the index)
+        download_vars = []
+        for i, var_name in enumerate(save_var_names):
+            if var_name is not None:
+                download_vars += [i]
+        save_var_names = list(save_var_names[i] for i in download_vars)
+        
+        #Create variables
+        for var_name in save_var_names:
+            ncvars[var_name] = outdata.ncfile.createVariable(var_name, np.dtype('float32').char, ('time', 'y', 'x'), zlib=True, least_significant_digit=3)
+
+        #Create step sizes between each save
+        t_steps = np.empty_like(save_times)
+        t_steps[0] = save_times[0]
+        t_steps[1:] = save_times[1:] - save_times[0:-1]
+
+        profiling_data_sim_runner["end"]["t_sim_init"] = time.time()
+
+        #Start simulation loop
+        progress_printer = ProgressPrinter(save_times[-1], print_every=10)
+        for k in range(len(save_times)):
+            #Get target time and step size there
+            t_step = t_steps[k]
+            t_end = save_times[k]
+            
+            #Sanity check simulator
+            try:
+                sim.check()
+            except AssertionError as e:
+                logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e)))
+                return outdata.filename
+
+            profiling_data_sim_runner["start"]["t_full_step"] += time.time()
+
+            #Simulate
+            if (t_step > 0.0):
+                sim.simulate(t_step, dt)
+
+            profiling_data_sim_runner["end"]["t_full_step"] += time.time()
+
+            profiling_data_sim_runner["start"]["t_nc_write"] += time.time()
+
+            #Download
+            save_vars = sim.download(download_vars)
+
+            #Save to file
+            for i, var_name in enumerate(save_var_names):
+                ncvars[var_name][k, :] = save_vars[i]
+
+            profiling_data_sim_runner["end"]["t_nc_write"] += time.time()
+
+            #Write progress to screen
+            print_string = progress_printer.getPrintString(t_end)
+            if (print_string):
+                logger.debug(print_string)
+                
+        logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps()))
+        
+    return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi
+    #return outdata.filename
+
+
+class Timer(object):
+    """
+    Class which keeps track of time spent for a section of code
+    """
+    def __init__(self, tag, log_level=logging.DEBUG):
+        self.tag = tag
+        self.log_level = log_level
+        self.logger = logging.getLogger(__name__)
+        
+    def __enter__(self):
+        self.start = time.time()
+        return self
+    
+    def __exit__(self, *args):
+        self.end = time.time()
+        self.secs = self.end - self.start
+        self.msecs = self.secs * 1000 # millisecs
+        self.logger.log(self.log_level, "%s: %f ms", self.tag, self.msecs)
+
+    def elapsed(self):
+        return time.time() - self.start
+            
+
+class PopenFileBuffer(object):
+    """
+    Simple class for holding a set of tempfiles
+    for communicating with a subprocess
+    """
+    def __init__(self):
+        self.stdout = tempfile.TemporaryFile(mode='w+t')
+        self.stderr = tempfile.TemporaryFile(mode='w+t')
+
+    def __del__(self):
+        self.stdout.close()
+        self.stderr.close()
+
+    def read(self):
+        self.stdout.seek(0)
+        cout = self.stdout.read()
+        self.stdout.seek(0, 2)
+
+        self.stderr.seek(0)
+        cerr = self.stderr.read()
+        self.stderr.seek(0, 2)
+
+        return cout, cerr
+
+class IPEngine(object):
+    """
+    Class for starting IPEngines for MPI processing in IPython
+    """
+    def __init__(self, n_engines):
+        self.logger = logging.getLogger(__name__)
+        
+        #Start ipcontroller
+        self.logger.info("Starting IPController")
+        self.c_buff = PopenFileBuffer()
+        c_cmd = ["ipcontroller",  "--ip='*'"]
+        c_params = dict()
+        c_params['stderr'] = self.c_buff.stderr
+        c_params['stdout'] = self.c_buff.stdout
+        c_params['shell'] = False
+        if os.name == 'nt':
+            c_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+        self.c = subprocess.Popen(c_cmd, **c_params)
+        
+        #Wait until controller is running
+        time.sleep(3)
+        
+        #Start engines
+        self.logger.info("Starting IPEngines")
+        self.e_buff = PopenFileBuffer()
+        e_cmd = ["mpiexec", "-n", str(n_engines), "ipengine", "--mpi"]
+        e_params = dict()
+        e_params['stderr'] = self.e_buff.stderr
+        e_params['stdout'] = self.e_buff.stdout
+        e_params['shell'] = False
+        if os.name == 'nt':
+            e_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+        self.e = subprocess.Popen(e_cmd, **e_params)
+
+        # attach to a running cluster
+        import ipyparallel
+        self.cluster = ipyparallel.Client()#profile='mpi')
+        time.sleep(3)
+        while(len(self.cluster.ids) != n_engines):
+            time.sleep(0.5)
+            self.logger.info("Waiting for cluster...")
+            self.cluster = ipyparallel.Client()#profile='mpi')
+        
+        self.logger.info("Done")
+        
+    def __del__(self):
+        self.shutdown()
+    
+    def shutdown(self):
+        if (self.e is not None):
+            if (os.name == 'nt'):
+                self.logger.warn("Sending CTRL+C to IPEngine")
+                self.e.send_signal(signal.CTRL_C_EVENT)
+                
+            try:
+                self.e.communicate(timeout=3)
+                self.e.kill()
+            except subprocess.TimeoutExpired:
+                self.logger.warn("Killing IPEngine")
+                self.e.kill()
+                self.e.communicate()
+            self.e = None
+                
+            cout, cerr = self.e_buff.read()
+            self.logger.info("IPEngine cout: {:s}".format(cout))
+            self.logger.info("IPEngine cerr: {:s}".format(cerr))
+            self.e_buff = None
+            
+            gc.collect()
+            
+        if (self.c is not None):
+            if (os.name == 'nt'):
+                self.logger.warn("Sending CTRL+C to IPController")
+                self.c.send_signal(signal.CTRL_C_EVENT)
+                
+            try:
+                self.c.communicate(timeout=3)
+                self.c.kill()
+            except subprocess.TimeoutExpired:
+                self.logger.warn("Killing IPController")
+                self.c.kill()
+                self.c.communicate()
+            self.c = None
+                
+            cout, cerr = self.c_buff.read()
+            self.logger.info("IPController cout: {:s}".format(cout))
+            self.logger.info("IPController cerr: {:s}".format(cerr))
+            self.c_buff = None
+        
+            gc.collect()
+        
+
+class DataDumper(object):
+    """
+    Simple class for holding a netCDF4 object
+    (handles opening and closing in a nice way)
+    Use as 
+    with DataDumper("filename") as data:
+        ...
+    """
+    def __init__(self, filename, *args, **kwargs):
+        self.logger = logging.getLogger(__name__)
+        
+        #Create directory if needed
+        filename = os.path.abspath(filename)
+        dirname = os.path.dirname(filename)
+        if dirname and not os.path.isdir(dirname):
+            self.logger.info("Creating directory " + dirname)
+            os.makedirs(dirname)
+        
+        #Get mode of file if we have that
+        mode = None
+        if (args):
+            mode = args[0]
+        elif (kwargs and 'mode' in kwargs.keys()):
+            mode = kwargs['mode']
+            
+        #Create new unique file if writing
+        if (mode):
+            if (("w" in mode) or ("+" in mode) or ("a" in mode)):
+                i = 0
+                stem, ext = os.path.splitext(filename)
+                while (os.path.isfile(filename)):
+                    filename = "{:s}_{:04d}{:s}".format(stem, i, ext)
+                    i = i+1
+        self.filename = os.path.abspath(filename)
+        
+        #Save arguments
+        self.args = args
+        self.kwargs = kwargs
+                
+        #Log output
+        self.logger.info("Initialized " + self.filename)
+        
+        
+    def __enter__(self):
+        self.logger.info("Opening " + self.filename)
+        if (self.args):
+            self.logger.info("Arguments: " + str(self.args))
+        if (self.kwargs):
+            self.logger.info("Keyword arguments: " + str(self.kwargs))
+        self.ncfile = netCDF4.Dataset(self.filename, *self.args, **self.kwargs)
+        return self
+        
+    def __exit__(self, *args):
+        self.logger.info("Closing " + self.filename)
+        self.ncfile.close()
+        
+        
+    def toJson(in_dict):
+        out_dict = in_dict.copy()
+
+        for key in out_dict:
+            if isinstance(out_dict[key], np.ndarray):
+                out_dict[key] = out_dict[key].tolist()
+            else:
+                try:
+                    json.dumps(out_dict[key])
+                except:
+                    out_dict[key] = str(out_dict[key])
+
+        return json.dumps(out_dict)
+        
+
+
+class ProgressPrinter(object):
+    """
+    Small helper class for 
+    """
+    def __init__(self, total_steps, print_every=5):
+        self.logger = logging.getLogger(__name__)
+        self.start = time.time()
+        self.total_steps = total_steps
+        self.print_every = print_every
+        self.next_print_time = self.print_every
+        self.last_step = 0
+        self.secs_per_iter = None
+        
+    def getPrintString(self, step):
+        elapsed =  time.time() - self.start
+        if (elapsed > self.next_print_time):            
+            dt = elapsed - (self.next_print_time - self.print_every)
+            dsteps = step - self.last_step
+            steps_remaining = self.total_steps - step
+                        
+            if (dsteps == 0):
+                return
+                
+            self.last_step = step
+            self.next_print_time = elapsed + self.print_every
+            
+            if not self.secs_per_iter:
+                self.secs_per_iter = dt / dsteps
+            self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps)
+            
+            remaining_time = steps_remaining * self.secs_per_iter
+
+            return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format(
+                ProgressPrinter.progressBar(step, self.total_steps), 
+                ProgressPrinter.timeString(elapsed + remaining_time), 
+                ProgressPrinter.timeString(elapsed), 
+                ProgressPrinter.timeString(remaining_time))
+
+    def timeString(seconds):
+        seconds = int(max(seconds, 1))
+        minutes, seconds = divmod(seconds, 60)
+        hours, minutes = divmod(minutes, 60)
+        periods = [('h', hours), ('m', minutes), ('s', seconds)]
+        time_string = ' '.join('{}{}'.format(value, name)
+                                for name, value in periods
+                                if value)
+        return time_string
+
+    def progressBar(step, total_steps, width=30):
+        progress = np.round(width * step / total_steps).astype(np.int32)
+        progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%"
+        return progressbar
+
+
+"""
+Class that holds 2D data 
+"""
+class CudaArray2D:
+    """
+    Uploads initial data to the CUDA device
+    """
+    def __init__(self, stream, nx, ny, x_halo, y_halo, cpu_data=None, dtype=np.float32):
+        self.logger =  logging.getLogger(__name__)
+        self.nx = nx
+        self.ny = ny
+        self.x_halo = x_halo
+        self.y_halo = y_halo
+        
+        nx_halo = nx + 2*x_halo
+        ny_halo = ny + 2*y_halo
+      
+        #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny)
+        #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here
+        #Initialize an array on GPU with zeros
+        #self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype)
+        #data.strides[0] == nx_halo*np.float32().itemsize
+        #data.strides[1] == np.float32().itemsize
+        num_bytes = ny_halo*nx_halo * np.float32().itemsize
+        
+        #data_h  = np.zeros((ny_halo, nx_halo), dtype)
+        # init device array and upload host data
+        self.data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(ny_halo, nx_halo))
+
+        #num_bytes = ny*nx * np.float32().itemsize
+        #cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable))
+        # copy data from host to device
+        #hip_check(hip.hipMemcpy(self.data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice))
+
+        #https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemPoolCreate
+        #self.memorypool = PageLockedMemoryPool()
+       
+        #If we don't have any data, just allocate and return
+        if cpu_data is None:
+            return
+            
+        #Make sure data is in proper format
+        assert cpu_data.shape == (ny_halo, nx_halo) or cpu_data.shape == (self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.ny, self.nx)), str((ny_halo, nx_halo)))
+        assert cpu_data.itemsize == 4, "Wrong size of data type"
+        assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)"
+
+
+        #Create copy object from host to device
+        x = (nx_halo - cpu_data.shape[1]) // 2
+        y = (ny_halo - cpu_data.shape[0]) // 2
+        self.upload(stream, cpu_data, extent=[x, y, cpu_data.shape[1], cpu_data.shape[0]])
+
+        #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny)
+
+        
+    def __del__(self, *args):
+        #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny)
+        #self.data.gpudata.free()
+        #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data), self.nx, self.ny)
+        hip_check(hip.hipFree(self.data))
+        #hip_check(hip.hipFreeAsync(self.data, self.stream))
+        self.data = None
+        
+    """
+    Enables downloading data from GPU to Python
+    """
+    def download(self, stream, cpu_data=None, asynch=False, extent=None):
+        if (extent is None):
+            x = self.x_halo
+            y = self.y_halo
+            nx = self.nx
+            ny = self.ny
+        else:
+            x, y, nx, ny = extent
+        
+        if (cpu_data is None):
+            #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny)
+            #Allocate host memory
+            #The following fails, don't know why (crashes python)
+            #allocate a pinned (page-locked) memory array
+            #cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE)
+            #see here type of memory: https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemoryType
+            cpu_data = np.zeros((ny, nx), dtype=np.float32)
+            #num_bytes = cpu_data.size * cpu_data.itemsize
+            #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can                     #be accessed directly by the GPU device            
+            #hipHostMallocDefault:Memory is mapped and portable (default allocation)
+            #hipHostMallocPortable: memory is explicitely portable across different devices
+            #cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable))
+            #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32)
+            #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32)
+
+        assert nx == cpu_data.shape[1]
+        assert ny == cpu_data.shape[0]
+        assert x+nx <= self.nx + 2*self.x_halo
+        assert y+ny <= self.ny + 2*self.y_halo
+
+        #Cuda
+        """
+        #Create copy object from device to host
+        copy = cuda.Memcpy2D()
+        copy.set_src_device(self.data.gpudata)
+        copy.set_dst_host(cpu_data)
+        
+        #Set offsets and pitch of source
+        copy.src_x_in_bytes = int(x)*self.data.strides[1]
+        copy.src_y = int(y)
+        copy.src_pitch = self.data.strides[0]
+        
+        #Set width in bytes to copy for each row and
+        #number of rows to copy
+        copy.width_in_bytes = int(nx)*cpu_data.itemsize
+        copy.height = int(ny)
+        """
+
+        #copy from device to host
+
+        #host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault))
+        #device_pointer = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault))
+
+
+        copy_download = {
+            'srcXInBytes': int(x)*np.float32().itemsize,
+            'srcY': int(y),
+            'srcMemoryType': hip.hipMemoryType.hipMemoryTypeDevice,#hipMemoryTypeManaged
+            'srcDevice': self.data,
+            'srcPitch': self.data.shape[0]*np.float32().itemsize,
+
+            'dstXInBytes': 0,
+            'dstY': 0,
+            'dstMemoryType': hip.hipMemoryType.hipMemoryTypeHost,
+            'dstHost': cpu_data, #device_pointer,
+            'dstPitch': cpu_data.strides[0],
+
+            'WidthInBytes': int(nx)*cpu_data.itemsize,
+            'Height': int(ny)
+        }
+
+        # Perform the copy back to host
+        Copy = hip.hip_Memcpy2D(**copy_download)
+
+        #err = hip.hipMemcpyParam2D(Copy)
+        err = hip.hipMemcpyParam2DAsync(Copy, stream)
+        if err is None:
+            print("--download - DtoH: Failed to copy 2D data to Host")
+            print("--I stop:", err)
+            exit()
+
+        #copy(stream)
+        
+        if asynch==False:
+            #stream.synchronize()
+            hip_check(hip.hipStreamSynchronize(stream))
+        return cpu_data
+        
+        
+    def upload(self, stream, cpu_data, extent=None):
+        if (extent is None):
+            x = self.x_halo
+            y = self.y_halo
+            nx = self.nx
+            ny = self.ny
+        else:
+            x, y, nx, ny = extent
+        
+        assert(nx == cpu_data.shape[1])
+        assert(ny == cpu_data.shape[0])
+        assert(x+nx <= self.nx + 2*self.x_halo)
+        assert(y+ny <= self.ny + 2*self.y_halo)
+        
+        #Cuda
+        """
+        #Create copy object from device to host
+        #Well this copy from src:host to dst:device AND NOT from device to host
+        copy = cuda.Memcpy2D()
+        copy.set_dst_device(self.data.gpudata)
+        copy.set_src_host(cpu_data)
+        
+        #Set offsets and pitch of source
+        copy.dst_x_in_bytes = int(x)*self.data.strides[1]
+        copy.dst_y = int(y)
+        copy.dst_pitch = self.data.strides[0]
+        
+        #Set width in bytes to copy for each row and
+        #number of rows to copy
+        copy.width_in_bytes = int(nx)*cpu_data.itemsize
+        copy.height = int(ny)
+        """
+
+
+        #Copy from host to device
+
+        #host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault))
+        #device_pointer = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault))
+
+        copy_upload = {
+            'srcXInBytes': 0,
+            'srcY': 0,
+            'srcMemoryType': hip.hipMemoryType.hipMemoryTypeHost,
+            'srcHost': cpu_data, #device_pointer 
+            'srcPitch': cpu_data.strides[0],  # assuming float32 (4 bytes)
+
+            'dstXInBytes': int(x)*np.float32().itemsize,
+            'dstY': int(y),
+            'dstMemoryType': hip.hipMemoryType.hipMemoryTypeDevice, #hipMemoryTypeManaged
+            'dstDevice': self.data,
+            'dstPitch': self.data.shape[0]*np.float32().itemsize,
+
+            'WidthInBytes': int(nx)*cpu_data.itemsize,
+            'Height': int(ny)
+        }
+
+
+        # Perform the copy HtoD
+        Copy = hip.hip_Memcpy2D(**copy_upload)
+        
+        #err = hip.hipMemcpyParam2D(Copy)
+        err = hip.hipMemcpyParam2DAsync(Copy, stream)
+        
+        if err is None:
+            print("--Upload - HtoD: Failed to copy 2D data to Device")
+            print("--I stop:", err)
+            exit()
+
+        #copy(stream)
+        
+        
+        
+"""
+Class that holds 2D data 
+"""
+class CudaArray3D:
+    """
+    Uploads initial data to the CL device
+    """
+    def __init__(self, stream, nx, ny, nz, x_halo, y_halo, z_halo, cpu_data=None, dtype=np.float32):
+        self.logger =  logging.getLogger(__name__)
+        self.nx = nx
+        self.ny = ny
+        self.nz = nz
+        self.x_halo = x_halo
+        self.y_halo = y_halo
+        self.z_halo = z_halo
+        
+        nx_halo = nx + 2*x_halo
+        ny_halo = ny + 2*y_halo
+        nz_halo = nz + 2*z_halo
+        
+        #self.logger.debug("Allocating [%dx%dx%d] buffer", self.nx, self.ny, self.nz)
+        #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here
+        #self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype)
+        
+        """
+        num_bytes = nz_halo*ny_halo*nx_halo * np.float32().itemsize
+        # init device array and upload host data
+        self.data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(nz_halo, ny_halo, nx_halo))
+        """
+
+        #For returning to download
+        #self.memorypool = PageLockedMemoryPool()
+        
+        #If we don't have any data, just allocate and return
+        if cpu_data is None:
+            return
+            
+        #Make sure data is in proper format
+        assert cpu_data.shape == (nz_halo, ny_halo, nx_halo) or cpu_data.shape == (self.nz, self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.nz, self.ny, self.nx)), str((nz_halo, ny_halo, nx_halo)))
+        assert cpu_data.itemsize == 4, "Wrong size of data type"
+        assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)"
+        
+        #Cuda
+        """
+        #Create copy object from host to device
+        copy = cuda.Memcpy3D()
+        copy.set_src_host(cpu_data)
+        copy.set_dst_device(self.data.gpudata)
+        
+        #Set offsets of destination
+        x_offset = (nx_halo - cpu_data.shape[2]) // 2
+        y_offset = (ny_halo - cpu_data.shape[1]) // 2
+        z_offset = (nz_halo - cpu_data.shape[0]) // 2
+        copy.dst_x_in_bytes = x_offset*self.data.strides[1]
+        copy.dst_y = y_offset
+        copy.dst_z = z_offset
+        
+        #Set pitch of destination
+        copy.dst_pitch = self.data.strides[0]
+        
+        #Set width in bytes to copy for each row and
+        #number of rows to copy
+        width = max(self.nx, cpu_data.shape[2])
+        height = max(self.ny, cpu_data.shape[1])
+        depth = max(self.nz, cpu-data.shape[0])
+        copy.width_in_bytes = width*cpu_data.itemsize
+        copy.height = height
+        copy.depth = depth
+        
+        #Perform the copy
+        copy(stream)
+        #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny)
+        """
+
+        #copy from host to device
+        #src
+        host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault))
+        src_ptr = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault))
+        #src_ptr = hip.hipPitchedPtr()
+
+        #dst
+        # Allocate 3D pitched memory on the device
+        self.data = hip.hipPitchedPtr()
+        c_extent = hip.hipExtent(nx_halo*np.float32().itemsize, ny_halo, nz_halo)
+        #hip.hipMalloc3D(pitchedDevPtr-OUT, extent-IN)
+        err, = hip.hipMalloc3D(self.data, c_extent)
+        dst_pitch = nx_halo * np.float32().itemsize
+
+        #include offset: do we need make_hipPitchedPtr
+        x_offset = (nx_halo - cpu_data.shape[2]) // 2
+        y_offset = (ny_halo - cpu_data.shape[1]) // 2
+        z_offset = (nz_halo - cpu_data.shape[0]) // 2
+
+        if err != hip.hipError_t.hipSuccess:
+            raise RuntimeError(f"Error from hipMalloc3D: {hip.hipGetErrorString(err)}")
+
+        copy_upload = {
+                'srcPos': hip.hipPos(0, 0, 0),
+                'srcPtr': src_ptr,
+                'dstPos': hip.hipPos(0, 0, 0),
+                'dstPtr': self.data,
+                'extent': c_extent,
+                'kind':   hip.hipMemcpyKind.hipMemcpyHostToDevice
+        }
+
+        # Perform the copy
+        copy = hip.hipMemcpy3DParms(**copy_upload)
+        err = hip.hipMemcpy3DAsync(copy, stream)
+        #copy = hip_check(hip.hipMemcpyAsync(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,stream))
+
+
+        #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data), self.nx, self.ny)
+
+        
+    def __del__(self, *args):
+        #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny)
+        #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data), self.nx, self.ny)
+        #self.data.gpudata.free()
+        hip_check(hip.hipFree(self.data))
+        #hip_check(hip.hipFreeAsync(self.data, self.stream))
+        self.data = None
+        
+    """
+    Enables downloading data from GPU to Python
+    """
+    def download(self, stream, asynch=False):
+        #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny)
+        #Allocate host memory
+        #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32)
+        cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32)
+        #cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32)
+       
+        #Cuda
+        """
+        #Create copy object from device to host
+        copy = cuda.Memcpy2D()
+        copy.set_src_device(self.data.gpudata)
+        copy.set_dst_host(cpu_data)
+        
+        #Set offsets and pitch of source
+        copy.src_x_in_bytes = self.x_halo*self.data.strides[1]
+        copy.src_y = self.y_halo
+        copy.src_z = self.z_halo
+        copy.src_pitch = self.data.strides[0]
+        
+        #Set width in bytes to copy for each row and
+        #number of rows to copy
+        copy.width_in_bytes = self.nx*cpu_data.itemsize
+        copy.height = self.ny
+        copy.depth = self.nz
+        
+        copy(stream)
+        """
+        #copy from device to host
+        num_bytes = cpu_data.size * cpu_data.itemsize
+        #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind)
+        copy = hip_check(hip.hipMemcpyAsync(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+
+        if asynch==False:
+            #stream.synchronize()
+            hip_check(hip.hipStreamSynchronize(stream))
+
+        return cpu_data
+
+        
+"""
+A class representing an Arakawa A type (unstaggered, logically Cartesian) grid
+"""
+class ArakawaA2D:
+    def __init__(self, stream, nx, ny, halo_x, halo_y, cpu_variables):
+        """
+        Uploads initial data to the GPU device
+        """
+        self.logger =  logging.getLogger(__name__)
+        self.gpu_variables = []
+       
+        for cpu_variable in cpu_variables:
+            self.gpu_variables += [CudaArray2D(stream, nx, ny, halo_x, halo_y, cpu_variable)]
+
+    
+    def __getitem__(self, key):
+        assert type(key) == int, "Indexing is int based"
+        if (key > len(self.gpu_variables) or key < 0):
+            raise IndexError("Out of bounds")
+        return self.gpu_variables[key]
+    
+    def download(self, stream, variables=None):
+        """
+        Enables downloading data from the GPU device to Python
+        """
+        if variables is None:
+            variables=range(len(self.gpu_variables))
+
+        cpu_variables = []
+        for i in variables:
+            assert i < len(self.gpu_variables), "Variable {:d} is out of range".format(i)
+            cpu_variables += [self.gpu_variables[i].download(stream, asynch=True)]
+
+        #print("--FIN: sum:", np.array(cpu_variables).sum())
+
+        #stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(stream))
+        return cpu_variables
+    
+    #hipblas
+    def sum_hipblas(self, num_elements, data):
+        num_bytes_r = np.dtype(np.float32).itemsize
+        result_d = hip_check(hip.hipMalloc(num_bytes_r))
+        result_h0 = np.zeros(1, dtype=np.float32)
+
+        # call hipblasSaxpy + initialization
+        handle = hip_check(hipblas.hipblasCreate())
+        #hip_check(hipblas.hipblasSaxpy(handle, num_elements, ctypes.addressof(alpha), x_d, 1, y_d, 1))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasSasum(handle, num_elements, data, 1, result_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in result_d) back to host (store in result_h)
+        hip_check(hip.hipMemcpy(result_h0,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost))
+
+        result_h = result_h0[0]
+
+        # clean up
+        #hip_check(hip.hipFree(data))
+        return result_h
+
+    def check(self):
+        """
+        Checks that data is still sane
+        """
+        for i, gpu_variable in enumerate(self.gpu_variables):
+            #compute sum with hipblas
+            #var_sum = pycuda.gpuarray.sum(gpu_variable.data).get()
+            var_sum = self.sum_hipblas(gpu_variable.data.size,gpu_variable.data)
+            #print(f"GPU: Sum for column {i}: {var_sum}")
+
+            self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny))
+            assert np.isnan(var_sum) == False, "Data contains NaN values!"

GPUSimulators/CudaContext.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements Cuda context handling
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+
+import os
+import ctypes
+import numpy as np
+import time
+import re
+import io
+import hashlib
+import logging
+import gc
+
+#import pycuda.compiler as cuda_compiler
+#import pycuda.gpuarray
+#import pycuda.driver as cuda
+
+from hip import hip,hiprtc
+
+from GPUSimulators import Autotuner, Common
+
+
+"""
+Class which keeps track of the CUDA context and some helper functions
+"""
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+class CudaContext(object):
+
+    def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True):
+        """
+        Create a new CUDA context
+        Set device to an id or pci_bus_id to select a specific GPU
+        Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context
+        """
+        self.use_cache = use_cache
+        self.logger =  logging.getLogger(__name__)
+        self.modules = {}
+        
+        self.module_path = os.path.dirname(os.path.realpath(__file__))
+        
+        #Initialize cuda (must be first call to PyCUDA)
+        ##cuda.init(flags=0)
+        
+        ##self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT))
+ 
+        #Print some info about CUDA
+        ##self.logger.info("CUDA version %s", str(cuda.get_version()))
+        #self.logger.info("Driver version %s",  str(cuda.get_driver_version()))
+        self.logger.info("HIP runtime  version %s", str(hip_check(hip.hipRuntimeGetVersion())))
+        self.logger.info("Driver version %s",  str(hip_check(hip.hipDriverGetVersion())))
+
+        if device is None:
+            device = 0
+      
+        num_gpus = hip_check(hip.hipGetDeviceCount())
+        hip.hipSetDevice(device)
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,device))
+        arch = props.gcnArchName
+        #self.cuda_device = cuda.Device(device)
+        #self.cuda_device = hip_check(hip.hipCtxGetDevice())
+        #self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id())
+       
+        # Allocate memory to store the PCI BusID
+        pciBusId = ctypes.create_string_buffer(64)
+        # PCI Bus Id
+        #hip_check(hip.hipDeviceGetPCIBusId(pciBusId, 64, device))
+        pciBusId = hip_check(hip.hipDeviceGetPCIBusId(64, device))
+
+
+        #self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, num_gpus,arch,pciBusId.value.decode('utf-8')[5:7]) 
+        self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, num_gpus,arch,pciBusId[5:7])
+        #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability()))
+        self.logger.debug(" => compute capability: %s", hip_check(hip.hipDeviceComputeCapability(device)))
+
+        # Create the CUDA context
+        if context_flags is None:
+        #    context_flags=cuda.ctx_flags.SCHED_AUTO
+             context_flags=hip_check(hip.hipSetDeviceFlags(hip.hipDeviceScheduleAuto)) 
+        #self.cuda_context = self.cuda_device.make_context(flags=context_flags)
+        self.cuda_context = hip_check(hip.hipCtxCreate(0, device))
+
+        #free, total = cuda.mem_get_info()
+        total = hip_check(hip.hipDeviceTotalMem(device))
+        #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024)))
+        self.logger.debug(" => Total memory: %d MB available", int(total/(1024*1024)))
+
+        ##self.logger.info("Created context handle <%s>", str(self.cuda_context.handle))
+        self.logger.info("Created context handle <%s>", str(self.cuda_context)) 
+
+        #Create cache dir for cubin files
+        self.cache_path = os.path.join(self.module_path, "cuda_cache") 
+        if (self.use_cache):
+            if not os.path.isdir(self.cache_path):
+                os.mkdir(self.cache_path)
+            self.logger.info("Using CUDA cache dir %s", self.cache_path)
+            
+        self.autotuner = None
+        """
+        if (autotuning):
+            self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience")
+            self.autotuner = Autotuner.Autotuner()
+        """    
+    
+    def __del__(self, *args):
+        #self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle))
+        #self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context))
+        """
+        # Loop over all contexts in stack, and remove "this"
+        other_contexts = []
+        #while (cuda.Context.get_current() != None):
+        while (hip.hipCtxGetCurrent() != None):
+            #context = cuda.Context.get_current()
+            context = hip_check(hip.hipCtxGetCurrent())
+            #if (context.handle != self.cuda_context.handle):
+            if (context != self.cuda_context):
+                #self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle))
+                #self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context), str(context))
+                other_contexts = [context] + other_contexts
+                #cuda.Context.pop()
+                hip.hipCtxPopCurrent()
+            else:
+                #self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle))
+                self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context), str(context))
+                #cuda.Context.pop()
+                hip.hipCtxPopCurrent()
+
+        # Add all the contexts we popped that were not our own
+        for context in other_contexts:
+            #self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle))
+            self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context), str(context))
+            #cuda.Context.push(context)
+            hip_check(hip.hipCtxPushCurrent(context))
+
+        #self.logger.debug("<%s> Detaching", str(self.cuda_context.handle))
+        self.logger.debug("<%s> Detaching", str(self.cuda_context))
+        #self.cuda_context.detach()
+        hip_check(hip.hipCtxDestroy(self.cuda_context))
+        """ 
+        
+    def __str__(self):
+        #return "CudaContext id " + str(self.cuda_context.handle)
+        return "CudaContext id " + str(self.cuda_context)
+        
+    
+    def hash_kernel(kernel_filename, include_dirs):        
+        # Generate a kernel ID for our caches
+        num_includes = 0
+        max_includes = 100
+        kernel_hasher = hashlib.md5()
+        logger = logging.getLogger(__name__)
+        
+        # Loop over file and includes, and check if something has changed
+        files = [kernel_filename]
+        while len(files):
+        
+            if (num_includes > max_includes):
+                raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename))
+        
+            filename = files.pop()
+            
+            #logger.debug("Hashing %s", filename)
+                
+            modified = os.path.getmtime(filename)
+                
+            # Open the file
+            with io.open(filename, "r") as file:
+            
+                # Search for #inclue <something> and also hash the file
+                file_str = file.read()
+                kernel_hasher.update(file_str.encode('utf-8'))
+                kernel_hasher.update(str(modified).encode('utf-8'))
+                
+                #Find all includes
+                includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M)
+                
+            # Loop over everything that looks like an include
+            for include_file in includes:
+                
+                #Search through include directories for the file
+                file_path = os.path.dirname(filename)
+                for include_path in [file_path] + include_dirs:
+                
+                    # If we find it, add it to list of files to check
+                    temp_path = os.path.join(include_path, include_file)
+                    if (os.path.isfile(temp_path)):
+                        files = files + [temp_path]
+                        num_includes = num_includes + 1 #For circular includes...
+                        break
+            
+        return kernel_hasher.hexdigest()
+
+
+    """
+    Reads a text file and creates an OpenCL kernel from that
+    """
+    def get_module(self, kernel_filename, 
+                    include_dirs=[], \
+                    defines={}, \
+                    compile_args={'no_extern_c', True}, jit_compile_args={}):
+        """
+        Helper function to print compilation output
+        """
+        def cuda_compile_message_handler(compile_success_bool, info_str, error_str):
+            self.logger.debug("Compilation returned %s", str(compile_success_bool))
+            if info_str:
+                self.logger.debug("Info: %s", info_str)
+            if error_str:
+                self.logger.debug("Error: %s", error_str)
+       
+        kernel_filename = os.path.normpath(kernel_filename)
+        kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename))
+        #self.logger.debug("Getting %s", kernel_filename)
+            
+        # Create a hash of the kernel options
+        options_hasher = hashlib.md5()
+        options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8'));
+        options_hash = options_hasher.hexdigest()
+       
+        # Create hash of kernel souce
+        source_hash = CudaContext.hash_kernel( \
+                    kernel_path, \
+                    include_dirs=[self.module_path] + include_dirs)
+       
+        # Create final hash
+        root, ext = os.path.splitext(kernel_filename)
+        kernel_hash = root \
+                + "_" + source_hash \
+                + "_" + options_hash \
+                + ext
+        cached_kernel_filename = os.path.join(self.cache_path, kernel_hash)
+        
+        # If we have the kernel in our hashmap, return it
+        if (kernel_hash in self.modules.keys()):
+            self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash)
+            return self.modules[kernel_hash]
+        
+        # If we have it on disk, return it
+        elif (self.use_cache and os.path.isfile(cached_kernel_filename)):
+            self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash)
+                
+            with io.open(cached_kernel_filename, "rb") as file:
+                file_str = file.read()
+                #module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args)
+                module = hip_check(hip.hipModuleLoadDataEx(file_str, 0, None)) 
+                print("HIP module loaded: to be checked!")
+            self.modules[kernel_hash] = module
+            return module
+            
+        # Otherwise, compile it from source
+        else:
+            self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash)
+                
+            #Create kernel string
+            kernel_string = ""
+            for key, value in defines.items():
+                kernel_string += "#define {:s} {:s}\n".format(str(key), str(value))
+            kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename))
+            if (self.use_cache):
+                cached_kernel_dir = os.path.dirname(cached_kernel_filename)
+                if not os.path.isdir(cached_kernel_dir):
+                    os.mkdir(cached_kernel_dir)
+                with io.open(cached_kernel_filename + ".txt", "w") as file:
+                    file.write(kernel_string)
+            
+            """cuda
+            with Common.Timer("compiler") as timer:
+
+                import warnings
+                with warnings.catch_warnings():
+                    warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning)
+
+                    cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args)
+                module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args)
+
+                if (self.use_cache):
+                    with io.open(cached_kernel_filename, "wb") as file:
+                        file.write(cubin)
+                
+            self.modules[kernel_hash] = module
+            return module
+            """
+
+    """
+    Clears the kernel cache (useful for debugging & development)
+    """
+    def clear_kernel_cache(self):
+        self.logger.debug("Clearing cache")
+        self.modules = {}
+        gc.collect()
+        
+    """
+    Synchronizes all streams etc
+    """
+    def synchronize(self):
+        #self.cuda_context.synchronize()
+        hip_check(hip.hipCtxSynchronize())

GPUSimulators/EE2D_KP07_dimsplit.py

@@ -0,0 +1,651 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the 2nd order HLL flux
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import os
+import sys
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+    
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result    
+
+"""
+Class that solves the SW equations using the Forward-Backward linear scheme
+"""
+class EE2D_KP07_dimsplit (BaseSimulator):
+
+    """
+    Initialization routine
+    rho: Density
+    rho_u: Momentum along x-axis
+    rho_v: Momentum along y-axis
+    E: energy
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis
+    dy: Grid cell spacing along y-axis
+    dt: Size of each timestep 
+    g: Gravitational constant
+    gamma: Gas constant
+    p: pressure
+    """
+
+    def __init__(self, 
+                 context, 
+                 rho, rho_u, rho_v, E, 
+                 nx, ny, 
+                 dx, dy,  
+                 g, 
+                 gamma, 
+                 theta=1.3, 
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=8):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale, 
+            2, 
+            block_width, block_height)
+        self.g = np.float32(g)
+        self.gamma = np.float32(gamma)
+        self.theta = np.float32(theta) 
+
+
+        #Get cuda kernels
+        """ Cuda
+        module = context.get_module("cuda/EE2D_KP07_dimsplit.cu.hip", 
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        #compile and load to the device
+        self.kernel = module.get_function("KP07DimsplitKernel")
+        self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        
+        #source code
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'EE2D_KP07_dimsplit.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #EulerCommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'EulerCommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+
+        #limiters.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h'))
+        with open(header_file_path, 'r') as file:
+            header_limiters = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"EulerCommon.h",b"limiters.h"]))    
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <EE2D_KP07_dimsplit.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        #extract the arch of the device
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0)) #only one device 0
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+        
+        #Get the device kernel named "KP07DimsplitKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07DimsplitKernel"))
+
+        print()
+        print("--Get the device kernel *KP07DimsplitKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [rho, rho_u, rho_v, E])
+
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [None, None, None, None])
+
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+        # init device array cfl_data
+        dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho)))
+        dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho)))
+        self.dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+       
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+
+    def substep(self, dt, step_number, external=True, internal=True):
+            self.substepDimsplit(0.5*dt, step_number, external, internal)
+    
+    def substepDimsplit(self, dt, substep, external, internal):
+
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+        u03_strides0 = self.u0[3].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+        u13_strides0 = self.u1[3].data.shape[0]*np.float32().itemsize
+
+        if external and internal:
+                    
+            #print("COMPLETE DOMAIN (dt=" + str(dt) + ")")
+
+            """ Cuda
+            self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                0, 0, 
+                self.nx, self.ny)
+            """
+
+            #hip.hipModuleLaunchKernel(f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, stream, kernelParams, extra)
+
+            #The argument grid/block requires 3 components x,y and z. in 2D z=1.
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta), 
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),    
+                        self.u0[1].data, ctypes.c_int(u01_strides0),   
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),   
+                        self.u1[0].data, ctypes.c_int(u10_strides0),   
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),   
+                        self.cfl_data, 
+                        ctypes.c_int(0), ctypes.c_int(0),
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),   
+                    )
+                )
+            )
+
+            #print("--External & Internal: Launching Kernel is ok")
+
+            return
+        
+        if external and not internal:
+            ###################################
+            # XXX: Corners are treated twice! #
+            ###################################
+
+            ns_grid_size = (self.grid_size[0], 1, 1)
+            # NORTH
+            # (x0, y0) x (x1, y1)
+            #  (0, ny-y_halo) x (nx, ny)
+            """ Cuda
+            self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, 
+                self.nx, self.ny,
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                0, self.ny - int(self.u0[0].y_halo),
+                self.nx, self.ny)
+            """
+          
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *ns_grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),
+                        self.cfl_data,
+                        ctypes.c_int(0), ctypes.c_int(self.ny - self.u0[0].y_halo),
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                    )
+                )
+            )
+
+            hip_check(hip.hipStreamSynchronize(self.stream))
+
+            #print()
+            #print("--I m at the NORTH:")
+            #print()
+            # SOUTH
+            # (x0, y0) x (x1, y1)
+            #   (0, 0) x (nx, y_halo)
+            """ Cuda
+            self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, 
+                self.nx, self.ny,
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                0, 0,
+                self.nx, int(self.u0[0].y_halo))
+            """
+
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *ns_grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),
+                        self.cfl_data,
+                        ctypes.c_int(0), ctypes.c_int(0),
+                        ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo),
+                    )
+                )
+            )
+            
+            hip_check(hip.hipStreamSynchronize(self.stream))
+
+            we_grid_size = (1, self.grid_size[1], 1)
+            # WEST
+            # (x0, y0) x (x1, y1)
+            #  (0, 0) x (x_halo, ny)
+            """ Cuda
+            self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, 
+                self.nx, self.ny,
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                0, 0,
+                int(self.u0[0].x_halo), self.ny)
+            """
+
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *we_grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),
+                        self.cfl_data,
+                        ctypes.c_int(0), ctypes.c_int(0),
+                        ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny),
+                    )
+                )
+            )
+
+            hip_check(hip.hipStreamSynchronize(self.stream))
+
+            # EAST
+            # (x0, y0) x (x1, y1)
+            #   (nx-x_halo, 0) x (nx, ny)
+            """ Cuda
+            self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, 
+                self.nx, self.ny,
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                self.nx - int(self.u0[0].x_halo), 0,
+                self.nx, self.ny)
+            """
+
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *we_grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),
+                        self.cfl_data,
+                        ctypes.c_int(self.nx - self.u0[0].x_halo), ctypes.c_int(0),
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                    )
+                )
+            )
+
+#            print("--External and not Internal: Launching Kernel is ok")
+            return
+
+        if internal and not external:
+           
+            # INTERNAL DOMAIN
+            #         (x0, y0) x (x1, y1)
+            # (x_halo, y_halo) x (nx - x_halo, ny - y_halo)
+            """
+            self.kernel.prepared_async_call(self.grid_size, self.block_size, self.internal_stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.gamma, 
+                self.theta, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u0[3].data.gpudata, self.u0[3].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0], 
+                self.u1[3].data.gpudata, self.u1[3].data.strides[0],
+                self.cfl_data.gpudata,
+                int(self.u0[0].x_halo), int(self.u0[0].y_halo),
+                self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo))
+            """
+
+            hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.internal_stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.gamma),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u0[3].data, ctypes.c_int(u03_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.u1[3].data, ctypes.c_int(u13_strides0),
+                        self.cfl_data,
+                        ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.u0[0].y_halo),
+                        ctypes.c_int(self.nx - self.u0[0].x_halo), ctypes.c_int(self.ny - self.u0[0].y_halo),
+                    )
+                )
+            )
+
+ #           print("--Internal and not External: Launching Kernel is ok")
+            return
+
+    def swapBuffers(self):
+        self.u0, self.u1 = self.u1, self.u0
+        return
+        
+    def getOutput(self):
+        return self.u0
+
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+        return
+        
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/FORCE.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations 
+"""
+class FORCE (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            1,
+            block_width, block_height)
+        self.g = np.float32(g) 
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_FORCE.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("FORCEKernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_FORCE.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_FORCE.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .FORCEKernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "FORCEKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"FORCEKernel"))
+
+        print()
+        print("--Get the device kernel *FORCEKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0
+        """
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .FORCEKernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt

GPUSimulators/HLL.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations using the Harten-Lax -van Leer approximate Riemann solver
+"""
+class HLL (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            1,
+            block_width, block_height)
+        self.g = np.float32(g) 
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_HLL.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("HLLKernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_HLL.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_HLL.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .HLLKernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "FORCEKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"HLLKernel"))
+
+        print()
+        print("--Get the device kernel *HLLKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0
+        """
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .HLLKernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/HLL2.py

@@ -0,0 +1,313 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations 
+"""
+class HLL2 (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 theta=1.8,
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            2,
+            block_width, block_height)
+        self.g = np.float32(g) 
+        self.theta = np.float32(theta)
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_HLL2.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("HLL2Kernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_HLL2.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+        
+        #limiters.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h'))
+        with open(header_file_path, 'r') as file:
+            header_limiters = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_HLL2.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .HLL2Kernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "FORCEKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"HLL2Kernel"))
+
+        print()
+        print("--Get the device kernel *HLL2Kernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        self.substepDimsplit(dt*0.5, step_number)
+
+    def substepDimsplit(self, dt, substep):        
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.theta,
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0
+        """
+
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .HLL2Kernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/IPythonMagic.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements helpers for IPython / Jupyter and CUDA
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+import gc
+
+from IPython.core import magic_arguments
+from IPython.core.magic import line_magic, Magics, magics_class
+#import pycuda.driver as cuda
+from hip import hip, hiprtc
+
+from GPUSimulators import Common, CudaContext
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+@magics_class
+class MagicCudaContext(Magics): 
+    @line_magic
+    @magic_arguments.magic_arguments()
+    @magic_arguments.argument(
+        'name', type=str, help='Name of context to create')
+    @magic_arguments.argument(
+        '--blocking', '-b', action="store_true", help='Enable blocking context')
+    @magic_arguments.argument(
+        '--no_cache', '-nc', action="store_true", help='Disable caching of kernels')
+    @magic_arguments.argument(
+        '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels')
+
+    def cuda_context_handler(self, line):
+        args = magic_arguments.parse_argstring(self.cuda_context_handler, line)
+        self.logger =  logging.getLogger(__name__)
+        
+        self.logger.info("Registering %s in user workspace", args.name)
+        
+        context_flags = None
+        if (args.blocking):
+            #context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC
+            context_flags = hip_check(hip.hipSetDeviceFlags(hip.hipDeviceScheduleBlockingSync))
+        
+        if args.name in self.shell.user_ns.keys():
+            self.logger.debug("Context already registered! Ignoring")
+            return
+        else:
+            self.logger.debug("Creating context")
+            use_cache = False if args.no_cache else True
+            use_autotuning = False if args.no_autotuning else True
+            self.shell.user_ns[args.name] = CudaContext.CudaContext(context_flags=context_flags, use_cache=use_cache, autotuning=use_autotuning)
+        
+        # this function will be called on exceptions in any cell
+        def custom_exc(shell, etype, evalue, tb, tb_offset=None):
+            self.logger.exception("Exception caught: Resetting to CUDA context %s", args.name)
+            #while (cuda.Context.get_current() != None):
+            while (hip.hipCtxGetCurrent() != None):    
+                #context = cuda.Context.get_current()
+                context = hip_check(hip.hipCtxGetCurrent())
+                self.logger.info("Popping <%s>", str(context.handle))
+                #cuda.Context.pop()
+                hip.hipCtxPopCurrent()
+            if args.name in self.shell.user_ns.keys():
+                self.logger.info("Pushing <%s>", str(self.shell.user_ns[args.name].cuda_context.handle))
+                self.shell.user_ns[args.name].cuda_context.push()
+            else:
+                self.logger.error("No CUDA context called %s found (something is wrong)", args.name)
+                self.logger.error("CUDA will not work now")
+
+            self.logger.debug("==================================================================")
+            
+            # still show the error within the notebook, don't just swallow it
+            shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset)
+
+        # this registers a custom exception handler for the whole current notebook
+        get_ipython().set_custom_exc((Exception,), custom_exc)
+        
+        
+        # Handle CUDA context when exiting python
+        import atexit
+        def exitfunc():
+            self.logger.info("Exitfunc: Resetting CUDA context stack")
+            #while (cuda.Context.get_current() != None):
+            while (hip.hipCtxGetCurrent() != None):    
+                #context = cuda.Context.get_current()
+                context = hip_check(hip.hipCtxGetCurrent())
+                self.logger.info("`-> Popping <%s>", str(context.handle))
+                #cuda.Context.pop()
+                hip.hipCtxPopCurrent()
+            self.logger.debug("==================================================================")
+        atexit.register(exitfunc)
+        
+        
+        
+        
+        
+        
+        
+        
+@magics_class
+class MagicLogger(Magics): 
+    logger_initialized = False
+    
+    @line_magic
+    @magic_arguments.magic_arguments()
+    @magic_arguments.argument(
+        'name', type=str, help='Name of context to create')
+    @magic_arguments.argument(
+        '--out', '-o', type=str, default='output.log', help='The filename to store the log to')
+    @magic_arguments.argument(
+        '--level', '-l', type=int, default=20, help='The level of logging to screen [0, 50]')
+    @magic_arguments.argument(
+        '--file_level', '-f', type=int, default=10, help='The level of logging to file [0, 50]')
+    def setup_logging(self, line):
+        if (self.logger_initialized):
+            logging.getLogger('GPUSimulators').info("Global logger already initialized!")
+            return;
+        else:
+            self.logger_initialized = True
+            
+            args = magic_arguments.parse_argstring(self.setup_logging, line)
+            import sys
+            
+            #Get root logger
+            logger = logging.getLogger('GPUSimulators')
+            logger.setLevel(min(args.level, args.file_level))
+
+            #Add log to screen
+            ch = logging.StreamHandler()
+            ch.setLevel(args.level)
+            logger.addHandler(ch)
+            logger.log(args.level, "Console logger using level %s", logging.getLevelName(args.level))
+            
+            #Get the outfilename (try to evaluate if Python expression...)
+            try:
+                outfile = eval(args.out, self.shell.user_global_ns, self.shell.user_ns)
+            except:
+                outfile = args.out
+            
+            #Add log to file
+            logger.log(args.level, "File logger using level %s to %s", logging.getLevelName(args.file_level), outfile)
+            
+            fh = logging.FileHandler(outfile)
+            formatter = logging.Formatter('%(asctime)s:%(name)s:%(levelname)s: %(message)s')
+            fh.setFormatter(formatter)
+            fh.setLevel(args.file_level)
+            logger.addHandler(fh)
+        
+        logger.info("Python version %s", sys.version)
+        self.shell.user_ns[args.name] = logger
+
+
+
+        
+
+
+@magics_class
+class MagicMPI(Magics): 
+    
+    @line_magic
+    @magic_arguments.magic_arguments()
+    @magic_arguments.argument(
+        'name', type=str, help='Name of context to create')
+    @magic_arguments.argument(
+        '--num_engines', '-n', type=int, default=4, help='Number of engines to start')
+    def setup_mpi(self, line):
+        args = magic_arguments.parse_argstring(self.setup_mpi, line)
+        logger = logging.getLogger('GPUSimulators')
+        if args.name in self.shell.user_ns.keys():
+            logger.warning("MPI alreay set up, resetting")
+            self.shell.user_ns[args.name].shutdown()
+            self.shell.user_ns[args.name] = None
+            gc.collect()
+        self.shell.user_ns[args.name] = Common.IPEngine(args.num_engines)
+
+        
+
+
+
+
+
+
+# Register 
+ip = get_ipython()
+ip.register_magics(MagicCudaContext)
+ip.register_magics(MagicLogger)
+ip.register_magics(MagicMPI)
+

GPUSimulators/KP07.py

@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations using the Forward-Backward linear scheme
+"""
+class KP07 (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 theta=1.3,
+                 cfl_scale=0.9,
+                 order=2,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            order,             
+            block_width, block_height)
+        self.g = np.float32(g) 
+        self.theta = np.float32(theta)
+        self.order = np.int32(order)
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_KP07.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("KP07Kernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_KP07.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+        
+        #limiters.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h'))
+        with open(header_file_path, 'r') as file:
+            header_limiters = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_KP07.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .KP07Kernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "FORCEKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07Kernel"))
+
+        print()
+        print("--Get the device kernel *KP07Kernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        self.substepRK(dt, step_number)
+
+    def substepRK(self, dt, substep):
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.theta,
+                Simulator.stepOrderToCodedInt(step=substep, order=self.order),
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0        
+        """
+
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.theta),
+                        Simulator.stepOrderToCodedInt(step=substep, order=self.order),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .KP07Kernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5**(self.order-1)

GPUSimulators/KP07_dimsplit.py

@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations using the dimentionally split KP07 scheme
+"""
+class KP07_dimsplit (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 theta=1.3,
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            2,             
+            block_width, block_height)
+        self.gc_x = 2
+        self.gc_y = 2
+        self.g = np.float32(g) 
+        self.theta = np.float32(theta)
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("KP07DimsplitKernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_KP07_dimsplit.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+        
+        #limiters.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h'))
+        with open(header_file_path, 'r') as file:
+            header_limiters = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_KP07_dimsplit.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .KP07DimsplitKernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "FORCEKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07DimsplitKernel"))
+
+        print()
+        print("--Get the device kernel *KP07DimsplitKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        self.gc_x, self.gc_y, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        self.gc_x, self.gc_y,
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        self.substepDimsplit(dt*0.5, step_number)
+
+    def substepDimsplit(self, dt, substep):
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.theta,
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0        
+        """
+
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_float(self.theta),
+                        ctypes.c_int(substep),   
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .KP07DimsplitKernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/LxF.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations using the Lax Friedrichs scheme
+"""
+class LxF (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            1,
+            block_width, block_height)
+        self.g = np.float32(g) 
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_LxF.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("LxFKernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_LxF.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_LxF.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .LxFKernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "LxFKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"LxFKernel"))
+
+        print()
+        print("--Get the device kernel *LxFKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        1, 1, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0
+        """
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .LxFKernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/MPISimulator.py

@@ -0,0 +1,550 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements MPI simulator class
+
+Copyright (C) 2018 SINTEF Digital
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+import logging
+from GPUSimulators import Simulator
+import numpy as np
+from mpi4py import MPI
+import time
+
+#import pycuda.driver as cuda
+#import nvtx
+from hip import hip, hiprtc
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+class MPIGrid(object):
+    """
+    Class which represents an MPI grid of nodes. Facilitates easy communication between
+    neighboring nodes
+    """
+    def __init__(self, comm, ndims=2):
+        self.logger =  logging.getLogger(__name__)
+        
+        assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment"
+        assert comm.size >= 1, "Must have at least one node"
+        
+        self.grid = MPIGrid.getGrid(comm.size, ndims)
+        self.comm = comm
+        
+        self.logger.debug("Created MPI grid: {:}. Rank {:d} has coordinate {:}".format(
+                self.grid, self.comm.rank, self.getCoordinate()))
+
+    def getCoordinate(self, rank=None):
+        if (rank is None):
+            rank = self.comm.rank
+        i = (rank  % self.grid[0])
+        j = (rank // self.grid[0])
+        return i, j
+
+    def getRank(self, i, j):
+        return j*self.grid[0] + i
+
+    def getEast(self):
+        i, j = self.getCoordinate(self.comm.rank)
+        i = (i+1) % self.grid[0]
+        return self.getRank(i, j)
+
+    def getWest(self):
+        i, j = self.getCoordinate(self.comm.rank)
+        i = (i+self.grid[0]-1) % self.grid[0]
+        return self.getRank(i, j)
+
+    def getNorth(self):
+        i, j = self.getCoordinate(self.comm.rank)
+        j = (j+1) % self.grid[1]
+        return self.getRank(i, j)
+
+    def getSouth(self):
+        i, j = self.getCoordinate(self.comm.rank)
+        j = (j+self.grid[1]-1) % self.grid[1]
+        return self.getRank(i, j)
+    
+    def getGrid(num_nodes, num_dims):
+        assert(isinstance(num_nodes, int))
+        assert(isinstance(num_dims, int))
+        
+        # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors
+        # Original code by https://stackoverflow.com/users/3928385/ishamael
+        # Factorizes a number into n roughly equal factors
+
+        #Dictionary to remember already computed permutations
+        memo = {}
+        def dp(n, left): # returns tuple (cost, [factors])
+            """
+            Recursively searches through all factorizations
+            """
+
+            #Already tried: return existing result
+            if (n, left) in memo: 
+                return memo[(n, left)]
+
+            #Spent all factors: return number itself
+            if left == 1:
+                return (n, [n])
+
+            #Find new factor
+            i = 2
+            best = n
+            bestTuple = [n]
+            while i * i < n:
+                #If factor found
+                if n % i == 0:
+                    #Factorize remainder
+                    rem = dp(n // i, left - 1)
+
+                    #If new permutation better, save it
+                    if rem[0] + i < best:
+                        best = rem[0] + i
+                        bestTuple = [i] + rem[1]
+                i += 1
+
+            #Store calculation
+            memo[(n, left)] = (best, bestTuple)
+            return memo[(n, left)]
+
+
+        grid = dp(num_nodes, num_dims)[1]
+
+        if (len(grid) < num_dims):
+            #Split problematic 4
+            if (4 in grid):
+                grid.remove(4)
+                grid.append(2)
+                grid.append(2)
+
+            #Pad with ones to guarantee num_dims
+            grid = grid + [1]*(num_dims - len(grid))
+        
+        #Sort in descending order
+        grid = np.sort(grid)
+        grid = grid[::-1]
+
+        # XXX: We only use vertical (north-south) partitioning for now
+        grid[0] = 1
+        grid[1] = num_nodes
+        
+        return grid
+
+
+    def gather(self, data, root=0):
+        out_data = None
+        if (self.comm.rank == root):
+            out_data = np.empty([self.comm.size] + list(data.shape), dtype=data.dtype)
+        self.comm.Gather(data, out_data, root)
+        return out_data
+        
+    def getLocalRank(self):
+        """
+        Returns the local rank on this node for this MPI process
+        """
+        
+        # This function has been adapted from 
+        # https://github.com/SheffieldML/PyDeepGP/blob/master/deepgp/util/parallel.py
+        # by Zhenwen Dai released under BSD 3-Clause "New" or "Revised" License:
+        # 
+        # Copyright (c) 2016, Zhenwen Dai
+        # All rights reserved.
+        # 
+        # Redistribution and use in source and binary forms, with or without
+        # modification, are permitted provided that the following conditions are met:
+        # 
+        # * Redistributions of source code must retain the above copyright notice, this
+        #   list of conditions and the following disclaimer.
+        # 
+        # * Redistributions in binary form must reproduce the above copyright notice,
+        #   this list of conditions and the following disclaimer in the documentation
+        #   and/or other materials provided with the distribution.
+        # 
+        # * Neither the name of DGP nor the names of its
+        #   contributors may be used to endorse or promote products derived from
+        #   this software without specific prior written permission.
+        # 
+        # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+        #Get this ranks unique (physical) node name
+        node_name = MPI.Get_processor_name()
+        
+        #Gather the list of all node names on all nodes
+        node_names = self.comm.allgather(node_name)
+                
+        #Loop over all node names up until our rank
+        #and count how many duplicates of our nodename we find
+        local_rank = len([0 for name in node_names[:self.comm.rank] if name==node_name])
+        
+        return local_rank
+
+
+class MPISimulator(Simulator.BaseSimulator):
+    """
+    Class which handles communication between simulators on different MPI nodes
+    """
+
+    def __init__(self, sim, grid):        
+        self.profiling_data_mpi = { 'start': {}, 'end': {} }
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange"] = 0
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] = 0
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] = 0
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] = 0
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] = 0
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] = 0
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] = 0
+        self.profiling_data_mpi["start"]["t_mpi_step"] = 0
+        self.profiling_data_mpi["end"]["t_mpi_step"] = 0
+        self.profiling_data_mpi["n_time_steps"] = 0
+        self.logger =  logging.getLogger(__name__)
+        
+        autotuner = sim.context.autotuner
+        sim.context.autotuner = None;
+        boundary_conditions = sim.getBoundaryConditions()
+        super().__init__(sim.context, 
+            sim.nx, sim.ny, 
+            sim.dx, sim.dy, 
+            boundary_conditions,
+            sim.cfl_scale,
+            sim.num_substeps,  
+            sim.block_size[0], sim.block_size[1])
+        sim.context.autotuner = autotuner
+        
+        self.sim = sim
+        self.grid = grid
+        
+        #Get neighbor node ids
+        self.east = grid.getEast()
+        self.west = grid.getWest()
+        self.north = grid.getNorth()
+        self.south = grid.getSouth()
+        
+        #Get coordinate of this node
+        #and handle global boundary conditions
+        new_boundary_conditions = Simulator.BoundaryCondition({
+            'north': Simulator.BoundaryCondition.Type.Dirichlet,
+            'south': Simulator.BoundaryCondition.Type.Dirichlet,
+            'east': Simulator.BoundaryCondition.Type.Dirichlet,
+            'west': Simulator.BoundaryCondition.Type.Dirichlet
+        })
+        gi, gj = grid.getCoordinate()
+        #print("gi: " + str(gi) + ", gj: " + str(gj))
+        if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic):
+            self.west = None
+            new_boundary_conditions.west = boundary_conditions.west;
+        if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic):
+            self.south = None
+            new_boundary_conditions.south = boundary_conditions.south;
+        if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic):
+            self.east = None
+            new_boundary_conditions.east = boundary_conditions.east;
+        if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic):
+            self.north = None
+            new_boundary_conditions.north = boundary_conditions.north;
+        sim.setBoundaryConditions(new_boundary_conditions)
+                
+        #Get number of variables
+        self.nvars = len(self.getOutput().gpu_variables)
+        
+        #Shorthands for computing extents and sizes
+        gc_x = int(self.sim.getOutput()[0].x_halo)
+        gc_y = int(self.sim.getOutput()[0].y_halo)
+        nx = int(self.sim.nx)
+        ny = int(self.sim.ny)
+        
+        #Set regions for ghost cells to read from
+        #These have the format [x0, y0, width, height]
+        self.read_e = np.array([  nx,    0, gc_x, ny + 2*gc_y])
+        self.read_w = np.array([gc_x,    0, gc_x, ny + 2*gc_y])
+        self.read_n = np.array([gc_x,   ny,   nx,        gc_y])
+        self.read_s = np.array([gc_x, gc_y,   nx,        gc_y])
+        
+        #Set regions for ghost cells to write to
+        self.write_e = self.read_e + np.array([gc_x, 0, 0, 0])
+        self.write_w = self.read_w - np.array([gc_x, 0, 0, 0])
+        self.write_n = self.read_n + np.array([0, gc_y, 0, 0])
+        self.write_s = self.read_s - np.array([0, gc_y, 0, 0])
+        
+        #Allocate data for receiving
+        #Note that east and west also transfer ghost cells
+        #whilst north/south only transfer internal cells
+        #Reuses the width/height defined in the read-extets above
+        """
+        self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32)
+        self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32)
+        self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32)
+        self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32)
+        """
+
+        #HIP
+        self.in_e = np.zeros((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32)
+        num_bytes_e = self.in_e.size * self.in_e.itemsize
+        #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can be accessed directly by the GPU device
+        #hipHostMallocDefault:Memory is mapped and portable (default allocation)
+        #hipHostMallocPortable: memory is explicitely portable across different devices
+        #self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.in_e, hip.hipHostMallocPortable))
+
+        #print("--hip.hipGetDeviceFlags():", hip.hipGetDeviceFlags())
+
+        self.in_w = np.zeros((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32)
+        num_bytes_w = self.in_w.size * self.in_w.itemsize
+        #self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.in_w, hip.hipHostMallocPortable))
+
+        self.in_n = np.zeros((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32)
+        num_bytes_n = self.in_n.size * self.in_n.itemsize
+        #self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.in_n, hip.hipHostMallocPortable))
+
+        self.in_s = np.zeros((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32)
+        num_bytes_s = self.in_s.size * self.in_s.itemsize
+        #self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.in_s, hip.hipHostMallocPortable))
+
+        #Allocate data for sending
+        """
+        self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e)
+        self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w)
+        self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n)
+        self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s)
+        """
+
+        self.out_e = np.zeros((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32)
+        num_bytes_e = self.out_e.size * self.out_e.itemsize
+        #self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.out_e, hip.hipHostMallocPortable))
+
+        self.out_w = np.zeros((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32)
+        num_bytes_w = self.out_w.size * self.out_w.itemsize
+        #self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.out_w, hip.hipHostMallocPortable))
+
+        self.out_n = np.zeros((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32)
+        num_bytes_n = self.out_n.size * self.out_n.itemsize
+        #self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.out_n, hip.hipHostMallocPortable))
+
+        self.out_s = np.zeros((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32)
+        num_bytes_s = self.out_s.size * self.out_s.itemsize
+        #self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable))
+        #hip_check(hip.hipHostGetDevicePointer(self.out_s, hip.hipHostMallocPortable))
+
+
+        self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name()))
+
+        self.full_exchange()
+        #hip_check(hip.hipDeviceSynchronize())
+        #sim.context.synchronize()
+    
+    def substep(self, dt, step_number):
+        
+        #nvtx.mark("substep start", color="yellow")
+
+        self.profiling_data_mpi["start"]["t_mpi_step"] += time.time()
+        
+        #nvtx.mark("substep external", color="blue")
+        self.sim.substep(dt, step_number, external=True, internal=False) # only "internal ghost cells"
+        
+        #nvtx.mark("substep internal", color="red")
+        self.sim.substep(dt, step_number, internal=True, external=False) # "internal ghost cells" excluded
+
+        #nvtx.mark("substep full", color="blue")
+        #self.sim.substep(dt, step_number, external=True, internal=True)
+
+        self.sim.swapBuffers()
+
+        self.profiling_data_mpi["end"]["t_mpi_step"] += time.time()
+        
+        #nvtx.mark("exchange", color="blue")
+        self.full_exchange()
+
+        #nvtx.mark("sync start", color="blue")
+        #self.sim.stream.synchronize()
+        #self.sim.internal_stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sim.stream))
+        hip_check(hip.hipStreamSynchronize(self.sim.internal_stream))
+        #nvtx.mark("sync end", color="blue")
+        
+        self.profiling_data_mpi["n_time_steps"] += 1
+
+    def getOutput(self):
+        return self.sim.getOutput()
+        
+    def synchronize(self):
+        self.sim.synchronize()
+        
+    def check(self):
+        return self.sim.check()
+        
+    def computeDt(self):
+        local_dt = np.array([np.float32(self.sim.computeDt())]);
+        global_dt = np.empty(1, dtype=np.float32)
+        self.grid.comm.Allreduce(local_dt, global_dt, op=MPI.MIN)
+        self.logger.debug("Local dt: {:f}, global dt: {:f}".format(local_dt[0], global_dt[0]))
+        return global_dt[0]
+        
+        
+    def getExtent(self):
+        """
+        Function which returns the extent of node with rank 
+        rank in the grid
+        """
+        width = self.sim.nx*self.sim.dx
+        height = self.sim.ny*self.sim.dy
+        i, j = self.grid.getCoordinate()
+        x0 = i * width
+        y0 = j * height 
+        x1 = x0 + width
+        y1 = y0 + height
+        return [x0, x1, y0, y1]
+
+    def full_exchange(self):
+        ####
+        # First transfer internal cells north-south
+        ####
+        
+        #Download from the GPU
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time()
+            
+        if self.north is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_n[k,:,:], asynch=True, extent=self.read_n)
+        if self.south is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_s[k,:,:], asynch=True, extent=self.read_s)
+        #self.sim.stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sim.stream))
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time()
+        
+        #Send/receive to north/south neighbours
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        comm_send = []
+        comm_recv = []
+        if self.north is not None:
+            comm_send += [self.grid.comm.Isend(self.out_n, dest=self.north, tag=4*self.nt + 0)]
+            comm_recv += [self.grid.comm.Irecv(self.in_n, source=self.north, tag=4*self.nt + 1)]
+        if self.south is not None:
+            comm_send += [self.grid.comm.Isend(self.out_s, dest=self.south, tag=4*self.nt + 1)]
+            comm_recv += [self.grid.comm.Irecv(self.in_s, source=self.south, tag=4*self.nt + 0)]
+        
+        #Wait for incoming transfers to complete
+        for comm in comm_recv:
+            comm.wait()
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        #Upload to the GPU
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time()
+        
+        if self.north is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].upload(self.sim.stream, self.in_n[k,:,:], extent=self.write_n)
+        if self.south is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].upload(self.sim.stream, self.in_s[k,:,:], extent=self.write_s)
+                
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time()
+        
+        #Wait for sending to complete
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        for comm in comm_send:
+            comm.wait()
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        ####
+        # Then transfer east-west including ghost cells that have been filled in by north-south transfer above
+        ####
+        
+        #Download from the GPU
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time()
+        
+        if self.east is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_e[k,:,:], asynch=True, extent=self.read_e)
+        if self.west is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_w[k,:,:], asynch=True, extent=self.read_w)
+        #self.sim.stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sim.stream))
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time()
+        
+        #Send/receive to east/west neighbours
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        comm_send = []
+        comm_recv = []
+        if self.east is not None:
+            comm_send += [self.grid.comm.Isend(self.out_e, dest=self.east, tag=4*self.nt + 2)]
+            comm_recv += [self.grid.comm.Irecv(self.in_e, source=self.east, tag=4*self.nt + 3)]
+        if self.west is not None:
+            comm_send += [self.grid.comm.Isend(self.out_w, dest=self.west, tag=4*self.nt + 3)]
+            comm_recv += [self.grid.comm.Irecv(self.in_w, source=self.west, tag=4*self.nt + 2)]
+        
+        #Wait for incoming transfers to complete
+        for comm in comm_recv:
+            comm.wait()
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        #Upload to the GPU
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time()
+        
+        if self.east is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].upload(self.sim.stream, self.in_e[k,:,:], extent=self.write_e)
+        if self.west is not None:
+            for k in range(self.nvars):
+                self.sim.u0[k].upload(self.sim.stream, self.in_w[k,:,:], extent=self.write_w)
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time()
+        
+        #Wait for sending to complete
+        self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time()
+        
+        for comm in comm_send:
+            comm.wait()
+        
+        self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time()

GPUSimulators/SHMEMSimulator.py

@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements SHMEM simulator group class
+
+Copyright (C) 2020 Norwegian Meteorological Institute
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+import logging
+from GPUSimulators import Simulator, CudaContext
+import numpy as np
+
+#import pycuda.driver as cuda
+from hip import hip, hiprtc
+
+import time
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+class SHMEMSimulator(Simulator.BaseSimulator):
+    """
+    Class which handles communication and synchronization between simulators in different 
+    contexts (presumably on different GPUs)
+    """
+
+    def __init__(self, sims, grid):
+        self.logger =  logging.getLogger(__name__)
+    
+        assert(len(sims) > 1)
+
+        self.sims = sims
+
+        # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator?
+        # See also getOutput() and check().
+        #
+        # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of
+        # SHMEMSimulators that have BaseSimulator as a superclass.
+        #
+        # This would also eliminate the need for all the array bookkeeping in this class.
+        autotuner = sims[0].context.autotuner
+        sims[0].context.autotuner = None
+        boundary_conditions = sims[0].getBoundaryConditions()
+        super().__init__(sims[0].context, 
+            sims[0].nx, sims[0].ny, 
+            sims[0].dx, sims[0].dy, 
+            boundary_conditions,
+            sims[0].cfl_scale,
+            sims[0].num_substeps,  
+            sims[0].block_size[0], sims[0].block_size[1])
+        sims[0].context.autotuner = autotuner
+        
+        self.sims = sims
+        self.grid = grid
+
+        self.east = [None] * len(self.sims)
+        self.west = [None] * len(self.sims)
+        self.north = [None] * len(self.sims)
+        self.south = [None] * len(self.sims)
+
+        self.nvars = [None] * len(self.sims)
+
+        self.read_e = [None] * len(self.sims)
+        self.read_w = [None] * len(self.sims)
+        self.read_n = [None] * len(self.sims)
+        self.read_s = [None] * len(self.sims)
+        
+        self.write_e = [None] * len(self.sims)
+        self.write_w = [None] * len(self.sims)
+        self.write_n = [None] * len(self.sims)
+        self.write_s = [None] * len(self.sims)
+
+        self.e = [None] * len(self.sims)
+        self.w = [None] * len(self.sims)
+        self.n = [None] * len(self.sims)
+        self.s = [None] * len(self.sims)
+        
+        for i, sim in enumerate(self.sims):
+            #Get neighbor subdomain ids
+            self.east[i] = grid.getEast(i)
+            self.west[i] = grid.getWest(i)
+            self.north[i] = grid.getNorth(i)
+            self.south[i] = grid.getSouth(i)
+            
+            #Get coordinate of this subdomain
+            #and handle global boundary conditions
+            new_boundary_conditions = Simulator.BoundaryCondition({
+                'north': Simulator.BoundaryCondition.Type.Dirichlet,
+                'south': Simulator.BoundaryCondition.Type.Dirichlet,
+                'east': Simulator.BoundaryCondition.Type.Dirichlet,
+                'west': Simulator.BoundaryCondition.Type.Dirichlet
+            })
+            gi, gj = grid.getCoordinate(i)
+            if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic):
+                self.west = None
+                new_boundary_conditions.west = boundary_conditions.west;
+            if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic):
+                self.south = None
+                new_boundary_conditions.south = boundary_conditions.south;
+            if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic):
+                self.east = None
+                new_boundary_conditions.east = boundary_conditions.east;
+            if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic):
+                self.north = None
+                new_boundary_conditions.north = boundary_conditions.north;
+            sim.setBoundaryConditions(new_boundary_conditions)
+                    
+            #Get number of variables
+            self.nvars[i] = len(sim.getOutput().gpu_variables)
+            
+            #Shorthands for computing extents and sizes
+            gc_x = int(sim.getOutput()[0].x_halo)
+            gc_y = int(sim.getOutput()[0].y_halo)
+            nx = int(sim.nx)
+            ny = int(sim.ny)
+            
+            #Set regions for ghost cells to read from
+            #These have the format [x0, y0, width, height]
+            self.read_e[i] = np.array([  nx,    0, gc_x, ny + 2*gc_y])
+            self.read_w[i] = np.array([gc_x,    0, gc_x, ny + 2*gc_y])
+            self.read_n[i] = np.array([gc_x,   ny,   nx,        gc_y])
+            self.read_s[i] = np.array([gc_x, gc_y,   nx,        gc_y])
+            
+            #Set regions for ghost cells to write to
+            self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0])
+            self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0])
+            self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0])
+            self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0])
+            
+            #Allocate host data
+            #Note that east and west also transfer ghost cells
+            #whilst north/south only transfer internal cells
+            #Reuses the width/height defined in the read-extets above
+            self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32)
+            self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32)
+            self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32)
+            self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32)
+
+        self.logger.debug("Initialized {:d} subdomains".format(len(self.sims)))
+    
+
+    def substep(self, dt, step_number):
+        self.exchange()
+
+        for i, sim in enumerate(self.sims):
+            sim.substep(dt, step_number)
+    
+    def getOutput(self):
+        # XXX: Does not return what we would expect.
+        # Returns first subdomain, but we want the whole domain.
+        return self.sims[0].getOutput() 
+        
+    def synchronize(self):
+        for sim in self.sims:
+            sim.synchronize()
+        
+    def check(self):
+        # XXX: Does not return what we would expect.
+        # Checks only first subdomain, but we want to check the whole domain.
+        return self.sims[0].check()
+    
+    def computeDt(self):
+        global_dt = float("inf")
+
+        for sim in self.sims:
+            sim.context.synchronize()
+
+        for sim in self.sims:
+            local_dt = sim.computeDt()
+            if local_dt < global_dt:
+                global_dt = local_dt
+            self.logger.debug("Local dt: {:f}".format(local_dt))
+
+        self.logger.debug("Global dt: {:f}".format(global_dt))
+        return global_dt
+        
+    def getExtent(self, index=0):
+        """
+        Function which returns the extent of the subdomain with index 
+        index in the grid
+        """
+        width = self.sims[index].nx*self.sims[index].dx
+        height = self.sims[index].ny*self.sims[index].dy
+        i, j = self.grid.getCoordinate(index)
+        x0 = i * width
+        y0 = j * height 
+        x1 = x0 + width
+        y1 = y0 + height
+        return [x0, x1, y0, y1]
+        
+    def exchange(self):
+        ####
+        # First transfer internal cells north-south
+        ####
+        for i in range(len(self.sims)):
+            self.ns_download(i)
+        
+        for i in range(len(self.sims)):
+            self.ns_upload(i)
+
+        ####
+        # Then transfer east-west including ghost cells that have been filled in by north-south transfer above
+        ####
+        for i in range(len(self.sims)):
+            self.ew_download(i)
+        
+        for i in range(len(self.sims)):
+            self.ew_upload(i)
+
+    def ns_download(self, i):
+        #Download from the GPU
+        if self.north[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i])
+        if self.south[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i])
+        #self.sims[i].stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sims[i].stream))
+
+    def ns_upload(self, i):
+        #Upload to the GPU
+        if self.north[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i])
+        if self.south[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i])
+        
+    def ew_download(self, i):
+        #Download from the GPU
+        if self.east[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i])
+        if self.west[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i])
+        #self.sims[i].stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sims[i].stream))
+
+    def ew_upload(self, i):
+        #Upload to the GPU
+        if self.east[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i])
+                #test_east = np.ones_like(self.e[self.east[i]][k,:,:])
+                #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i])
+        if self.west[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i])
+                #test_west = np.ones_like(self.e[self.west[i]][k,:,:])
+                #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i])

GPUSimulators/SHMEMSimulatorGroup.py

@@ -0,0 +1,414 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements SHMEM simulator group class
+
+Copyright (C) 2020 Norwegian Meteorological Institute
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+import logging
+from GPUSimulators import Simulator, CudaContext
+import numpy as np
+
+#import pycuda.driver as cuda
+from hip import hip, hiprtc
+
+import time
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+class SHMEMGrid(object):
+    """
+    Class which represents an SHMEM grid of GPUs. Facilitates easy communication between
+    neighboring subdomains in the grid. Contains one CUDA context per subdomain.
+    """
+
+    def __init__(self, ngpus=None, ndims=2):
+        self.logger =  logging.getLogger(__name__)
+
+        #cuda.init(flags=0)
+        self.logger.info("Initializing HIP")
+        #num_cuda_devices = cuda.Device.count()
+        num_cuda_devices = hip_check(hip.hipGetDeviceCount())
+
+        if ngpus is None:
+            ngpus = num_cuda_devices
+        
+        # XXX: disabled for testing on single-GPU system
+        #assert ngpus <= num_cuda_devices, "Trying to allocate more GPUs than are available in the system."   
+        #assert ngpus >= 2, "Must have at least two GPUs available to run multi-GPU simulations."
+
+        assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment"
+        
+        self.ngpus = ngpus
+        self.ndims = ndims
+
+        self.grid = SHMEMGrid.getGrid(self.ngpus, self.ndims)
+        
+        self.logger.debug("Created {:}-dimensional SHMEM grid, using {:} GPUs".format(
+                self.ndims, self.ngpus))    
+
+        # XXX: Is this a natural place to store the contexts? Consider moving contexts out of this 
+        # class, into notebook / calling script (shmemTesting.py)
+        self.cuda_contexts = []
+
+        for i in range(self.ngpus):
+            # XXX: disabled for testing on single-GPU system
+            #self.cuda_contexts.append(CudaContext.CudaContext(device=i, autotuning=False))
+            self.cuda_contexts.append(CudaContext.CudaContext(device=0, autotuning=False))
+
+    def getCoordinate(self, index):
+        i = (index  % self.grid[0])
+        j = (index // self.grid[0])
+        return i, j
+
+    def getIndex(self, i, j):
+        return j*self.grid[0] + i
+
+    def getEast(self, index):
+        i, j = self.getCoordinate(index)
+        i = (i+1) % self.grid[0]
+        return self.getIndex(i, j)
+
+    def getWest(self, index):
+        i, j = self.getCoordinate(index)
+        i = (i+self.grid[0]-1) % self.grid[0]
+        return self.getIndex(i, j)
+
+    def getNorth(self, index):
+        i, j = self.getCoordinate(index)
+        j = (j+1) % self.grid[1]
+        return self.getIndex(i, j)
+
+    def getSouth(self, index):
+        i, j = self.getCoordinate(index)
+        j = (j+self.grid[1]-1) % self.grid[1]
+        return self.getIndex(i, j)
+    
+    def getGrid(num_gpus, num_dims):
+        assert(isinstance(num_gpus, int))
+        assert(isinstance(num_dims, int))
+        
+        # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors
+        # Original code by https://stackoverflow.com/users/3928385/ishamael
+        # Factorizes a number into n roughly equal factors
+
+        #Dictionary to remember already computed permutations
+        memo = {}
+        def dp(n, left): # returns tuple (cost, [factors])
+            """
+            Recursively searches through all factorizations
+            """
+
+            #Already tried: return existing result
+            if (n, left) in memo: 
+                return memo[(n, left)]
+
+            #Spent all factors: return number itself
+            if left == 1:
+                return (n, [n])
+
+            #Find new factor
+            i = 2
+            best = n
+            bestTuple = [n]
+            while i * i < n:
+                #If factor found
+                if n % i == 0:
+                    #Factorize remainder
+                    rem = dp(n // i, left - 1)
+
+                    #If new permutation better, save it
+                    if rem[0] + i < best:
+                        best = rem[0] + i
+                        bestTuple = [i] + rem[1]
+                i += 1
+
+            #Store calculation
+            memo[(n, left)] = (best, bestTuple)
+            return memo[(n, left)]
+
+
+        grid = dp(num_gpus, num_dims)[1]
+
+        if (len(grid) < num_dims):
+            #Split problematic 4
+            if (4 in grid):
+                grid.remove(4)
+                grid.append(2)
+                grid.append(2)
+
+            #Pad with ones to guarantee num_dims
+            grid = grid + [1]*(num_dims - len(grid))
+        
+        #Sort in descending order
+        grid = np.sort(grid)
+        grid = grid[::-1]
+        
+        return grid
+
+class SHMEMSimulatorGroup(object):
+    """
+    Class which handles communication and synchronization between simulators in different 
+    contexts (typically on different GPUs)
+    """
+    def __init__(self, sims, grid):
+        self.logger =  logging.getLogger(__name__)
+    
+        assert(len(sims) > 1)
+
+        self.sims = sims
+
+        # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator?
+        # See also getOutput() and check().
+        #
+        # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of
+        # SHMEMSimulators that have BaseSimulator as a superclass.
+        #
+        # This would also eliminate the need for all the array bookkeeping in this class.
+        #
+        CONT HERE! Model shmemTesting after mpiTesting and divide existing functionality between SHMEMSimulatorGroup and SHMEMSimulator
+        
+        autotuner = sims[0].context.autotuner
+        sims[0].context.autotuner = None
+        boundary_conditions = sims[0].getBoundaryConditions()
+        super().__init__(sims[0].context, 
+            sims[0].nx, sims[0].ny, 
+            sims[0].dx, sims[0].dy, 
+            boundary_conditions,
+            sims[0].cfl_scale,
+            sims[0].num_substeps,  
+            sims[0].block_size[0], sims[0].block_size[1])
+        sims[0].context.autotuner = autotuner
+        
+        self.sims = sims
+        self.grid = grid
+
+        self.east = [None] * len(self.sims)
+        self.west = [None] * len(self.sims)
+        self.north = [None] * len(self.sims)
+        self.south = [None] * len(self.sims)
+
+        self.nvars = [None] * len(self.sims)
+
+        self.read_e = [None] * len(self.sims)
+        self.read_w = [None] * len(self.sims)
+        self.read_n = [None] * len(self.sims)
+        self.read_s = [None] * len(self.sims)
+        
+        self.write_e = [None] * len(self.sims)
+        self.write_w = [None] * len(self.sims)
+        self.write_n = [None] * len(self.sims)
+        self.write_s = [None] * len(self.sims)
+
+        self.e = [None] * len(self.sims)
+        self.w = [None] * len(self.sims)
+        self.n = [None] * len(self.sims)
+        self.s = [None] * len(self.sims)
+        
+        for i, sim in enumerate(self.sims):
+            #Get neighbor subdomain ids
+            self.east[i] = grid.getEast(i)
+            self.west[i] = grid.getWest(i)
+            self.north[i] = grid.getNorth(i)
+            self.south[i] = grid.getSouth(i)
+            
+            #Get coordinate of this subdomain
+            #and handle global boundary conditions
+            new_boundary_conditions = Simulator.BoundaryCondition({
+                'north': Simulator.BoundaryCondition.Type.Dirichlet,
+                'south': Simulator.BoundaryCondition.Type.Dirichlet,
+                'east': Simulator.BoundaryCondition.Type.Dirichlet,
+                'west': Simulator.BoundaryCondition.Type.Dirichlet
+            })
+            gi, gj = grid.getCoordinate(i)
+            if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic):
+                self.west = None
+                new_boundary_conditions.west = boundary_conditions.west;
+            if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic):
+                self.south = None
+                new_boundary_conditions.south = boundary_conditions.south;
+            if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic):
+                self.east = None
+                new_boundary_conditions.east = boundary_conditions.east;
+            if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic):
+                self.north = None
+                new_boundary_conditions.north = boundary_conditions.north;
+            sim.setBoundaryConditions(new_boundary_conditions)
+                    
+            #Get number of variables
+            self.nvars[i] = len(sim.getOutput().gpu_variables)
+            
+            #Shorthands for computing extents and sizes
+            gc_x = int(sim.getOutput()[0].x_halo)
+            gc_y = int(sim.getOutput()[0].y_halo)
+            nx = int(sim.nx)
+            ny = int(sim.ny)
+            
+            #Set regions for ghost cells to read from
+            #These have the format [x0, y0, width, height]
+            self.read_e[i] = np.array([  nx,    0, gc_x, ny + 2*gc_y])
+            self.read_w[i] = np.array([gc_x,    0, gc_x, ny + 2*gc_y])
+            self.read_n[i] = np.array([gc_x,   ny,   nx,        gc_y])
+            self.read_s[i] = np.array([gc_x, gc_y,   nx,        gc_y])
+            
+            #Set regions for ghost cells to write to
+            self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0])
+            self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0])
+            self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0])
+            self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0])
+            
+            #Allocate host data
+            #Note that east and west also transfer ghost cells
+            #whilst north/south only transfer internal cells
+            #Reuses the width/height defined in the read-extets above
+            self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32)
+            self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32)
+            self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32)
+            self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32)
+
+        self.logger.debug("Initialized {:d} subdomains".format(len(self.sims)))
+    
+
+    def substep(self, dt, step_number):
+        self.exchange()
+
+        for i, sim in enumerate(self.sims):
+            sim.substep(dt, step_number)
+    
+    def getOutput(self):
+        # XXX: Does not return what we would expect.
+        # Returns first subdomain, but we want the whole domain.
+        return self.sims[0].getOutput() 
+        
+    def synchronize(self):
+        for sim in self.sims:
+            sim.synchronize()
+        
+    def check(self):
+        # XXX: Does not return what we would expect.
+        # Checks only first subdomain, but we want to check the whole domain.
+        return self.sims[0].check()
+    
+    def computeDt(self):
+        global_dt = float("inf")
+
+        for sim in self.sims:
+            sim.context.synchronize()
+
+        for sim in self.sims:
+            local_dt = sim.computeDt()
+            if local_dt < global_dt:
+                global_dt = local_dt
+            self.logger.debug("Local dt: {:f}".format(local_dt))
+
+        self.logger.debug("Global dt: {:f}".format(global_dt))
+        return global_dt
+        
+    def getExtent(self, index=0):
+        """
+        Function which returns the extent of the subdomain with index 
+        index in the grid
+        """
+        width = self.sims[index].nx*self.sims[index].dx
+        height = self.sims[index].ny*self.sims[index].dy
+        i, j = self.grid.getCoordinate(index)
+        x0 = i * width
+        y0 = j * height 
+        x1 = x0 + width
+        y1 = y0 + height
+        return [x0, x1, y0, y1]
+        
+    def exchange(self):
+        ####
+        # First transfer internal cells north-south
+        ####
+        for i in range(len(self.sims)):
+            self.ns_download(i)
+        
+        for i in range(len(self.sims)):
+            self.ns_upload(i)
+
+        ####
+        # Then transfer east-west including ghost cells that have been filled in by north-south transfer above
+        ####
+        for i in range(len(self.sims)):
+            self.ew_download(i)
+        
+        for i in range(len(self.sims)):
+            self.ew_upload(i)
+
+    def ns_download(self, i):
+        #Download from the GPU
+        if self.north[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i])
+        if self.south[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i])
+        #self.sims[i].stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sims[i].stream))
+
+
+    def ns_upload(self, i):
+        #Upload to the GPU
+        if self.north[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i])
+        if self.south[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i])
+        
+    def ew_download(self, i):
+        #Download from the GPU
+        if self.east[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i])
+        if self.west[i] is not None:
+            for k in range(self.nvars[i]):
+                # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west)
+                self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i])
+        #self.sims[i].stream.synchronize()
+        hip_check(hip.hipStreamSynchronize(self.sims[i].stream))
+
+    def ew_upload(self, i):
+        #Upload to the GPU
+        if self.east[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i])
+                #test_east = np.ones_like(self.e[self.east[i]][k,:,:])
+                #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i])
+        if self.west[i] is not None:
+            for k in range(self.nvars[i]):
+                self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i])
+                #test_west = np.ones_like(self.e[self.west[i]][k,:,:])
+                #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i])

GPUSimulators/Simulator.py

@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the classical Lax-Friedrichs numerical
+scheme for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+import numpy as np
+import math
+import logging
+from enum import IntEnum
+
+#import pycuda.compiler as cuda_compiler
+#import pycuda.gpuarray
+#import pycuda.driver as cuda
+
+from hip import hip, hiprtc
+
+from GPUSimulators import Common
+
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+class BoundaryCondition(object):    
+    """
+    Class for holding boundary conditions for global boundaries
+    """
+    
+    
+    class Type(IntEnum):
+        """
+        Enum that describes the different types of boundary conditions
+        WARNING: MUST MATCH THAT OF common.h IN CUDA
+        """
+        Dirichlet = 0,
+        Neumann = 1,
+        Periodic = 2,
+        Reflective = 3
+
+    def __init__(self, types={ 
+                    'north': Type.Reflective, 
+                    'south': Type.Reflective, 
+                    'east': Type.Reflective, 
+                    'west': Type.Reflective 
+                 }):
+        """
+        Constructor
+        """
+        self.north = types['north']
+        self.south = types['south']
+        self.east = types['east']
+        self.west = types['west']
+        
+        if (self.north == BoundaryCondition.Type.Neumann \
+                or self.south == BoundaryCondition.Type.Neumann \
+                or self.east == BoundaryCondition.Type.Neumann \
+                or self.west == BoundaryCondition.Type.Neumann):
+            raise(NotImplementedError("Neumann boundary condition not supported"))
+            
+    def __str__(self):
+        return  '[north={:s}, south={:s}, east={:s}, west={:s}]'.format(str(self.north), str(self.south), str(self.east), str(self.west))
+
+        
+    def asCodedInt(self):
+        """
+        Helper function which packs four boundary conditions into one integer
+        """
+        bc = 0
+        bc = bc | (self.north & 0x0000000F) << 24
+        bc = bc | (self.south & 0x0000000F) << 16
+        bc = bc | (self.east  & 0x0000000F) <<  8
+        bc = bc | (self.west  & 0x0000000F) <<  0
+        
+        #for t in types:
+        #    print("{0:s}, {1:d}, {1:032b}, {1:08b}".format(t, types[t]))
+        #print("bc: {0:032b}".format(bc))
+        
+        return np.int32(bc)
+        
+    def getTypes(bc):
+        types = {}
+        types['north'] = BoundaryCondition.Type((bc >> 24) & 0x0000000F)
+        types['south'] = BoundaryCondition.Type((bc >> 16) & 0x0000000F)
+        types['east']  = BoundaryCondition.Type((bc >>  8) & 0x0000000F)
+        types['west']  = BoundaryCondition.Type((bc >>  0) & 0x0000000F)
+        return types
+        
+    
+    
+class BaseSimulator(object):
+
+    def __init__(self, 
+                 context, 
+                 nx, ny, 
+                 dx, dy, 
+                 boundary_conditions,
+                 cfl_scale,
+                 num_substeps,
+                 block_width, block_height):
+        """
+        Initialization routine
+        context: GPU context to use
+        kernel_wrapper: wrapper function of GPU kernel
+        h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+        hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+        hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+        nx: Number of cells along x-axis
+        ny: Number of cells along y-axis
+        dx: Grid cell spacing along x-axis (20 000 m)
+        dy: Grid cell spacing along y-axis (20 000 m)
+        dt: Size of each timestep (90 s)
+        cfl_scale: Courant number
+        num_substeps: Number of substeps to perform for a full step
+        """
+        #Get logger
+        self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
+        
+        #Save input parameters
+        #Notice that we need to specify them in the correct dataformat for the
+        #GPU kernel
+        self.context = context
+        self.nx = np.int32(nx)
+        self.ny = np.int32(ny)
+        self.dx = np.float32(dx)
+        self.dy = np.float32(dy)
+        self.setBoundaryConditions(boundary_conditions)
+        self.cfl_scale = cfl_scale
+        self.num_substeps = num_substeps
+        
+        #Handle autotuning block size
+        if (self.context.autotuner):
+            peak_configuration = self.context.autotuner.get_peak_performance(self.__class__)
+            block_width = int(peak_configuration["block_width"])
+            block_height = int(peak_configuration["block_height"])
+            self.logger.debug("Used autotuning to get block size [%d x %d]", block_width, block_height)
+
+        #Compute kernel launch parameters
+        """
+        self.block_size = (block_width, block_height, 1) 
+        self.grid_size = ( 
+                       int(np.ceil(self.nx / float(self.block_size[0]))), 
+                       int(np.ceil(self.ny / float(self.block_size[1]))) 
+                      )
+        """
+        self.block_size = hip.dim3(block_width, block_height) 
+        #self.grid_size = hip.dim3(math.ceil(self.nx/block_width),math.ceil(self.ny/block_height))
+        self.grid_size = hip.dim3(math.ceil((self.nx+block_width-1)/block_width),math.ceil((self.ny+block_height-1)/block_height))
+
+        #Create a CUDA stream
+        #self.stream = cuda.Stream()
+        #self.internal_stream = cuda.Stream()
+        self.stream = hip_check(hip.hipStreamCreate())
+        self.internal_stream = hip_check(hip.hipStreamCreate())
+
+        #Keep track of simulation time and number of timesteps
+        self.t = 0.0
+        self.nt = 0
+        
+
+    def __str__(self):
+        return "{:s} [{:d}x{:d}]".format(self.__class__.__name__, self.nx, self.ny)
+
+
+    def simulate(self, t, dt=None):
+        """ 
+        Function which simulates t_end seconds using the step function
+        Requires that the step() function is implemented in the subclasses
+        """
+
+        printer = Common.ProgressPrinter(t)
+        
+        t_start = self.simTime()
+        t_end = t_start + t
+        
+        update_dt = True
+        if (dt is not None):
+            update_dt = False
+            self.dt = dt
+        
+        while(self.simTime() < t_end):
+            # Update dt every 100 timesteps and cross your fingers it works
+            # for the next 100
+            if (update_dt and (self.simSteps() % 100 == 0)):
+                self.dt = self.computeDt()*self.cfl_scale
+        
+            # Compute timestep for "this" iteration (i.e., shorten last timestep)
+            current_dt = np.float32(min(self.dt, t_end-self.simTime()))
+
+            # Stop if end reached (should not happen)
+            if (current_dt <= 0.0):
+                self.logger.warning("Timestep size {:d} is less than or equal to zero!".format(self.simSteps()))
+                break
+        
+            # Step forward in time
+            self.step(current_dt)
+
+            #Print info
+            print_string = printer.getPrintString(self.simTime() - t_start)
+            if (print_string):
+                self.logger.info("%s: %s", self, print_string)
+                try:
+                    self.check()
+                except AssertionError as e:
+                    e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),)
+                    raise
+
+
+    def step(self, dt):
+        """
+        Function which performs one single timestep of size dt
+        """
+        for i in range(self.num_substeps):
+            self.substep(dt, i)
+            
+        self.t += dt
+        self.nt += 1
+
+    def download(self, variables=None):
+        return self.getOutput().download(self.stream, variables)
+        
+    def synchronize(self):
+        #self.stream.synchronize()
+        #Synchronize the stream to ensure operations in the stream is complete
+        hip_check(hip.hipStreamSynchronize(self.stream))
+        
+    def simTime(self):
+        return self.t
+
+    def simSteps(self):
+        return self.nt
+       
+    def getExtent(self):
+        return [0, 0, self.nx*self.dx, self.ny*self.dy]
+        
+    def setBoundaryConditions(self, boundary_conditions):
+        self.logger.debug("Boundary conditions set to {:s}".format(str(boundary_conditions)))
+        self.boundary_conditions = boundary_conditions.asCodedInt()
+        
+    def getBoundaryConditions(self):
+        return BoundaryCondition(BoundaryCondition.getTypes(self.boundary_conditions))
+        
+    def substep(self, dt, step_number):
+        """
+        Function which performs one single substep with stepsize dt
+        """
+        raise(NotImplementedError("Needs to be implemented in subclass"))
+        
+    def getOutput(self):
+        raise(NotImplementedError("Needs to be implemented in subclass"))
+
+    def check(self):
+        self.logger.warning("check() is not implemented - please implement")
+        #raise(NotImplementedError("Needs to be implemented in subclass"))
+        
+    def computeDt(self):
+        raise(NotImplementedError("Needs to be implemented in subclass"))
+       
+        
+        
+        
+def stepOrderToCodedInt(step, order):
+    """
+    Helper function which packs the step and order into a single integer
+    """
+    step_order = (step << 16) | (order & 0x0000ffff)
+    #print("Step:  {0:032b}".format(step))
+    #print("Order: {0:032b}".format(order))
+    #print("Mix:   {0:032b}".format(step_order))
+    return np.int32(step_order)

GPUSimulators/WAF.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements the FORCE flux
+for the shallow water equations
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+#Import packages we need
+from GPUSimulators import Simulator, Common
+from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition
+import numpy as np
+import ctypes
+
+#from pycuda import gpuarray
+from hip import hip,hiprtc
+from hip import hipblas
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+"""
+Class that solves the SW equations using the Forward-Backward linear scheme
+"""
+class WAF (Simulator.BaseSimulator):
+
+    """
+    Initialization routine
+    h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells
+    hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells
+    hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells
+    nx: Number of cells along x-axis
+    ny: Number of cells along y-axis
+    dx: Grid cell spacing along x-axis (20 000 m)
+    dy: Grid cell spacing along y-axis (20 000 m)
+    dt: Size of each timestep (90 s)
+    g: Gravitational accelleration (9.81 m/s^2)
+    """
+
+    def __init__(self, 
+                 context, 
+                 h0, hu0, hv0, 
+                 nx, ny, 
+                 dx, dy, 
+                 g, 
+                 cfl_scale=0.9,
+                 boundary_conditions=BoundaryCondition(), 
+                 block_width=16, block_height=16):
+                 
+        # Call super constructor
+        super().__init__(context, 
+            nx, ny, 
+            dx, dy, 
+            boundary_conditions,
+            cfl_scale,
+            2,
+            block_width, block_height)
+        self.g = np.float32(g) 
+
+        #Get cuda kernels
+        """
+        module = context.get_module("cuda/SWE2D_WAF.cu",
+                                        defines={
+                                            'BLOCK_WIDTH': self.block_size[0], 
+                                            'BLOCK_HEIGHT': self.block_size[1]
+                                        }, 
+                                        compile_args={
+                                            'no_extern_c': True,
+                                            'options': ["--use_fast_math"], 
+                                        }, 
+                                        jit_compile_args={})
+        self.kernel = module.get_function("WAFKernel")
+        self.kernel.prepare("iiffffiPiPiPiPiPiPiP")
+        """
+
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        # Specify the relative path to the "cuda" directory
+        cuda_dir = os.path.join(current_dir, 'cuda')
+
+        #kernel source
+        kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_WAF.cu.hip'))
+        with open(kernel_file_path, 'r') as file:
+            kernel_source = file.read()
+
+        #headers
+        #common.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h'))
+        with open(header_file_path, 'r') as file:
+            header_common = file.read()
+
+        #SWECommon.h
+        header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h'))
+        with open(header_file_path, 'r') as file:
+            header_EulerCommon = file.read()
+
+        #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames)
+
+        prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"]))
+
+        # Check if the program is created successfully
+        if prog is not None:
+            print("--This is <SWE2D_WAF.cu.hip>")
+            print("--HIPRTC program created successfully")
+            print()
+        else:
+            print("--Failed to create HIPRTC program")
+            print("--I stop:", err)
+            exit()
+
+        props = hip.hipDeviceProp_t()
+        hip_check(hip.hipGetDeviceProperties(props,0))
+        arch = props.gcnArchName
+
+        print(f"Compiling kernel .WAFKernel. for {arch}")
+
+        cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()]
+
+        err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags)
+        # Check if the program is compiled successfully
+        if err is not None:
+            print("--Compilation:", err)
+            print("--The program is compiled successfully")
+        else:
+            print("--Compilation:", err)
+            print("--Failed to compile the program")
+            print("--I stop:", err)
+
+        if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS:
+            log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog))
+            log = bytearray(log_size)
+            hip_check(hiprtc.hiprtcGetProgramLog(prog, log))
+            raise RuntimeError(log.decode())
+
+        code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog))
+        code = bytearray(code_size)
+        hip_check(hiprtc.hiprtcGetCode(prog, code))
+
+        #Load the code as a module
+        self.module = hip_check(hip.hipModuleLoadData(code))
+
+        #Get the device kernel named named "LxFKernel"
+        self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"WAFKernel"))
+
+        print()
+        print("--Get the device kernel *WAFKernel* is created successfully--")
+        print("--kernel", self.kernel)
+        print()
+
+        #Create data by uploading to device
+        self.u0 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [h0, hu0, hv0])
+        self.u1 = Common.ArakawaA2D(self.stream, 
+                        nx, ny, 
+                        2, 2, 
+                        [None, None, None])
+        #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32)
+
+        dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0)))
+        dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0)))
+        dt = min(dt_x, dt_y)
+        #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute
+        #self.cfl_data.fill(self.dt, stream=self.stream)
+        grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size
+
+        data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32)
+        num_bytes = data_h.size * data_h.itemsize
+        data_h.fill(self.dt)
+
+        self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure(
+                 typestr="float32",shape=(grid_dim_x, grid_dim_y))
+
+        hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream))
+        #sets the memory region pointed to by x_d to zero asynchronously
+        #initiates the memset operation asynchronously
+        #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream))
+
+    def substep(self, dt, step_number):
+        self.substepDimsplit(dt*0.5, step_number)
+
+    def substepDimsplit(self, dt, substep):    
+        #Cuda
+        """
+        self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, 
+                self.nx, self.ny, 
+                self.dx, self.dy, dt, 
+                self.g, 
+                substep,
+                self.boundary_conditions, 
+                self.u0[0].data.gpudata, self.u0[0].data.strides[0], 
+                self.u0[1].data.gpudata, self.u0[1].data.strides[0], 
+                self.u0[2].data.gpudata, self.u0[2].data.strides[0], 
+                self.u1[0].data.gpudata, self.u1[0].data.strides[0], 
+                self.u1[1].data.gpudata, self.u1[1].data.strides[0], 
+                self.u1[2].data.gpudata, self.u1[2].data.strides[0],
+                self.cfl_data.gpudata)
+        self.u0, self.u1 = self.u1, self.u0        
+        """
+        u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize
+        u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize
+        u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize
+
+        u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize
+        u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize
+        u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize
+
+        #launch kernel
+        hip_check(
+                hip.hipModuleLaunchKernel(
+                    self.kernel,
+                    *self.grid_size, #grid
+                    *self.block_size, #block
+                    sharedMemBytes=0, #65536,
+                    stream=self.stream,
+                    kernelParams=None,
+                    extra=(     # pass kernel's arguments
+                        ctypes.c_int(self.nx), ctypes.c_int(self.ny),
+                        ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt),
+                        ctypes.c_float(self.g),
+                        ctypes.c_int(substep),
+                        ctypes.c_int(self.boundary_conditions),
+                        self.u0[0].data, ctypes.c_int(u00_strides0),
+                        self.u0[1].data, ctypes.c_int(u01_strides0),
+                        self.u0[2].data, ctypes.c_int(u02_strides0),
+                        self.u1[0].data, ctypes.c_int(u10_strides0),
+                        self.u1[1].data, ctypes.c_int(u11_strides0),
+                        self.u1[2].data, ctypes.c_int(u12_strides0),
+                        self.cfl_data,
+                    )
+                )
+            )
+
+        self.u0, self.u1 = self.u1, self.u0
+            
+        #print("--Launching Kernel .WAFKernel. is ok")
+
+    def getOutput(self):
+        return self.u0
+        
+    def check(self):
+        self.u0.check()
+        self.u1.check()
+    
+    # computing min with hipblas: the output is an index
+    def min_hipblas(self, num_elements, cfl_data, stream):
+        num_bytes = num_elements * np.dtype(np.float32).itemsize
+        num_bytes_i = np.dtype(np.int32).itemsize
+        indx_d = hip_check(hip.hipMalloc(num_bytes_i))
+        indx_h = np.zeros(1, dtype=np.int32)
+        x_temp = np.zeros(num_elements, dtype=np.float32)
+
+        #print("--size.data:", cfl_data.size)
+        handle = hip_check(hipblas.hipblasCreate())
+
+        #hip_check(hipblas.hipblasGetStream(handle, stream))
+        #"incx" [int] specifies the increment for the elements of x. incx must be > 0.
+        hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d))
+
+        # destruction of handle
+        hip_check(hipblas.hipblasDestroy(handle))
+
+        # copy result (stored in indx_d) back to the host (store in indx_h)
+        hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream))
+        #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream))
+        hip_check(hip.hipStreamSynchronize(stream))
+
+        min_value = x_temp.flatten()[indx_h[0]-1]
+
+        # clean up
+        hip_check(hip.hipStreamDestroy(stream))
+        hip_check(hip.hipFree(cfl_data))
+        return min_value
+
+    def computeDt(self):
+        #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get();
+        max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream)
+        return max_dt*0.5

GPUSimulators/__init__.py

@@ -0,0 +1,5 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+
+
+# Nothing general to do

GPUSimulators/cuda/EE2D_KP07_dimsplit.cu

@@ -0,0 +1,250 @@
+ /*
+This kernel implements the Central Upwind flux function to
+solve the Euler equations 
+
+Copyright (C) 2018  SINTEF Digital
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "EulerCommon.h"
+#include "limiters.h"
+
+
+__device__
+void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float gamma_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float4 Q_rl = make_float4(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1],
+                                            Q[3][j][i+1] - 0.5f*Qx[3][j][i+1]);
+            const float4 Q_rr = make_float4(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1],
+                                            Q[3][j][i+1] + 0.5f*Qx[3][j][i+1]);
+
+            const float4 Q_ll = make_float4(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i],
+                                            Q[3][j][i] - 0.5f*Qx[3][j][i]);
+            const float4 Q_lr = make_float4(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i],
+                                            Q[3][j][i] + 0.5f*Qx[3][j][i]);
+
+
+            //Evolve half a timestep (predictor step)
+            const float4 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, gamma_) - F_func(Q_rr, gamma_));
+            const float4 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, gamma_) - F_func(Q_lr, gamma_));
+
+            // Compute flux based on prediction
+            //const float4 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, gamma_);
+            const float4 flux = HLL_flux(Q_l_bar, Q_r_bar, gamma_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+            F[3][j][i] = flux.w;
+        }
+    }
+}
+
+__device__
+void computeFluxG(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float gamma_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float4 Q_rl = make_float4(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i],
+                                            Q[3][j+1][i] - 0.5f*Qy[3][j+1][i]);
+            const float4 Q_rr = make_float4(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i],
+                                            Q[3][j+1][i] + 0.5f*Qy[3][j+1][i]);
+
+            const float4 Q_ll = make_float4(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i],
+                                            Q[3][j][i] - 0.5f*Qy[3][j][i]);
+            const float4 Q_lr = make_float4(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i],
+                                            Q[3][j][i] + 0.5f*Qy[3][j][i]);
+
+            //Evolve half a timestep (predictor step)
+            const float4 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, gamma_) - F_func(Q_rr, gamma_));
+            const float4 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, gamma_) - F_func(Q_lr, gamma_));
+            
+            // Compute flux based on prediction
+            const float4 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, gamma_);
+            //const float4 flux = HLL_flux(Q_l_bar, Q_r_bar, gamma_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+            G[3][j][i] = flux.w;
+        }
+    }
+}
+
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+    
+__global__ void KP07DimsplitKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        float gamma_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* rho0_ptr_, int rho0_pitch_,
+        float* rho_u0_ptr_, int rho_u0_pitch_,
+        float* rho_v0_ptr_, int rho_v0_pitch_,
+        float* E0_ptr_, int E0_pitch_,
+        
+        //Output h^{n+1}
+        float* rho1_ptr_, int rho1_pitch_,
+        float* rho_u1_ptr_, int rho_u1_pitch_,
+        float* rho_v1_ptr_, int rho_v1_pitch_,
+        float* E1_ptr_, int E1_pitch_, 
+        
+        //Output CFL
+        float* cfl_,
+
+        //Subarea of internal domain to compute
+        int x0=0, int y0=0,
+        int x1=0, int y1=0) {
+
+    if(x1 == 0)
+        x1 = nx_;
+
+    if(y1 == 0)
+        y1 = ny_;
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 4;
+    
+    //Shared memory variables
+    __shared__ float  Q[4][h+2*gc_y][w+2*gc_x];
+    __shared__ float Qx[4][h+2*gc_y][w+2*gc_x];
+    __shared__ float  F[4][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>(  rho0_ptr_,   rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y,  1,  1>(    E0_ptr_,     E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, gamma_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, gamma_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();    
+        
+        //Gravity source term
+        if (g_ > 0.0f) {
+            const int i = threadIdx.x + gc_x;
+            const int j = threadIdx.y + gc_y;
+            const float rho_v = Q[2][j][i];
+            Q[2][j][i] -= g_*Q[0][j][i]*dt_;
+            Q[3][j][i] -= g_*rho_v*dt_;
+            __syncthreads();
+        }
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, gamma_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, gamma_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Gravity source term
+        if (g_ > 0.0f) {
+            const int i = threadIdx.x + gc_x;
+            const int j = threadIdx.y + gc_y;
+            const float rho_v = Q[2][j][i];
+            Q[2][j][i] -= g_*Q[0][j][i]*dt_;
+            Q[3][j][i] -= g_*rho_v*dt_;
+            __syncthreads();
+        }
+    }
+
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>(  rho1_ptr_,   rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(    E1_ptr_,     E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_);
+    }
+}
+
+
+} // extern "C"

GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip

@@ -0,0 +1,250 @@
+#include "hip/hip_runtime.h"
+ /*
+This kernel implements the Central Upwind flux function to
+solve the Euler equations 
+
+Copyright (C) 2018  SINTEF Digital
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "EulerCommon.h"
+#include "limiters.h"
+
+__device__
+void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float gamma_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float4 Q_rl = make_float4(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1],
+                                            Q[3][j][i+1] - 0.5f*Qx[3][j][i+1]);
+            const float4 Q_rr = make_float4(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1],
+                                            Q[3][j][i+1] + 0.5f*Qx[3][j][i+1]);
+
+            const float4 Q_ll = make_float4(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i],
+                                            Q[3][j][i] - 0.5f*Qx[3][j][i]);
+            const float4 Q_lr = make_float4(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i],
+                                            Q[3][j][i] + 0.5f*Qx[3][j][i]);
+
+
+            //Evolve half a timestep (predictor step)
+            const float4 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, gamma_) - F_func(Q_rr, gamma_));
+            const float4 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, gamma_) - F_func(Q_lr, gamma_));
+
+            // Compute flux based on prediction
+            //const float4 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, gamma_);
+            const float4 flux = HLL_flux(Q_l_bar, Q_r_bar, gamma_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+            F[3][j][i] = flux.w;
+        }
+    }
+}
+
+__device__
+void computeFluxG(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float gamma_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float4 Q_rl = make_float4(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i],
+                                            Q[3][j+1][i] - 0.5f*Qy[3][j+1][i]);
+            const float4 Q_rr = make_float4(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i],
+                                            Q[3][j+1][i] + 0.5f*Qy[3][j+1][i]);
+
+            const float4 Q_ll = make_float4(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i],
+                                            Q[3][j][i] - 0.5f*Qy[3][j][i]);
+            const float4 Q_lr = make_float4(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i],
+                                            Q[3][j][i] + 0.5f*Qy[3][j][i]);
+
+            //Evolve half a timestep (predictor step)
+            const float4 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, gamma_) - F_func(Q_rr, gamma_));
+            const float4 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, gamma_) - F_func(Q_lr, gamma_));
+            
+            // Compute flux based on prediction
+            const float4 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, gamma_);
+            //const float4 flux = HLL_flux(Q_l_bar, Q_r_bar, gamma_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+            G[3][j][i] = flux.w;
+        }
+    }
+}
+
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+    
+__global__ void KP07DimsplitKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        float gamma_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* rho0_ptr_, int rho0_pitch_,
+        float* rho_u0_ptr_, int rho_u0_pitch_,
+        float* rho_v0_ptr_, int rho_v0_pitch_,
+        float* E0_ptr_, int E0_pitch_,
+        
+        //Output h^{n+1}
+        float* rho1_ptr_, int rho1_pitch_,
+        float* rho_u1_ptr_, int rho_u1_pitch_,
+        float* rho_v1_ptr_, int rho_v1_pitch_,
+        float* E1_ptr_, int E1_pitch_, 
+        
+        //Output CFL
+        float* cfl_,
+
+        //Subarea of internal domain to compute
+        int x0=0, int y0=0,
+        int x1=0, int y1=0) {
+
+    if(x1 == 0)
+        x1 = nx_;
+
+    if(y1 == 0)
+        y1 = ny_;
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 4;
+    
+    //Shared memory variables
+    __shared__ float  Q[4][h+2*gc_y][w+2*gc_x];
+    __shared__ float Qx[4][h+2*gc_y][w+2*gc_x];
+    __shared__ float  F[4][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>(  rho0_ptr_,   rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+    readBlock<w, h, gc_x, gc_y,  1,  1>(    E0_ptr_,     E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1);
+
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, gamma_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, gamma_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();    
+        
+        //Gravity source term
+        if (g_ > 0.0f) {
+            const int i = threadIdx.x + gc_x;
+            const int j = threadIdx.y + gc_y;
+            const float rho_v = Q[2][j][i];
+            Q[2][j][i] -= g_*Q[0][j][i]*dt_;
+            Q[3][j][i] -= g_*rho_v*dt_;
+            __syncthreads();
+        }
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, gamma_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, gamma_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Gravity source term
+        if (g_ > 0.0f) {
+            const int i = threadIdx.x + gc_x;
+            const int j = threadIdx.y + gc_y;
+            const float rho_v = Q[2][j][i];
+            Q[2][j][i] -= g_*Q[0][j][i]*dt_;
+            Q[3][j][i] -= g_*rho_v*dt_;
+            __syncthreads();
+        }
+    }
+
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>(  rho1_ptr_,   rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    writeBlock<w, h, gc_x, gc_y>(    E1_ptr_,     E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_);
+    }
+}
+
+
+} // extern "C"

GPUSimulators/cuda/EulerCommon.h

@@ -0,0 +1,187 @@
+/*
+These CUDA functions implement different types of numerical flux 
+functions for the shallow water equations
+
+Copyright (C) 2016, 2017, 2018 SINTEF Digital
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+#include "limiters.h"
+
+
+
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x],
+        float shmem[h+2*gc_y][w+2*gc_x],
+        const int nx_, const int ny_,
+        const float dx_, const float dy_, const float gamma_,
+        float* output_) {
+    //Index of thread within block
+    const int tx = threadIdx.x + gc_x;
+    const int ty = threadIdx.y + gc_y;
+    
+    //Index of cell within domain
+    const int ti = blockDim.x*blockIdx.x + tx;
+    const int tj = blockDim.y*blockIdx.y + ty;
+    
+    //Only internal cells
+    if (ti < nx_+gc_x && tj < ny_+gc_y) {
+        const float rho = Q[0][ty][tx];
+        const float u   = Q[1][ty][tx] / rho;
+        const float v   = Q[2][ty][tx] / rho;
+        
+        const float max_u = dx_ / (fabsf(u) + sqrtf(gamma_*rho));
+        const float max_v = dy_ / (fabsf(v) + sqrtf(gamma_*rho));
+        
+        shmem[ty][tx] = fminf(max_u, max_v);
+    }
+    __syncthreads();
+    
+    //One row of threads loop over all rows
+    if (ti < nx_+gc_x && tj < ny_+gc_y) {
+        if (ty == gc_y) {
+            float min_val = shmem[ty][tx];
+            const int max_y = min(h, ny_+gc_y - tj);
+            for (int j=gc_y; j<max_y+gc_y; j++) {
+                min_val = fminf(min_val, shmem[j][tx]);
+            }
+            shmem[ty][tx] = min_val;
+        }
+    }
+    __syncthreads();
+    
+    //One thread loops over first row to find global max
+    if (tx == gc_x && ty == gc_y) {
+        float min_val = shmem[ty][tx];
+        const int max_x = min(w, nx_+gc_x - ti);
+        for (int i=gc_x; i<max_x+gc_x; ++i) {
+            min_val = fminf(min_val, shmem[ty][i]);
+        }
+        
+        const int idx = gridDim.x*blockIdx.y + blockIdx.x;
+        output_[idx] = min_val;
+    }
+}
+
+
+
+
+
+inline __device__ float pressure(float4 Q, float gamma) {
+    const float rho   = Q.x;
+    const float rho_u = Q.y;
+    const float rho_v = Q.z;
+    const float E     = Q.w;
+
+    return (gamma-1.0f)*(E-0.5f*(rho_u*rho_u + rho_v*rho_v)/rho);
+}
+
+
+__device__ float4 F_func(const float4 Q, float P) {
+    const float rho   = Q.x;
+    const float rho_u = Q.y;
+    const float rho_v = Q.z;
+    const float E     = Q.w;
+
+    const float u = rho_u/rho;
+
+    float4 F;
+
+    F.x = rho_u;
+    F.y = rho_u*u + P;
+    F.z = rho_v*u;
+    F.w = u*(E+P);
+
+    return F;
+}
+
+
+
+
+/**
+  * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 180)
+  */
+__device__ float4 HLL_flux(const float4 Q_l, const float4 Q_r, const float gamma) {
+    const float h_l = Q_l.x;
+    const float h_r = Q_r.x;
+    
+    // Calculate velocities
+    const float u_l = Q_l.y / h_l;
+    const float u_r = Q_r.y / h_r;
+    
+    // Calculate pressures
+    const float P_l = pressure(Q_l, gamma);
+    const float P_r = pressure(Q_r, gamma);
+    
+    // Estimate the potential wave speeds
+    const float c_l = sqrt(gamma*P_l/Q_l.x);
+    const float c_r = sqrt(gamma*P_r/Q_r.x);
+    
+    // Compute h in the "star region", h^dagger
+    const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r);
+    
+    const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) );
+    const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) );
+    
+    const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f;
+    const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f;
+    
+    // Compute wave speed estimates
+    const float S_l = u_l - c_l*q_l;
+    const float S_r = u_r + c_r*q_r;
+    
+    //Upwind selection
+    if (S_l >= 0.0f) {
+        return F_func(Q_l, P_l);
+    }
+    else if (S_r <= 0.0f) {
+        return F_func(Q_r, P_r);
+    }
+    //Or estimate flux in the star region
+    else {
+        const float4 F_l = F_func(Q_l, P_l);
+        const float4 F_r = F_func(Q_r, P_r);
+        const float4 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l);
+        return flux;
+    }
+}
+
+
+
+
+
+
+
+/**
+  * Central upwind flux function
+  */
+__device__ float4 CentralUpwindFlux(const float4 Qm, const float4 Qp, const float gamma) {
+    
+    const float Pp = pressure(Qp, gamma);
+    const float4 Fp = F_func(Qp, Pp);
+    const float up = Qp.y / Qp.x;   // rho*u / rho
+    const float cp = sqrt(gamma*Pp/Qp.x); // sqrt(gamma*P/rho)
+
+    const float Pm = pressure(Qm, gamma);
+    const float4 Fm = F_func(Qm, Pm);
+    const float um = Qm.y / Qm.x;   // rho*u / rho
+    const float cm = sqrt(gamma*Pm/Qm.x); // sqrt(gamma*P/rho)
+    
+    const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed
+    const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed
+    
+    return  ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am);
+}

GPUSimulators/cuda/SWE2D_FORCE.cu

@@ -0,0 +1,143 @@
+/*
+This OpenCL kernel implements the classical Lax-Friedrichs scheme
+for the shallow water equations, with edge fluxes.
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__ 
+void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_, const float dx_, const float dt_) {
+    //Compute fluxes along the x axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][j][i+1],
+                                          Q[1][j][i+1],
+                                          Q[2][j][i+1]);
+            const float3 Qm = make_float3(Q[0][j][i],
+                                          Q[1][j][i],
+                                          Q[2][j][i]);
+                                       
+            // Computed flux
+            const float3 flux = FORCE_1D_flux(Qm, Qp, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__ 
+void computeFluxG(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_, const float dy_, const float dt_) {
+    //Compute fluxes along the y axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][j+1][i],
+                                          Q[2][j+1][i],
+                                          Q[1][j+1][i]);
+            const float3 Qm = make_float3(Q[0][j][i],
+                                          Q[2][j][i],
+                                          Q[1][j][i]);
+
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = FORCE_1D_flux(Qm, Qp, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+extern "C" {
+__global__ void FORCEKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    __shared__ float Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    __syncthreads();
+    
+    //Compute flux along x, and evolve
+    computeFluxF(Q, F, g_, dx_, dt_);
+    __syncthreads();
+    evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+    __syncthreads();
+    
+    //Compute flux along y, and evolve
+    computeFluxG(Q, F, g_, dy_, dt_);
+    __syncthreads();
+    evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+    __syncthreads();
+    
+    //Write to main memory
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_FORCE.cu.hip

@@ -0,0 +1,144 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the classical Lax-Friedrichs scheme
+for the shallow water equations, with edge fluxes.
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__ 
+void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_, const float dx_, const float dt_) {
+    //Compute fluxes along the x axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][j][i+1],
+                                          Q[1][j][i+1],
+                                          Q[2][j][i+1]);
+            const float3 Qm = make_float3(Q[0][j][i],
+                                          Q[1][j][i],
+                                          Q[2][j][i]);
+                                       
+            // Computed flux
+            const float3 flux = FORCE_1D_flux(Qm, Qp, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__ 
+void computeFluxG(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_, const float dy_, const float dt_) {
+    //Compute fluxes along the y axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][j+1][i],
+                                          Q[2][j+1][i],
+                                          Q[1][j+1][i]);
+            const float3 Qm = make_float3(Q[0][j][i],
+                                          Q[2][j][i],
+                                          Q[1][j][i]);
+
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = FORCE_1D_flux(Qm, Qp, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+extern "C" {
+__global__ void FORCEKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    __shared__ float Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    __syncthreads();
+    
+    //Compute flux along x, and evolve
+    computeFluxF(Q, F, g_, dx_, dt_);
+    __syncthreads();
+    evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+    __syncthreads();
+    
+    //Compute flux along y, and evolve
+    computeFluxG(Q, F, g_, dy_, dt_);
+    __syncthreads();
+    evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+    __syncthreads();
+    
+    //Write to main memory
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_HLL.cu

@@ -0,0 +1,161 @@
+/*
+This GPU kernel implements the HLL flux
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Q_r = make_float3(Q[0][j][i+1],
+                                           Q[1][j][i+1],
+                                           Q[2][j][i+1]);
+            const float3 Q_l = make_float3(Q[0][j][i],
+                                           Q[1][j][i],
+                                           Q[2][j][i]);
+                       
+            // Computed flux
+            const float3 flux = HLL_flux(Q_l, Q_r, g_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_) {
+    //Compute fluxes along the y axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Q_r = make_float3(Q[0][j+1][i],
+                                           Q[2][j+1][i],
+                                           Q[1][j+1][i]);
+            const float3 Q_l = make_float3(Q[0][j][i],
+                                           Q[2][j][i],
+                                           Q[1][j][i]);
+                                       
+            // Computed flux
+            //Note that we here swap hu and hv back to the original
+            const float3 flux = HLL_flux(Q_l, Q_r, g_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C" {
+    
+__global__ void HLLKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    //Shared memory variables
+    __shared__ float Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Compute F flux
+    computeFluxF(Q, F, g_);
+    __syncthreads();
+    
+    evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+    __syncthreads();
+    
+    //Compute G flux
+    computeFluxG(Q, F, g_);
+    __syncthreads();
+    
+    evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+    __syncthreads();
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_HLL.cu.hip

@@ -0,0 +1,162 @@
+#include "hip/hip_runtime.h"
+/*
+This GPU kernel implements the HLL flux
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Q_r = make_float3(Q[0][j][i+1],
+                                           Q[1][j][i+1],
+                                           Q[2][j][i+1]);
+            const float3 Q_l = make_float3(Q[0][j][i],
+                                           Q[1][j][i],
+                                           Q[2][j][i]);
+                       
+            // Computed flux
+            const float3 flux = HLL_flux(Q_l, Q_r, g_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float g_) {
+    //Compute fluxes along the y axis
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Q_r = make_float3(Q[0][j+1][i],
+                                           Q[2][j+1][i],
+                                           Q[1][j+1][i]);
+            const float3 Q_l = make_float3(Q[0][j][i],
+                                           Q[2][j][i],
+                                           Q[1][j][i]);
+                                       
+            // Computed flux
+            //Note that we here swap hu and hv back to the original
+            const float3 flux = HLL_flux(Q_l, Q_r, g_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C" {
+    
+__global__ void HLLKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    //Shared memory variables
+    __shared__ float Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Compute F flux
+    computeFluxF(Q, F, g_);
+    __syncthreads();
+    
+    evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+    __syncthreads();
+    
+    //Compute G flux
+    computeFluxG(Q, F, g_);
+    __syncthreads();
+    
+    evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+    __syncthreads();
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_HLL2.cu

@@ -0,0 +1,216 @@
+/*
+This OpenCL kernel implements the second order HLL flux
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float3 Q_rl = make_float3(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Q_rr = make_float3(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1]);
+                                         
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i]);
+                                        
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+
+            // Compute flux based on prediction
+            const float3 flux = HLL_flux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float3 Q_rl = make_float3(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i]);
+            const float3 Q_rr = make_float3(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i]);
+                                        
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i]);
+                                     
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+            
+            // Compute flux based on prediction
+            const float3 flux = HLL_flux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+extern "C" {
+__global__ void HLL2Kernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+            
+    //Shared memory variables
+    __shared__ float  Q[3][h+4][w+4];
+    __shared__ float Qx[3][h+4][w+4];
+    __shared__ float  F[3][h+4][w+4];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+    
+    
+    
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_HLL2.cu.hip

@@ -0,0 +1,217 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the second order HLL flux
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float3 Q_rl = make_float3(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Q_rr = make_float3(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1]);
+                                         
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i]);
+                                        
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+
+            // Compute flux based on prediction
+            const float3 flux = HLL_flux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float3 Q_rl = make_float3(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i]);
+            const float3 Q_rr = make_float3(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i]);
+                                        
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i]);
+                                     
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+            
+            // Compute flux based on prediction
+            const float3 flux = HLL_flux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+extern "C" {
+__global__ void HLL2Kernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+            
+    //Shared memory variables
+    __shared__ float  Q[3][h+4][w+4];
+    __shared__ float Qx[3][h+4][w+4];
+    __shared__ float  F[3][h+4][w+4];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+    
+    
+    
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_KP07.cu

@@ -0,0 +1,233 @@
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1],
+                  const float g_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    {
+        int j=ty;
+        const int l = j + 2; //Skip ghost cells
+        for (int i=tx; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            const int k = i + 1;
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][l][k+1] - 0.5f*Qx[0][j][i+1],
+                                          Q[1][l][k+1] - 0.5f*Qx[1][j][i+1],
+                                          Q[2][l][k+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Qm = make_float3(Q[0][l][k  ] + 0.5f*Qx[0][j][i  ],
+                                          Q[1][l][k  ] + 0.5f*Qx[1][j][i  ],
+                                          Q[2][l][k  ] + 0.5f*Qx[2][j][i  ]);
+                                       
+            // Computed flux
+            const float3 flux = CentralUpwindFlux(Qm, Qp, g_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }    
+}
+
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1],
+                  const float g_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    for (int j=ty; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        const int l = j + 1;
+        {
+            int i=tx;
+            const int k = i + 2; //Skip ghost cells
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][l+1][k] - 0.5f*Qy[0][j+1][i],
+                                          Q[2][l+1][k] - 0.5f*Qy[2][j+1][i],
+                                          Q[1][l+1][k] - 0.5f*Qy[1][j+1][i]);
+            const float3 Qm = make_float3(Q[0][l  ][k] + 0.5f*Qy[0][j  ][i],
+                                          Q[2][l  ][k] + 0.5f*Qy[2][j  ][i],
+                                          Q[1][l  ][k] + 0.5f*Qy[1][j  ][i]);
+                                       
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = CentralUpwindFlux(Qm, Qp, g_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+__device__ void minmodSlopeX(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float theta_) {
+    //Reconstruct slopes along x axis
+    for (int p=0; p<3; ++p) {
+        {
+            const int j = threadIdx.y+2;
+            for (int i=threadIdx.x+1; i<BLOCK_WIDTH+3; i+=BLOCK_WIDTH) {
+                Qx[p][j-2][i-1] = minmodSlope(Q[p][j][i-1], Q[p][j][i], Q[p][j][i+1], theta_);
+            }
+        }
+    }
+}
+
+
+/**
+  * Reconstructs a minmod slope for a whole block along the ordinate
+  */
+__device__ void minmodSlopeY(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float theta_) {
+    //Reconstruct slopes along y axis
+    for (int p=0; p<3; ++p) {
+        const int i = threadIdx.x + 2;
+        for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+3; j+=BLOCK_HEIGHT) {
+            {
+                Qy[p][j-1][i-2] = minmodSlope(Q[p][j-1][i], Q[p][j][i], Q[p][j+1][i], theta_);
+            }
+        }
+    }
+}
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+__global__ void KP07Kernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_order_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+        
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+
+    //Index of cell within domain
+    const int ti = blockDim.x*blockIdx.x + threadIdx.x + 2; //Skip global ghost cells, i.e., +2
+    const int tj = blockDim.y*blockIdx.y + threadIdx.y + 2;
+    
+    //Shared memory variables
+    __shared__ float Q[3][h+4][w+4];
+    __shared__ float Qx[3][h+2][w+2];
+    __shared__ float Qy[3][h+2][w+2];
+    __shared__ float  F[3][h+1][w+1];
+    __shared__ float  G[3][h+1][w+1];
+    
+    
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    
+    //Reconstruct slopes along x and axis
+    minmodSlopeX(Q, Qx, theta_);
+    minmodSlopeY(Q, Qy, theta_);
+    __syncthreads();
+    
+    
+    //Compute fluxes along the x and y axis
+    computeFluxF(Q, Qx, F, g_);
+    computeFluxG(Q, Qy, G, g_);
+    __syncthreads();
+    
+    
+    //Sum fluxes and advance in time for all internal cells
+    if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) {
+        const int i = tx + 2; //Skip local ghost cells, i.e., +2
+        const int j = ty + 2;
+        
+        Q[0][j][i] += (F[0][ty][tx] - F[0][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[0][ty][tx] - G[0][ty+1][tx  ]) * dt_ / dy_;
+        Q[1][j][i] += (F[1][ty][tx] - F[1][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[1][ty][tx] - G[1][ty+1][tx  ]) * dt_ / dy_;
+        Q[2][j][i] += (F[2][ty][tx] - F[2][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[2][ty][tx] - G[2][ty+1][tx  ]) * dt_ / dy_;
+
+        float* const h_row  = (float*) ((char*) h1_ptr_ + h1_pitch_*tj);
+        float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj);
+        float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj);
+
+        if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) {
+            //Write to main memory
+            h_row[ti]  = 0.5f*(h_row[ti]  + Q[0][j][i]);
+            hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]);
+            hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]);
+        }
+        else {
+            h_row[ti]  = Q[0][j][i];
+            hu_row[ti] = Q[1][j][i];
+            hv_row[ti] = Q[2][j][i];
+        }
+    }
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+} //extern "C"

GPUSimulators/cuda/SWE2D_KP07.cu.hip

@@ -0,0 +1,234 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1],
+                  const float g_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    {
+        int j=ty;
+        const int l = j + 2; //Skip ghost cells
+        for (int i=tx; i<BLOCK_WIDTH+1; i+=BLOCK_WIDTH) {
+            const int k = i + 1;
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][l][k+1] - 0.5f*Qx[0][j][i+1],
+                                          Q[1][l][k+1] - 0.5f*Qx[1][j][i+1],
+                                          Q[2][l][k+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Qm = make_float3(Q[0][l][k  ] + 0.5f*Qx[0][j][i  ],
+                                          Q[1][l][k  ] + 0.5f*Qx[1][j][i  ],
+                                          Q[2][l][k  ] + 0.5f*Qx[2][j][i  ]);
+                                       
+            // Computed flux
+            const float3 flux = CentralUpwindFlux(Qm, Qp, g_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }    
+}
+
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  float G[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1],
+                  const float g_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    for (int j=ty; j<BLOCK_HEIGHT+1; j+=BLOCK_HEIGHT) {
+        const int l = j + 1;
+        {
+            int i=tx;
+            const int k = i + 2; //Skip ghost cells
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][l+1][k] - 0.5f*Qy[0][j+1][i],
+                                          Q[2][l+1][k] - 0.5f*Qy[2][j+1][i],
+                                          Q[1][l+1][k] - 0.5f*Qy[1][j+1][i]);
+            const float3 Qm = make_float3(Q[0][l  ][k] + 0.5f*Qy[0][j  ][i],
+                                          Q[2][l  ][k] + 0.5f*Qy[2][j  ][i],
+                                          Q[1][l  ][k] + 0.5f*Qy[1][j  ][i]);
+                                       
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = CentralUpwindFlux(Qm, Qp, g_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+__device__ void minmodSlopeX(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float theta_) {
+    //Reconstruct slopes along x axis
+    for (int p=0; p<3; ++p) {
+        {
+            const int j = threadIdx.y+2;
+            for (int i=threadIdx.x+1; i<BLOCK_WIDTH+3; i+=BLOCK_WIDTH) {
+                Qx[p][j-2][i-1] = minmodSlope(Q[p][j][i-1], Q[p][j][i], Q[p][j][i+1], theta_);
+            }
+        }
+    }
+}
+
+
+/**
+  * Reconstructs a minmod slope for a whole block along the ordinate
+  */
+__device__ void minmodSlopeY(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float Qy[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2],
+                  const float theta_) {
+    //Reconstruct slopes along y axis
+    for (int p=0; p<3; ++p) {
+        const int i = threadIdx.x + 2;
+        for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+3; j+=BLOCK_HEIGHT) {
+            {
+                Qy[p][j-1][i-2] = minmodSlope(Q[p][j-1][i], Q[p][j][i], Q[p][j+1][i], theta_);
+            }
+        }
+    }
+}
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+__global__ void KP07Kernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_order_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+        
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+
+    //Index of cell within domain
+    const int ti = blockDim.x*blockIdx.x + threadIdx.x + 2; //Skip global ghost cells, i.e., +2
+    const int tj = blockDim.y*blockIdx.y + threadIdx.y + 2;
+    
+    //Shared memory variables
+    __shared__ float Q[3][h+4][w+4];
+    __shared__ float Qx[3][h+2][w+2];
+    __shared__ float Qy[3][h+2][w+2];
+    __shared__ float  F[3][h+1][w+1];
+    __shared__ float  G[3][h+1][w+1];
+    
+    
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    
+    //Reconstruct slopes along x and axis
+    minmodSlopeX(Q, Qx, theta_);
+    minmodSlopeY(Q, Qy, theta_);
+    __syncthreads();
+    
+    
+    //Compute fluxes along the x and y axis
+    computeFluxF(Q, Qx, F, g_);
+    computeFluxG(Q, Qy, G, g_);
+    __syncthreads();
+    
+    
+    //Sum fluxes and advance in time for all internal cells
+    if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) {
+        const int i = tx + 2; //Skip local ghost cells, i.e., +2
+        const int j = ty + 2;
+        
+        Q[0][j][i] += (F[0][ty][tx] - F[0][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[0][ty][tx] - G[0][ty+1][tx  ]) * dt_ / dy_;
+        Q[1][j][i] += (F[1][ty][tx] - F[1][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[1][ty][tx] - G[1][ty+1][tx  ]) * dt_ / dy_;
+        Q[2][j][i] += (F[2][ty][tx] - F[2][ty  ][tx+1]) * dt_ / dx_ 
+                    + (G[2][ty][tx] - G[2][ty+1][tx  ]) * dt_ / dy_;
+
+        float* const h_row  = (float*) ((char*) h1_ptr_ + h1_pitch_*tj);
+        float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj);
+        float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj);
+
+        if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) {
+            //Write to main memory
+            h_row[ti]  = 0.5f*(h_row[ti]  + Q[0][j][i]);
+            hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]);
+            hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]);
+        }
+        else {
+            h_row[ti]  = Q[0][j][i];
+            hu_row[ti] = Q[1][j][i];
+            hv_row[ti] = Q[2][j][i];
+        }
+    }
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+} //extern "C"

GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu

@@ -0,0 +1,216 @@
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+template <int w, int h, int gc_x, int gc_y>
+__device__
+void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x],
+                  float Qx[3][h+2*gc_y][w+2*gc_x],
+                  float F[3][h+2*gc_y][w+2*gc_x],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+        for (int i=threadIdx.x+1; i<w+2*gc_x-2; i+=w) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float3 Q_rl = make_float3(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Q_rr = make_float3(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1]);
+
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i]);
+                                    
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+
+            // Compute flux based on prediction
+            const float3 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }    
+}
+
+template <int w, int h, int gc_x, int gc_y>
+__device__
+void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x],
+                  float Qy[3][h+2*gc_y][w+2*gc_x],
+                  float G[3][h+2*gc_y][w+2*gc_x],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<h+2*gc_y-2; j+=h) {
+        for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float3 Q_rl = make_float3(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i]);
+            const float3 Q_rr = make_float3(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i]);
+                                        
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i]);
+                                     
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+            
+            // Compute flux based on prediction
+            const float3 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+    
+    
+    
+    
+__global__ void KP07DimsplitKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_, 
+        
+        //Output CFL
+        float* cfl_) {
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+        
+    //Shared memory variables
+    __shared__ float  Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float Qx[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float  F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    if (step_ == 0) {
+        //Along X
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF<w, h, gc_x, gc_y>(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Along Y
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG<w, h, gc_x, gc_y>(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    else {
+        //Along Y
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG<w, h, gc_x, gc_y>(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Along X
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF<w, h, gc_x, gc_y>(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+
+
+
+
+
+
+
+
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip

@@ -0,0 +1,217 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+#include "limiters.h"
+
+
+template <int w, int h, int gc_x, int gc_y>
+__device__
+void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x],
+                  float Qx[3][h+2*gc_y][w+2*gc_x],
+                  float F[3][h+2*gc_y][w+2*gc_x],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+        for (int i=threadIdx.x+1; i<w+2*gc_x-2; i+=w) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            const float3 Q_rl = make_float3(Q[0][j][i+1] - 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] - 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] - 0.5f*Qx[2][j][i+1]);
+            const float3 Q_rr = make_float3(Q[0][j][i+1] + 0.5f*Qx[0][j][i+1],
+                                            Q[1][j][i+1] + 0.5f*Qx[1][j][i+1],
+                                            Q[2][j][i+1] + 0.5f*Qx[2][j][i+1]);
+
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] - 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] - 0.5f*Qx[2][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qx[0][j][i],
+                                            Q[1][j][i] + 0.5f*Qx[1][j][i],
+                                            Q[2][j][i] + 0.5f*Qx[2][j][i]);
+                                    
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dx_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dx_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+
+            // Compute flux based on prediction
+            const float3 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }    
+}
+
+template <int w, int h, int gc_x, int gc_y>
+__device__
+void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x],
+                  float Qy[3][h+2*gc_y][w+2*gc_x],
+                  float G[3][h+2*gc_y][w+2*gc_x],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<h+2*gc_y-2; j+=h) {
+        for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+            // Reconstruct point values of Q at the left and right hand side 
+            // of the cell for both the left (i) and right (i+1) cell 
+            //NOte that hu and hv are swapped ("transposing" the domain)!
+            const float3 Q_rl = make_float3(Q[0][j+1][i] - 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] - 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] - 0.5f*Qy[1][j+1][i]);
+            const float3 Q_rr = make_float3(Q[0][j+1][i] + 0.5f*Qy[0][j+1][i],
+                                            Q[2][j+1][i] + 0.5f*Qy[2][j+1][i],
+                                            Q[1][j+1][i] + 0.5f*Qy[1][j+1][i]);
+                                        
+            const float3 Q_ll = make_float3(Q[0][j][i] - 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] - 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] - 0.5f*Qy[1][j][i]);
+            const float3 Q_lr = make_float3(Q[0][j][i] + 0.5f*Qy[0][j][i],
+                                            Q[2][j][i] + 0.5f*Qy[2][j][i],
+                                            Q[1][j][i] + 0.5f*Qy[1][j][i]);
+                                     
+            //Evolve half a timestep (predictor step)
+            const float3 Q_r_bar = Q_rl + dt_/(2.0f*dy_) * (F_func(Q_rl, g_) - F_func(Q_rr, g_));
+            const float3 Q_l_bar = Q_lr + dt_/(2.0f*dy_) * (F_func(Q_ll, g_) - F_func(Q_lr, g_));
+            
+            // Compute flux based on prediction
+            const float3 flux = CentralUpwindFlux(Q_l_bar, Q_r_bar, g_);
+            
+            //Write to shared memory
+            //Note that we here swap hu and hv back to the original
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+/**
+  * This unsplit kernel computes the 2D numerical scheme with a TVD RK2 time integration scheme
+  */
+extern "C" {
+    
+    
+    
+    
+__global__ void KP07DimsplitKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        float theta_,
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_, 
+        
+        //Output CFL
+        float* cfl_) {
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+        
+    //Shared memory variables
+    __shared__ float  Q[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float Qx[vars][h+2*gc_y][w+2*gc_x];
+    __shared__ float  F[vars][h+2*gc_y][w+2*gc_x];
+    
+    //Read into shared memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    if (step_ == 0) {
+        //Along X
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF<w, h, gc_x, gc_y>(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Along Y
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG<w, h, gc_x, gc_y>(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    else {
+        //Along Y
+        minmodSlopeY<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxG<w, h, gc_x, gc_y>(Q, Qx, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Along X
+        minmodSlopeX<w, h, gc_x, gc_y, vars>(Q, Qx, theta_);
+        __syncthreads();
+        computeFluxF<w, h, gc_x, gc_y>(Q, Qx, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+
+
+
+
+
+
+
+
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_LxF.cu

@@ -0,0 +1,168 @@
+/*
+This OpenCL kernel implements the classical Lax-Friedrichs scheme
+for the shallow water equations, with edge fluxes.
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+template <int block_width, int block_height>
+__device__ 
+void computeFluxF(float Q[3][block_height+2][block_width+2],
+                  float F[3][block_height][block_width+1],
+                  const float g_, const float dx_, const float dt_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    {
+        const int j=ty;
+        const int l = j + 1; //Skip ghost cells
+        for (int i=tx; i<block_width+1; i+=block_width) {
+            const int k = i;
+            
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][l][k+1],
+                                          Q[1][l][k+1],
+                                          Q[2][l][k+1]);
+            const float3 Qm = make_float3(Q[0][l][k],
+                                          Q[1][l][k],
+                                          Q[2][l][k]);
+                                       
+            // Computed flux
+            const float3 flux = LxF_2D_flux(Qm, Qp, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */ 
+template <int block_width, int block_height>
+__device__
+void computeFluxG(float Q[3][block_height+2][block_width+2],
+                  float G[3][block_height+1][block_width],
+                  const float g_, const float dy_, const float dt_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    for (int j=ty; j<block_height+1; j+=block_height) {
+        const int l = j;
+        {
+            const int i=tx;
+            const int k = i + 1; //Skip ghost cells
+            
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][l+1][k],
+                                          Q[2][l+1][k],
+                                          Q[1][l+1][k]);
+            const float3 Qm = make_float3(Q[0][l][k],
+                                          Q[2][l][k],
+                                          Q[1][l][k]);
+
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = LxF_2D_flux(Qm, Qp, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }  
+}
+
+
+extern "C" {
+__global__ 
+void LxFKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    __shared__ float Q[vars][h+2][w+2];
+    __shared__ float F[vars][h  ][w+1];
+    __shared__ float G[vars][h+1][w  ];
+    
+    //Read from global memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Compute fluxes along the x and y axis
+    computeFluxF<w, h>(Q, F, g_, dx_, dt_);
+    computeFluxG<w, h>(Q, G, g_, dy_, dt_);
+    __syncthreads();
+
+    //Evolve for all cells
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    const int i = tx + 1; //Skip local ghost cells, i.e., +1
+    const int j = ty + 1;
+    
+    Q[0][j][i] += (F[0][ty][tx] - F[0][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[0][ty][tx] - G[0][ty+1][tx  ]) * dt_ / dy_;
+    Q[1][j][i] += (F[1][ty][tx] - F[1][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[1][ty][tx] - G[1][ty+1][tx  ]) * dt_ / dy_;
+    Q[2][j][i] += (F[2][ty][tx] - F[2][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[2][ty][tx] - G[2][ty+1][tx  ]) * dt_ / dy_;
+    __syncthreads();
+
+    //Write to main memory
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"
+

GPUSimulators/cuda/SWE2D_LxF.cu.hip

@@ -0,0 +1,169 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the classical Lax-Friedrichs scheme
+for the shallow water equations, with edge fluxes.
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+template <int block_width, int block_height>
+__device__ 
+void computeFluxF(float Q[3][block_height+2][block_width+2],
+                  float F[3][block_height][block_width+1],
+                  const float g_, const float dx_, const float dt_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    {
+        const int j=ty;
+        const int l = j + 1; //Skip ghost cells
+        for (int i=tx; i<block_width+1; i+=block_width) {
+            const int k = i;
+            
+            // Q at interface from the right and left
+            const float3 Qp = make_float3(Q[0][l][k+1],
+                                          Q[1][l][k+1],
+                                          Q[2][l][k+1]);
+            const float3 Qm = make_float3(Q[0][l][k],
+                                          Q[1][l][k],
+                                          Q[2][l][k]);
+                                       
+            // Computed flux
+            const float3 flux = LxF_2D_flux(Qm, Qp, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */ 
+template <int block_width, int block_height>
+__device__
+void computeFluxG(float Q[3][block_height+2][block_width+2],
+                  float G[3][block_height+1][block_width],
+                  const float g_, const float dy_, const float dt_) {
+    //Index of thread within block
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    
+    for (int j=ty; j<block_height+1; j+=block_height) {
+        const int l = j;
+        {
+            const int i=tx;
+            const int k = i + 1; //Skip ghost cells
+            
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Qp = make_float3(Q[0][l+1][k],
+                                          Q[2][l+1][k],
+                                          Q[1][l+1][k]);
+            const float3 Qm = make_float3(Q[0][l][k],
+                                          Q[2][l][k],
+                                          Q[1][l][k]);
+
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = LxF_2D_flux(Qm, Qp, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }  
+}
+
+
+extern "C" {
+__global__ 
+void LxFKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_,
+        
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_,
+        
+        //Output CFL
+        float* cfl_) {
+    
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 1;
+    const unsigned int gc_y = 1;
+    const unsigned int vars = 3;
+    
+    __shared__ float Q[vars][h+2][w+2];
+    __shared__ float F[vars][h  ][w+1];
+    __shared__ float G[vars][h+1][w  ];
+    
+    //Read from global memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    
+    //Compute fluxes along the x and y axis
+    computeFluxF<w, h>(Q, F, g_, dx_, dt_);
+    computeFluxG<w, h>(Q, G, g_, dy_, dt_);
+    __syncthreads();
+
+    //Evolve for all cells
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    const int i = tx + 1; //Skip local ghost cells, i.e., +1
+    const int j = ty + 1;
+    
+    Q[0][j][i] += (F[0][ty][tx] - F[0][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[0][ty][tx] - G[0][ty+1][tx  ]) * dt_ / dy_;
+    Q[1][j][i] += (F[1][ty][tx] - F[1][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[1][ty][tx] - G[1][ty+1][tx  ]) * dt_ / dy_;
+    Q[2][j][i] += (F[2][ty][tx] - F[2][ty  ][tx+1]) * dt_ / dx_ 
+                + (G[2][ty][tx] - G[2][ty+1][tx  ]) * dt_ / dy_;
+    __syncthreads();
+
+    //Write to main memory
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+    
+    //Compute the CFL for this block
+    if (cfl_ != NULL) {
+        writeCfl<w, h, gc_x, gc_y, vars>(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_);
+    }
+}
+
+} // extern "C"
+

GPUSimulators/cuda/SWE2D_WAF.cu

@@ -0,0 +1,178 @@
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Ql2 = make_float3(Q[0][j][i-1], Q[1][j][i-1], Q[2][j][i-1]);
+            const float3 Ql1 = make_float3(Q[0][j][i  ], Q[1][j][i  ], Q[2][j][i  ]);
+            const float3 Qr1 = make_float3(Q[0][j][i+1], Q[1][j][i+1], Q[2][j][i+1]);
+            const float3 Qr2 = make_float3(Q[0][j][i+2], Q[1][j][i+2], Q[2][j][i+2]);
+
+            // Computed flux
+            const float3 flux = WAF_1D_flux(Ql2, Ql1, Qr1, Qr2, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Ql2 = make_float3(Q[0][j-1][i], Q[2][j-1][i], Q[1][j-1][i]);
+            const float3 Ql1 = make_float3(Q[0][j  ][i], Q[2][j  ][i], Q[1][j  ][i]);
+            const float3 Qr1 = make_float3(Q[0][j+1][i], Q[2][j+1][i], Q[1][j+1][i]);
+            const float3 Qr2 = make_float3(Q[0][j+2][i], Q[2][j+2][i], Q[1][j+2][i]);
+            
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = WAF_1D_flux(Ql2, Ql1, Qr1, Qr2, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C" {
+__global__ void WAFKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_, 
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_) {   
+            
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+         
+    //Shared memory variables
+    __shared__ float Q[3][h+4][w+4];
+    __shared__ float F[3][h+4][w+4];
+    
+    
+    
+    //Read into shared memory Q from global memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    __syncthreads();
+    
+    
+    
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        computeFluxF(Q, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the y axis and evolve
+        computeFluxG(Q, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        computeFluxG(Q, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        computeFluxF(Q, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+
+
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+}
+
+} // extern "C"

GPUSimulators/cuda/SWE2D_WAF.cu.hip

@@ -0,0 +1,179 @@
+#include "hip/hip_runtime.h"
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+#include "common.h"
+#include "SWECommon.h"
+
+
+
+/**
+  * Computes the flux along the x axis for all faces
+  */
+__device__
+void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dx_, const float dt_) {
+    for (int j=threadIdx.y; j<BLOCK_HEIGHT+4; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x+1; i<BLOCK_WIDTH+2; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            const float3 Ql2 = make_float3(Q[0][j][i-1], Q[1][j][i-1], Q[2][j][i-1]);
+            const float3 Ql1 = make_float3(Q[0][j][i  ], Q[1][j][i  ], Q[2][j][i  ]);
+            const float3 Qr1 = make_float3(Q[0][j][i+1], Q[1][j][i+1], Q[2][j][i+1]);
+            const float3 Qr2 = make_float3(Q[0][j][i+2], Q[1][j][i+2], Q[2][j][i+2]);
+
+            // Computed flux
+            const float3 flux = WAF_1D_flux(Ql2, Ql1, Qr1, Qr2, g_, dx_, dt_);
+            F[0][j][i] = flux.x;
+            F[1][j][i] = flux.y;
+            F[2][j][i] = flux.z;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+/**
+  * Computes the flux along the y axis for all faces
+  */
+__device__
+void computeFluxG(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  float G[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4],
+                  const float g_, const float dy_, const float dt_) {
+    for (int j=threadIdx.y+1; j<BLOCK_HEIGHT+2; j+=BLOCK_HEIGHT) {
+        for (int i=threadIdx.x; i<BLOCK_WIDTH+4; i+=BLOCK_WIDTH) {
+            // Q at interface from the right and left
+            // Note that we swap hu and hv
+            const float3 Ql2 = make_float3(Q[0][j-1][i], Q[2][j-1][i], Q[1][j-1][i]);
+            const float3 Ql1 = make_float3(Q[0][j  ][i], Q[2][j  ][i], Q[1][j  ][i]);
+            const float3 Qr1 = make_float3(Q[0][j+1][i], Q[2][j+1][i], Q[1][j+1][i]);
+            const float3 Qr2 = make_float3(Q[0][j+2][i], Q[2][j+2][i], Q[1][j+2][i]);
+            
+            // Computed flux
+            // Note that we swap back
+            const float3 flux = WAF_1D_flux(Ql2, Ql1, Qr1, Qr2, g_, dy_, dt_);
+            G[0][j][i] = flux.x;
+            G[1][j][i] = flux.z;
+            G[2][j][i] = flux.y;
+        }
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C" {
+__global__ void WAFKernel(
+        int nx_, int ny_,
+        float dx_, float dy_, float dt_,
+        float g_, 
+        
+        int step_,
+        int boundary_conditions_,
+        
+        //Input h^n
+        float* h0_ptr_, int h0_pitch_,
+        float* hu0_ptr_, int hu0_pitch_,
+        float* hv0_ptr_, int hv0_pitch_,
+        
+        //Output h^{n+1}
+        float* h1_ptr_, int h1_pitch_,
+        float* hu1_ptr_, int hu1_pitch_,
+        float* hv1_ptr_, int hv1_pitch_) {   
+            
+    const unsigned int w = BLOCK_WIDTH;
+    const unsigned int h = BLOCK_HEIGHT;
+    const unsigned int gc_x = 2;
+    const unsigned int gc_y = 2;
+    const unsigned int vars = 3;
+         
+    //Shared memory variables
+    __shared__ float Q[3][h+4][w+4];
+    __shared__ float F[3][h+4][w+4];
+    
+    
+    
+    //Read into shared memory Q from global memory
+    readBlock<w, h, gc_x, gc_y,  1,  1>( h0_ptr_,  h0_pitch_, Q[0], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y, -1,  1>(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_);
+    readBlock<w, h, gc_x, gc_y,  1, -1>(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_);
+    __syncthreads();
+    
+    
+    
+    //Step 0 => evolve x first, then y
+    if (step_ == 0) {
+        //Compute fluxes along the x axis and evolve
+        computeFluxF(Q, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the y axis and evolve
+        computeFluxG(Q, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+    }
+    //Step 1 => evolve y first, then x
+    else {
+        //Compute fluxes along the y axis and evolve
+        computeFluxG(Q, F, g_, dy_, dt_);
+        __syncthreads();
+        evolveG<w, h, gc_x, gc_y, vars>(Q, F, dy_, dt_);
+        __syncthreads();
+        
+        //Compute fluxes along the x axis and evolve
+        computeFluxF(Q, F, g_, dx_, dt_);
+        __syncthreads();
+        evolveF<w, h, gc_x, gc_y, vars>(Q, F, dx_, dt_);
+        __syncthreads();
+    }
+
+
+    
+    // Write to main memory for all internal cells
+    writeBlock<w, h, gc_x, gc_y>( h1_ptr_,  h1_pitch_, Q[0], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1);
+    writeBlock<w, h, gc_x, gc_y>(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1);
+}
+
+} // extern "C"

GPUSimulators/cuda/SWECommon.h

@@ -0,0 +1,533 @@
+/*
+These CUDA functions implement different types of numerical flux 
+functions for the shallow water equations
+
+Copyright (C) 2016, 2017, 2018 SINTEF Digital
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+#include "limiters.h"
+
+
+
+
+
+
+__device__ float3 F_func(const float3 Q, const float g) {
+    float3 F;
+
+    F.x = Q.y;                              //hu
+    F.y = Q.y*Q.y / Q.x + 0.5f*g*Q.x*Q.x;   //hu*hu/h + 0.5f*g*h*h;
+    F.z = Q.y*Q.z / Q.x;                    //hu*hv/h;
+
+    return F;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+  * Superbee flux limiter for WAF.
+  * Related to superbee limiter so that WAF_superbee(r, c) = 1 - (1-|c|)*superbee(r)
+  * @param r_ the ratio of upwind change (see Toro 2001, p. 203/204)
+  * @param c_ the courant number for wave k, dt*S_k/dx
+  */
+__device__ float WAF_superbee(float r_, float c_) {
+    // r <= 0.0
+    if (r_ <= 0.0f) { 
+        return 1.0f;
+    }
+    // 0.0 <= r <= 1/2
+    else if (r_ <= 0.5f) { 
+        return 1.0f - 2.0f*(1.0f - fabsf(c_))*r_;
+    }
+    // 1/2 <= r <= 1
+    else if (r_ <= 1.0f) {
+        return fabs(c_);
+    }
+    // 1 <= r <= 2
+    else  if (r_ <= 2.0f) {
+        return 1.0f - (1.0f - fabsf(c_))*r_;
+    }
+    // r >= 2
+    else {
+        return 2.0f*fabsf(c_) - 1.0f;
+    }
+}
+
+
+
+
+__device__ float WAF_albada(float r_, float c_) {
+    if (r_ <= 0.0f) {
+        return 1.0f;
+    }
+    else {
+        return 1.0f - (1.0f - fabsf(c_)) * r_ * (1.0f + r_) / (1.0f + r_*r_);
+    }
+}
+
+__device__ float WAF_minbee(float r_, float c_) {
+    r_ = fmaxf(-1.0f, fminf(2.0f, r_));
+    if (r_ <= 0.0f) {
+        return 1.0f;
+    }
+    if (r_ >= 0.0f && r_ <= 1.0f) {
+        return 1.0f - (1.0f - fabsf(c_)) * r_;
+    }
+    else {
+        return fabsf(c_);
+    }
+}
+
+__device__ float WAF_minmod(float r_, float c_) {
+    return 1.0f - (1.0f - fabsf(c_)) * fmaxf(0.0f, fminf(1.0f, r_));
+}
+
+__device__ float limiterToWAFLimiter(float r_, float c_) {
+    return 1.0f - (1.0f - fabsf(c_))*r_;
+}
+
+// Compute h in the "star region", h^dagger
+__device__ __inline__ float computeHStar(float h_l, float h_r, float u_l, float u_r, float c_l, float c_r, float g_) {
+    
+    //This estimate for the h* gives rise to spurious oscillations. 
+    //return 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r);
+    
+    const float h_tmp = 0.5f * (c_l + c_r) + 0.25f * (u_l - u_r);
+    return h_tmp*h_tmp / g_;
+}
+
+/**
+  * Weighted average flux (Toro 2001, p 200) for interface {i+1/2}
+  * @param r_ The flux limiter parameter (see Toro 2001, p. 203)
+  * @param Q_l2 Q_{i-1}
+  * @param Q_l1 Q_{i}
+  * @param Q_r1 Q_{i+1}
+  * @param Q_r2 Q_{i+2}
+  */
+__device__ float3 WAF_1D_flux(const float3 Q_l2, const float3 Q_l1, const float3 Q_r1, const float3 Q_r2, const float g_, const float dx_, const float dt_) {     
+    const float h_l = Q_l1.x;
+    const float h_r = Q_r1.x;
+    
+    const float h_l2 = Q_l2.x;
+    const float h_r2 = Q_r2.x;
+    
+    // Calculate velocities
+    const float u_l = Q_l1.y / h_l;
+    const float u_r = Q_r1.y / h_r;
+    
+    const float u_l2 = Q_l2.y / h_l2;
+    const float u_r2 = Q_r2.y / h_r2;
+    
+    const float v_l = Q_l1.z / h_l;
+    const float v_r = Q_r1.z / h_r;
+    
+    const float v_l2 = Q_l2.z / h_l2;
+    const float v_r2 = Q_r2.z / h_r2;
+    
+    // Estimate the potential wave speeds
+    const float c_l = sqrt(g_*h_l);
+    const float c_r = sqrt(g_*h_r);
+    
+    const float c_l2 = sqrt(g_*h_l2);
+    const float c_r2 = sqrt(g_*h_r2);
+    
+    // Compute h in the "star region", h^dagger
+    const float h_dag_l = computeHStar(h_l2,  h_l, u_l2,  u_l, c_l2,  c_l, g_);
+    const float h_dag   = computeHStar( h_l,  h_r,  u_l,  u_r,  c_l,  c_r, g_);
+    const float h_dag_r = computeHStar( h_r, h_r2,  u_r, u_r2,  c_r, c_r2, g_);
+    
+    const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag ) ) / h_l;
+    const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag ) ) / h_r;
+    
+    const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f;
+    const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f;
+    
+    // Compute wave speed estimates
+    const float S_l = u_l - c_l*q_l; 
+    const float S_r = u_r + c_r*q_r;
+    const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) );
+    
+    const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1.0, S_star, v_l);
+    const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1.0, S_star, v_r);
+    
+    // Estimate the fluxes in the four regions
+    const float3 F_1 = F_func(Q_l1, g_);
+    const float3 F_4 = F_func(Q_r1, g_);
+    
+    const float3 F_2 = F_1 + S_l*(Q_star_l - Q_l1);
+    const float3 F_3 = F_4 + S_r*(Q_star_r - Q_r1);
+    //const float3 F_2 = F_func(Q_star_l, g_);
+    //const float3 F_3 = F_func(Q_star_r, g_);
+    
+    // Compute the courant numbers for the waves
+    const float c_1 = S_l * dt_ / dx_;
+    const float c_2 = S_star * dt_ / dx_;
+    const float c_3 = S_r * dt_ / dx_;
+    
+    // Compute the "upwind change" vectors for the i-3/2 and i+3/2 interfaces
+    const float eps = 1.0e-6f;
+    const float r_1 = desingularize( (c_1 > 0.0f) ? (h_dag_l - h_l2) : (h_dag_r - h_r), eps) / desingularize((h_dag - h_l), eps);
+    const float r_2 = desingularize( (c_2 > 0.0f) ? (v_l - v_l2) : (v_r2 - v_r), eps ) / desingularize((v_r - v_l), eps);
+    const float r_3 = desingularize( (c_3 > 0.0f) ? (h_l - h_dag_l) : (h_r2 - h_dag_r), eps ) / desingularize((h_r - h_dag), eps);
+        
+    // Compute the limiter
+    // We use h for the nonlinear waves, and v for the middle shear wave 
+    const float A_1 = copysign(1.0f, c_1) * limiterToWAFLimiter(generalized_minmod(r_1, 1.9f), c_1);
+    const float A_2 = copysign(1.0f, c_2) * limiterToWAFLimiter(generalized_minmod(r_2, 1.9f), c_2); 
+    const float A_3 = copysign(1.0f, c_3) * limiterToWAFLimiter(generalized_minmod(r_3, 1.9f), c_3);
+    
+    //Average the fluxes
+    const float3 flux = 0.5f*( F_1 + F_4 )
+                      - 0.5f*( A_1 * (F_2 - F_1)
+                             + A_2 * (F_3 - F_2)
+                             + A_3 * (F_4 - F_3) );
+
+    return flux;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+  * Central upwind flux function
+  */
+__device__ float3 CentralUpwindFlux(const float3 Qm, float3 Qp, const float g) {
+    const float3 Fp = F_func(Qp, g);
+    const float up = Qp.y / Qp.x;   // hu / h
+    const float cp = sqrt(g*Qp.x); // sqrt(g*h)
+
+    const float3 Fm = F_func(Qm, g);
+    const float um = Qm.y / Qm.x;   // hu / h
+    const float cm = sqrt(g*Qm.x); // sqrt(g*h)
+    
+    const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed
+    const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed
+    
+    return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am);
+}
+
+
+
+
+
+
+
+
+
+/**
+  * Godunovs centered scheme (Toro 2001, p 165)
+  */
+__device__ float3 GodC_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) {
+    const float3 F_l = F_func(Q_l, g_);
+    const float3 F_r = F_func(Q_r, g_);
+    
+    const float3 Q_godc = 0.5f*(Q_l + Q_r) + (dt_/dx_)*(F_l - F_r);
+    
+    return F_func(Q_godc, g_);
+}
+    
+
+
+    
+    
+
+
+
+
+/**
+  * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 180)
+  */
+__device__ float3 HLL_flux(const float3 Q_l, const float3 Q_r, const float g_) {    
+    const float h_l = Q_l.x;
+    const float h_r = Q_r.x;
+    
+    // Calculate velocities
+    const float u_l = Q_l.y / h_l;
+    const float u_r = Q_r.y / h_r;
+    
+    // Estimate the potential wave speeds
+    const float c_l = sqrt(g_*h_l);
+    const float c_r = sqrt(g_*h_r);
+    
+    // Compute h in the "star region", h^dagger
+    const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r);
+    
+    const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) );
+    const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) );
+    
+    const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f;
+    const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f;
+    
+    // Compute wave speed estimates
+    const float S_l = u_l - c_l*q_l;
+    const float S_r = u_r + c_r*q_r;
+    
+    //Upwind selection
+    if (S_l >= 0.0f) {
+        return F_func(Q_l, g_);
+    }
+    else if (S_r <= 0.0f) {
+        return F_func(Q_r, g_);
+    }
+    //Or estimate flux in the star region
+    else {
+        const float3 F_l = F_func(Q_l, g_);
+        const float3 F_r = F_func(Q_r, g_);
+        const float3 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l);
+        return flux;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+  * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 181)
+  */
+__device__ float3 HLLC_flux(const float3 Q_l, const float3 Q_r, const float g_) {    
+    const float h_l = Q_l.x;
+    const float h_r = Q_r.x;
+    
+    // Calculate velocities
+    const float u_l = Q_l.y / h_l;
+    const float u_r = Q_r.y / h_r;
+    
+    // Estimate the potential wave speeds
+    const float c_l = sqrt(g_*h_l);
+    const float c_r = sqrt(g_*h_r);
+    
+    // Compute h in the "star region", h^dagger
+    const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r);
+    
+    const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) );
+    const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) );
+    
+    const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f;
+    const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f;
+    
+    // Compute wave speed estimates
+    const float S_l = u_l - c_l*q_l;
+    const float S_r = u_r + c_r*q_r;
+    const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) );
+    
+    const float3 F_l = F_func(Q_l, g_);
+    const float3 F_r = F_func(Q_r, g_);
+    
+    //Upwind selection
+    if (S_l >= 0.0f) {
+        return F_l;
+    }
+    else if (S_r <= 0.0f) {
+        return F_r;
+    }
+    //Or estimate flux in the "left star" region
+    else if (S_l <= 0.0f && 0.0f <=S_star) {
+        const float v_l = Q_l.z / h_l;
+        const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1, S_star, v_l);
+        const float3 flux = F_l + S_l*(Q_star_l - Q_l);
+        return flux;
+    }
+    //Or estimate flux in the "righ star" region
+    else if (S_star <= 0.0f && 0.0f <=S_r) {
+        const float v_r = Q_r.z / h_r;
+        const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1, S_star, v_r);
+        const float3 flux = F_r + S_r*(Q_star_r - Q_r);
+        return flux;
+    }
+    else {
+        return make_float3(-99999.9f, -99999.9f, -99999.9f); //Something wrong here
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+  * Lax-Friedrichs flux (Toro 2001, p 163)
+  */
+__device__ float3 LxF_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) {
+    const float3 F_l = F_func(Q_l, g_);
+    const float3 F_r = F_func(Q_r, g_);
+    
+    return 0.5f*(F_l + F_r) + (dx_/(2.0f*dt_))*(Q_l - Q_r);
+}
+
+
+
+
+
+
+
+
+/**
+  * Lax-Friedrichs extended to 2D
+  */
+__device__ float3 LxF_2D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) {
+    const float3 F_l = F_func(Q_l, g_);
+    const float3 F_r = F_func(Q_r, g_);
+    
+    //Note numerical diffusion for 2D here (0.25)
+    return 0.5f*(F_l + F_r) + (dx_/(4.0f*dt_))*(Q_l - Q_r);
+}
+
+
+
+
+
+
+
+
+
+
+
+
+/**
+  * Richtmeyer / Two-step Lax-Wendroff flux (Toro 2001, p 164)
+  */
+__device__ float3 LxW2_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) {
+    const float3 F_l = F_func(Q_l, g_);
+    const float3 F_r = F_func(Q_r, g_);
+    
+    const float3 Q_lw2 = 0.5f*(Q_l + Q_r) + (dt_/(2.0f*dx_))*(F_l - F_r);
+    
+    return F_func(Q_lw2, g_);
+}
+
+
+
+
+
+
+
+
+
+
+ 
+    
+/**
+  * First Ordered Centered (Toro 2001, p.163)
+  */
+__device__ float3 FORCE_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) {
+    const float3 F_lf = LxF_1D_flux(Q_l, Q_r, g_, dx_, dt_);
+    const float3 F_lw2 = LxW2_1D_flux(Q_l, Q_r, g_, dx_, dt_);
+    return 0.5f*(F_lf + F_lw2);
+}
+
+
+
+
+
+
+
+
+
+
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x],
+        float shmem[h+2*gc_y][w+2*gc_x],
+        const int nx_, const int ny_,
+        const float dx_, const float dy_, const float g_,
+        float* output_) {
+    //Index of thread within block
+    const int tx = threadIdx.x + gc_x;
+    const int ty = threadIdx.y + gc_y;
+    
+    //Index of cell within domain
+    const int ti = blockDim.x*blockIdx.x + tx;
+    const int tj = blockDim.y*blockIdx.y + ty;
+    
+    //Only internal cells
+    if (ti < nx_+gc_x && tj < ny_+gc_y) {
+        const float h = Q[0][ty][tx];
+        const float u   = Q[1][ty][tx] / h;
+        const float v   = Q[2][ty][tx] / h;
+        
+        const float max_u = dx_ / (fabsf(u) + sqrtf(g_*h));
+        const float max_v = dy_ / (fabsf(v) + sqrtf(g_*h));
+        
+        shmem[ty][tx] = fminf(max_u, max_v);
+    }
+    __syncthreads();
+    
+    //One row of threads loop over all rows
+    if (ti < nx_+gc_x && tj < ny_+gc_y) {
+        if (ty == gc_y) {
+            float min_val = shmem[ty][tx];
+            const int max_y = min(h, ny_+gc_y - tj);
+            for (int j=gc_y; j<max_y+gc_y; j++) {
+                min_val = fminf(min_val, shmem[j][tx]);
+            }
+            shmem[ty][tx] = min_val;
+        }
+    }
+    __syncthreads();
+    
+    //One thread loops over first row to find global max
+    if (tx == gc_x && ty == gc_y) {
+        float min_val = shmem[ty][tx];
+        const int max_x = min(w, nx_+gc_x - ti);
+        for (int i=gc_x; i<max_x+gc_x; ++i) {
+            min_val = fminf(min_val, shmem[ty][i]);
+        }
+        
+        const int idx = gridDim.x*blockIdx.y + blockIdx.x;
+        output_[idx] = min_val;
+    }
+}
+

GPUSimulators/cuda/common.h

@@ -0,0 +1,560 @@
+/*
+This OpenCL kernel implements the Kurganov-Petrova numerical scheme 
+for the shallow water equations, described in 
+A. Kurganov & Guergana Petrova
+A Second-Order Well-Balanced Positivity Preserving Central-Upwind
+Scheme for the Saint-Venant System Communications in Mathematical
+Sciences, 5 (2007), 133-160. 
+
+Copyright (C) 2016  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <stddef.h>
+#include <float.h>
+
+/**
+  * Float3 operators 
+  */
+inline __device__ float3 operator*(const float a, const float3 b) {
+    return make_float3(a*b.x, a*b.y, a*b.z);
+}
+
+inline __device__ float3 operator/(const float3 a, const float b) {
+    return make_float3(a.x/b, a.y/b, a.z/b);
+}
+
+inline __device__ float3 operator-(const float3 a, const float3 b) {
+    return make_float3(a.x-b.x, a.y-b.y, a.z-b.z);
+}
+
+inline __device__ float3 operator+(const float3 a, const float3 b) {
+    return make_float3(a.x+b.x, a.y+b.y, a.z+b.z);
+}
+
+/**
+  * Float4 operators 
+  */
+inline __device__ float4 operator*(const float a, const float4 b) {
+    return make_float4(a*b.x, a*b.y, a*b.z, a*b.w);
+}
+
+inline __device__ float4 operator/(const float4 a, const float b) {
+    return make_float4(a.x/b, a.y/b, a.z/b, a.w/b);
+}
+
+inline __device__ float4 operator-(const float4 a, const float4 b) {
+    return make_float4(a.x-b.x, a.y-b.y, a.z-b.z, a.w-b.w);
+}
+
+inline __device__ float4 operator+(const float4 a, const float4 b) {
+    return make_float4(a.x+b.x, a.y+b.y, a.z+b.z, a.w+b.w);
+}
+
+
+
+
+inline __device__ __host__ float clamp(const float f, const float a, const float b) {
+    return fmaxf(a, fminf(f, b));
+}
+
+inline __device__ __host__ int clamp(const int f, const int a, const int b) {
+    return (f < b) ? ( (f > a) ? f : a) : b;
+}
+
+
+
+
+
+__device__ float desingularize(float x_, float eps_) {
+    return copysign(1.0f, x_)*fmaxf(fabsf(x_), fminf(x_*x_/(2.0f*eps_)+0.5f*eps_, eps_));
+}
+
+
+
+
+
+/**
+  * Returns the step stored in the leftmost 16 bits 
+  * of the 32 bit step-order integer
+  */
+inline __device__ int getStep(int step_order_) {
+    return step_order_ >> 16;
+}
+
+/**
+  * Returns the order stored in the rightmost 16 bits 
+  * of the 32 bit step-order integer
+  */
+inline __device__ int getOrder(int step_order_) {
+    return step_order_ & 0x0000FFFF;
+}
+
+
+enum BoundaryCondition {
+    Dirichlet = 0,
+    Neumann = 1,
+    Periodic = 2,
+    Reflective = 3
+};
+
+inline __device__ BoundaryCondition getBCNorth(int bc_) {
+    return static_cast<BoundaryCondition>((bc_ >> 24) & 0x0000000F);
+}
+
+inline __device__ BoundaryCondition getBCSouth(int bc_) {
+    return static_cast<BoundaryCondition>((bc_ >> 16) & 0x0000000F);
+}
+
+inline __device__ BoundaryCondition getBCEast(int bc_) {
+    return static_cast<BoundaryCondition>((bc_ >> 8) & 0x0000000F);
+}
+
+inline __device__ BoundaryCondition getBCWest(int bc_) {
+    return static_cast<BoundaryCondition>((bc_ >> 0) & 0x0000000F);
+}
+
+
+// West boundary
+template<int w, int h, int gc_x, int gc_y, int sign>
+__device__ void bcWestReflective(float Q[h+2*gc_y][w+2*gc_x], 
+                                const int nx_, const int ny_) {
+    for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+        const int i = threadIdx.x + gc_x;
+        const int ti = blockDim.x*blockIdx.x + i;
+        
+        if (gc_x >= 1 && ti == gc_x) {
+            Q[j][i-1] = sign*Q[j][i];
+        }
+        if (gc_x >= 2 && ti == gc_x + 1) {
+            Q[j][i-3] = sign*Q[j][i];
+        }
+        if (gc_x >= 3 && ti == gc_x + 2) {
+            Q[j][i-5] = sign*Q[j][i];
+        }
+        if (gc_x >= 4 && ti == gc_x + 3) {
+            Q[j][i-7] = sign*Q[j][i];
+        }
+        if (gc_x >= 5 && ti == gc_x + 4) {
+            Q[j][i-9] = sign*Q[j][i];
+        }
+    }
+}
+
+
+// East boundary
+template<int w, int h, int gc_x, int gc_y, int sign>
+__device__ void bcEastReflective(float Q[h+2*gc_y][w+2*gc_x], 
+                                const int nx_, const int ny_) {
+    for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+        const int i = threadIdx.x + gc_x;
+        const int ti = blockDim.x*blockIdx.x + i;
+        
+        if (gc_x >= 1 && ti == nx_ + gc_x - 1) {
+            Q[j][i+1] = sign*Q[j][i];
+        }
+        if (gc_x >= 2 && ti == nx_ + gc_x - 2) {
+            Q[j][i+3] = sign*Q[j][i];
+        }
+        if (gc_x >= 3 && ti == nx_ + gc_x - 3) {
+            Q[j][i+5] = sign*Q[j][i];
+        }
+        if (gc_x >= 4 && ti == nx_ + gc_x - 4) {
+            Q[j][i+7] = sign*Q[j][i];
+        }
+        if (gc_x >= 5 && ti == nx_ + gc_x - 5) {
+            Q[j][i+9] = sign*Q[j][i];
+        }
+    }
+}
+    
+    
+// South boundary
+template<int w, int h, int gc_x, int gc_y, int sign>
+__device__ void bcSouthReflective(float Q[h+2*gc_y][w+2*gc_x], 
+                                const int nx_, const int ny_) {
+    for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+        const int j = threadIdx.y + gc_y;
+        const int tj = blockDim.y*blockIdx.y + j;
+
+        if (gc_y >= 1 && tj == gc_y) {
+            Q[j-1][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 2 && tj == gc_y + 1) {
+            Q[j-3][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 3 && tj == gc_y + 2) {
+            Q[j-5][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 4 && tj == gc_y + 3) {
+            Q[j-7][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 5 && tj == gc_y + 4) {
+            Q[j-9][i] = sign*Q[j][i];
+        }
+    }
+}
+
+
+
+    
+// North boundary
+template<int w, int h, int gc_x, int gc_y, int sign>
+__device__ void bcNorthReflective(float Q[h+2*gc_y][w+2*gc_x], const int nx_, const int ny_) {
+    for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+        const int j = threadIdx.y + gc_y;
+        const int tj = blockDim.y*blockIdx.y + j;
+        
+        if (gc_y >= 1 && tj == ny_ + gc_y - 1) {
+            Q[j+1][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 2 && tj == ny_ + gc_y - 2) {
+            Q[j+3][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 3 && tj == ny_ + gc_y - 3) {
+            Q[j+5][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 4 && tj == ny_ + gc_y - 4) {
+            Q[j+7][i] = sign*Q[j][i];
+        }
+        if (gc_y >= 5 && tj == ny_ + gc_y - 5) {
+            Q[j+9][i] = sign*Q[j][i];
+        }
+    }
+}
+
+
+
+
+/**
+  * Alter the index l so that it gives periodic boundary conditions when reading
+  */
+template<int gc_x>
+inline __device__ int handlePeriodicBoundaryX(int k, int nx_, int boundary_conditions_) {
+    const int gc_pad = gc_x;
+    
+    //West boundary: add an offset to read from east of domain
+    if (gc_x > 0) {
+        if ((k < gc_pad) 
+                && getBCWest(boundary_conditions_) == Periodic) {
+            k += (nx_+2*gc_x - 2*gc_pad);
+        }
+        //East boundary: subtract an offset to read from west of domain
+        else if ((k >= nx_+2*gc_x-gc_pad) 
+                && getBCEast(boundary_conditions_) == Periodic) {
+            k -= (nx_+2*gc_x - 2*gc_pad);
+        }
+    }
+    
+    return k;
+}
+
+/**
+  * Alter the index l so that it gives periodic boundary conditions when reading
+  */
+template<int gc_y>
+inline __device__ int handlePeriodicBoundaryY(int l, int ny_, int boundary_conditions_) {
+    const int gc_pad = gc_y;
+    
+    //South boundary: add an offset to read from north of domain
+    if (gc_y > 0) {
+        if ((l < gc_pad) 
+                && getBCSouth(boundary_conditions_) == Periodic) {
+            l += (ny_+2*gc_y - 2*gc_pad);
+        }
+        //North boundary: subtract an offset to read from south of domain
+        else if ((l >= ny_+2*gc_y-gc_pad) 
+                && getBCNorth(boundary_conditions_) == Periodic) {
+            l -= (ny_+2*gc_y - 2*gc_pad);
+        }
+    }
+    
+    return l;
+}
+
+
+template<int w, int h, int gc_x, int gc_y, int sign_x, int sign_y>
+inline __device__ 
+void handleReflectiveBoundary(
+                float Q[h+2*gc_y][w+2*gc_x], 
+                const int nx_, const int ny_,
+                const int boundary_conditions_) {
+
+    //Handle reflective boundary conditions
+    if (getBCNorth(boundary_conditions_) == Reflective) {
+        bcNorthReflective<w, h, gc_x, gc_y, sign_y>(Q, nx_, ny_);
+        __syncthreads();
+    }
+    if (getBCSouth(boundary_conditions_) == Reflective) {
+        bcSouthReflective<w, h, gc_x, gc_y, sign_y>(Q, nx_, ny_);
+        __syncthreads();
+    }
+    if (getBCEast(boundary_conditions_) == Reflective) {
+        bcEastReflective<w, h, gc_x, gc_y, sign_x>(Q, nx_, ny_);
+        __syncthreads();
+    }
+    if (getBCWest(boundary_conditions_) == Reflective) {
+        bcWestReflective<w, h, gc_x, gc_y, sign_x>(Q, nx_, ny_);
+        __syncthreads();
+    }
+}
+
+/**
+  * Reads a block of data with ghost cells
+  */
+template<int w, int h, int gc_x, int gc_y, int sign_x, int sign_y>
+inline __device__ void readBlock(float* ptr_, int pitch_,
+                float Q[h+2*gc_y][w+2*gc_x], 
+                const int nx_, const int ny_,
+                const int boundary_conditions_,
+                 int x0, int y0,
+                 int x1, int y1) {
+    //Index of block within domain
+    const int bx = blockDim.x * blockIdx.x;
+    const int by = blockDim.y * blockIdx.y;
+
+    //Read into shared memory
+    //Loop over all variables
+    for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+        //Handle periodic boundary conditions here
+        int l = handlePeriodicBoundaryY<gc_y>(by + j + y0, ny_, boundary_conditions_);
+        l = min(l, min(ny_+2*gc_y-1, y1+2*gc_y-1));
+        float* row = (float*) ((char*) ptr_ + pitch_*l);
+        
+        for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+            //Handle periodic boundary conditions here
+            int k = handlePeriodicBoundaryX<gc_x>(bx + i + x0, nx_, boundary_conditions_);
+            k = min(k, min(nx_+2*gc_x-1, x1+2*gc_x-1));
+            
+            //Read from global memory
+            Q[j][i] = row[k];
+        }
+    }
+    __syncthreads();
+    
+    handleReflectiveBoundary<w, h, gc_x, gc_y, sign_x, sign_y>(Q, nx_, ny_, boundary_conditions_);
+}
+
+
+
+
+/**
+  * Writes a block of data to global memory for the shallow water equations.
+  */
+template<int w, int h, int gc_x, int gc_y>
+inline __device__ void writeBlock(float* ptr_, int pitch_,
+                 float shmem[h+2*gc_y][w+2*gc_x],
+                 const int nx_, const int ny_,
+                 int rk_step_, int rk_order_,
+                 int x0, int y0,
+                 int x1, int y1) {
+    
+    //Index of cell within domain
+    const int ti = blockDim.x*blockIdx.x + threadIdx.x + gc_x + x0;
+    const int tj = blockDim.y*blockIdx.y + threadIdx.y + gc_y + y0;
+
+    //In case we are writing only to a subarea given by (x0, y0) x (x1, y1)
+    const int max_ti = min(nx_+gc_x, x1+gc_x);
+    const int max_tj = min(ny_+gc_y, y1+gc_y);
+    
+    //Only write internal cells
+    if ((x0+gc_x <= ti) && (ti < max_ti) && (y0+gc_y <= tj) && (tj < max_tj)) {
+        //Index of thread within block
+        const int tx = threadIdx.x + gc_x;
+        const int ty = threadIdx.y + gc_y;
+        
+        float* const row  = (float*) ((char*) ptr_ + pitch_*tj);
+        
+        //Handle runge-kutta timestepping here
+        row[ti] = shmem[ty][tx];
+        
+        
+        
+        /**
+          * SSPRK1 (forward Euler)
+          * u^1   = u^n + dt*f(u^n)
+          */
+        if (rk_order_ == 1) {
+            row[ti] = shmem[ty][tx];
+        }
+        /**
+          * SSPRK2
+          * u^1   = u^n + dt*f(u^n)
+          * u^n+1 = 1/2*u^n + 1/2*(u^1 + dt*f(u^1))
+          */
+        else if (rk_order_ == 2) {
+            if (rk_step_ == 0) {
+                row[ti] = shmem[ty][tx];
+            }
+            else if (rk_step_ == 1) {
+                row[ti] = 0.5f*row[ti] + 0.5f*shmem[ty][tx];
+            }
+        }
+        /**
+          * SSPRK3
+          * u^1   = u^n + dt*f(u^n)
+          * u^2   = 3/4 * u^n + 1/4 * (u^1 + dt*f(u^1))
+          * u^n+1 = 1/3 * u^n + 2/3 * (u^2 + dt*f(u^2))
+          * FIXME: This is not correct now, need a temporary to hold intermediate step u^2
+          */
+        else if (rk_order_ == 3) {
+            if (rk_step_ == 0) {
+                row[ti] = shmem[ty][tx];
+            }
+            else if (rk_step_ == 1) {
+                row[ti] = 0.75f*row[ti] + 0.25f*shmem[ty][tx];
+            }
+            else if (rk_step_ == 2) {
+                const float t = 1.0f / 3.0f; //Not representable in base 2
+                row[ti] = t*row[ti] + (1.0f-t)*shmem[ty][tx];
+            }
+        }
+
+        // DEBUG
+        //row[ti] = 99.0;
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void evolveF(float Q[vars][h+2*gc_y][w+2*gc_x],
+              float F[vars][h+2*gc_y][w+2*gc_x],
+              const float dx_, const float dt_) {
+    for (int var=0; var < vars; ++var) {
+        for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+            for (int i=threadIdx.x+gc_x; i<w+gc_x; i+=w) {
+                Q[var][j][i] = Q[var][j][i] + (F[var][j][i-1] - F[var][j][i]) * dt_ / dx_;
+            }
+        }
+    }
+}
+
+
+
+
+
+
+/**
+  * Evolves the solution in time along the y axis (dimensional splitting)
+  */
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void evolveG(float Q[vars][h+2*gc_y][w+2*gc_x],
+              float G[vars][h+2*gc_y][w+2*gc_x],
+              const float dy_, const float dt_) {
+    for (int var=0; var < vars; ++var) {
+        for (int j=threadIdx.y+gc_y; j<h+gc_y; j+=h) {
+            for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+                Q[var][j][i] = Q[var][j][i] + (G[var][j-1][i] - G[var][j][i]) * dt_ / dy_;
+            }
+        }
+    }
+}
+
+
+
+
+
+/**
+  * Helper function for debugging etc.
+  */
+template<int shmem_width, int shmem_height, int vars>
+__device__ void memset(float Q[vars][shmem_height][shmem_width], float value) {
+    for (int k=0; k<vars; ++k) {
+        for (int j=threadIdx.y; j<shmem_height; ++j) {
+            for (int i=threadIdx.x; i<shmem_width; ++i) {
+                Q[k][j][i] = value;
+            }
+        }
+    }
+} 
+
+
+
+
+
+template <unsigned int threads>
+//__device__ void reduce_max(float* data, unsigned int n) {
+__device__ float reduce_max(float* data, unsigned int n) {	
+	__shared__ float sdata[threads];
+	unsigned int tid = threadIdx.x;
+
+	//Reduce to "threads" elements
+	sdata[tid] = FLT_MIN;
+	for (unsigned int i=tid; i<n; i += threads) {
+		
+		//sdata[tid] = max(sdata[tid], dt_ctx.L[i]);
+		sdata[tid] = max(sdata[tid], data[i]);
+
+    }
+	__syncthreads();
+
+	//Now, reduce all elements into a single element
+	if (threads >= 512) {
+		if (tid < 256) {
+            sdata[tid] = max(sdata[tid], sdata[tid + 256]);
+        }
+		__syncthreads();
+	}
+	if (threads >= 256) {
+		if (tid < 128) {
+            sdata[tid] = max(sdata[tid], sdata[tid + 128]);
+        }
+		__syncthreads();
+	}
+	if (threads >= 128) {
+		if (tid < 64) {
+            sdata[tid] = max(sdata[tid], sdata[tid + 64]);
+        }
+		__syncthreads();
+	}
+	if (tid < 32) {
+        volatile float* sdata_volatile = sdata;
+		if (threads >= 64) {
+            sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 32]);
+        }
+		if (tid < 16) {
+			if (threads >= 32) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 16]);
+			if (threads >= 16) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid +  8]);
+			if (threads >=  8) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid +  4]);
+			if (threads >=  4) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid +  2]);
+			if (threads >=  2) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid +  1]);
+		}
+
+		if (tid == 0) {
+            return sdata_volatile[0];
+		}
+	}
+}
+
+
+
+
+
+
+
+
+
+

GPUSimulators/cuda/limiters.h

@@ -0,0 +1,118 @@
+/*
+This file implements different flux and slope limiters
+
+Copyright (C) 2016, 2017, 2018 SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+
+
+
+
+
+/**
+  * Reconstructs a slope using the generalized minmod limiter based on three 
+  * consecutive values
+  */
+__device__ __inline__ float minmodSlope(float left, float center, float right, float theta) {
+    const float backward = (center - left) * theta;
+    const float central = (right - left) * 0.5f;
+    const float forward = (right - center) * theta;
+    
+	return 0.25f
+		*copysign(1.0f, backward)
+		*(copysign(1.0f, backward) + copysign(1.0f, central))
+		*(copysign(1.0f, central) + copysign(1.0f, forward))
+		*min( min(fabs(backward), fabs(central)), fabs(forward) );
+}
+
+
+
+
+/**
+  * Reconstructs a minmod slope for a whole block along the abscissa
+  */
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void minmodSlopeX(float Q[vars][h+2*gc_y][w+2*gc_x],
+                  float Qx[vars][h+2*gc_y][w+2*gc_x],
+                  const float theta_) {
+    //Reconstruct slopes along x axis
+    for (int p=0; p<vars; ++p) {
+        for (int j=threadIdx.y; j<h+2*gc_y; j+=h) {
+            for (int i=threadIdx.x+1; i<w+2*gc_x-1; i+=w) {
+                Qx[p][j][i] = minmodSlope(Q[p][j][i-1], Q[p][j][i], Q[p][j][i+1], theta_);
+            }
+        }
+    }
+}
+
+
+/**
+  * Reconstructs a minmod slope for a whole block along the ordinate
+  */
+template<int w, int h, int gc_x, int gc_y, int vars>
+__device__ void minmodSlopeY(float Q[vars][h+2*gc_y][w+2*gc_x],
+                  float Qy[vars][h+2*gc_y][w+2*gc_x],
+                  const float theta_) {
+    //Reconstruct slopes along y axis
+    for (int p=0; p<vars; ++p) {
+        for (int j=threadIdx.y+1; j<h+2*gc_y-1; j+=h) {
+            for (int i=threadIdx.x; i<w+2*gc_x; i+=w) {
+                Qy[p][j][i] = minmodSlope(Q[p][j-1][i], Q[p][j][i], Q[p][j+1][i], theta_);
+            }
+        }
+    }
+}
+
+
+
+
+__device__ float monotonized_central(float r_) {
+    return fmaxf(0.0f, fminf(2.0f, fminf(2.0f*r_, 0.5f*(1.0f+r_))));
+}
+
+__device__ float osher(float r_, float beta_) {
+    return fmaxf(0.0f, fminf(beta_, r_));
+}
+
+__device__ float sweby(float r_, float beta_) {
+    return fmaxf(0.0f, fmaxf(fminf(r_, beta_), fminf(beta_*r_, 1.0f)));
+}
+
+__device__ float minmod(float r_) {
+    return fmaxf(0.0f, fminf(1.0f, r_));
+}
+
+__device__ float generalized_minmod(float r_, float theta_) {
+    return fmaxf(0.0f, fminf(theta_*r_, fminf( (1.0f + r_) / 2.0f, theta_)));
+}
+
+__device__ float superbee(float r_) {
+    return fmaxf(0.0f, fmaxf(fminf(2.0f*r_, 1.0f), fminf(r_, 2.0f)));
+}
+
+__device__ float vanAlbada1(float r_) {
+    return (r_*r_ + r_) / (r_*r_ + 1.0f);
+}
+
+__device__ float vanAlbada2(float r_) {
+    return 2.0f*r_ / (r_*r_* + 1.0f);
+}
+
+__device__ float vanLeer(float r_) {
+    return (r_ + fabsf(r_)) / (1.0f + fabsf(r_));
+}

GPUSimulators/helpers/InitialConditions.py

@@ -0,0 +1,355 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements Cuda context handling
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+from GPUSimulators.Simulator import BoundaryCondition
+import numpy as np
+import gc
+
+
+def getExtent(width, height, nx, ny, grid, index=None):
+    if grid is not None:
+        gx = grid.grid[0]
+        gy = grid.grid[1]
+        if index is not None:
+            i, j = grid.getCoordinate(index)
+        else:
+            i, j = grid.getCoordinate()
+        
+        dx = (width / gx) / nx
+        dy = (height / gy) / ny
+        
+        x0 = width*i/gx + 0.5*dx
+        y0 = height*j/gy + 0.5*dy
+        x1 = width*(i+1)/gx - 0.5*dx
+        y1 = height*(j+1)/gy - 0.5*dx
+        
+    else:
+        dx = width / nx
+        dy = height / ny
+        
+        x0 = 0.5*dx
+        y0 = 0.5*dy
+        x1 = width-0.5*dx
+        y1 = height-0.5*dy
+        
+    return [x0, x1, y0, y1, dx, dy]
+
+        
+def downsample(highres_solution, x_factor, y_factor=None):
+    if (y_factor == None):
+        y_factor = x_factor
+
+    assert(highres_solution.shape[1] % x_factor == 0)
+    assert(highres_solution.shape[0] % y_factor == 0)
+    
+    if (x_factor*y_factor == 1):
+        return highres_solution
+    
+    if (len(highres_solution.shape) == 1):
+        highres_solution = highres_solution.reshape((1, highres_solution.size))
+
+    nx = highres_solution.shape[1] / x_factor
+    ny = highres_solution.shape[0] / y_factor
+
+    return highres_solution.reshape([int(ny), int(y_factor), int(nx), int(x_factor)]).mean(3).mean(1)
+
+
+
+
+    
+def bump(nx, ny, width, height, 
+        bump_size=None, 
+        ref_nx=None, ref_ny=None,
+        x_center=0.5, y_center=0.5,
+        h_ref=0.5, h_amp=0.1, u_ref=0.0, u_amp=0.1, v_ref=0.0, v_amp=0.1):
+    
+    if (ref_nx == None):
+        ref_nx = nx
+    assert(ref_nx >= nx)
+      
+    if (ref_ny == None):
+        ref_ny = ny
+    assert(ref_ny >= ny)
+        
+    if (bump_size == None):
+        bump_size = width/5.0
+    
+    ref_dx = width / float(ref_nx)
+    ref_dy = height / float(ref_ny)
+
+    x_center = ref_dx*ref_nx*x_center
+    y_center = ref_dy*ref_ny*y_center
+    
+    x = ref_dx*(np.arange(0, ref_nx, dtype=np.float32)+0.5) - x_center
+    y = ref_dy*(np.arange(0, ref_ny, dtype=np.float32)+0.5) - y_center
+    xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')
+    r = np.sqrt(xv**2 + yv**2)
+    xv = None
+    yv = None
+    gc.collect()
+    
+    #Generate highres then downsample
+    #h_highres = 0.5 + 0.1*np.exp(-(xv**2/size + yv**2/size))
+    h_highres = h_ref + h_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size)
+    h = downsample(h_highres, ref_nx/nx, ref_ny/ny)
+    h_highres = None
+    gc.collect()
+    
+    #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size))
+    u_highres = u_ref + u_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size)
+    hu = downsample(u_highres, ref_nx/nx, ref_ny/ny)*h
+    u_highres = None
+    gc.collect()
+    
+    #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size))
+    v_highres = v_ref + v_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size)
+    hv = downsample(v_highres, ref_nx/nx, ref_ny/ny)*h
+    v_highres = None
+    gc.collect()
+    
+    dx = width/nx
+    dy = height/ny
+    
+    return h, hu, hv, dx, dy
+
+
+def genShockBubble(nx, ny, gamma, grid=None):
+    """
+    Generate Shock-bubble interaction case for the Euler equations
+    """
+    
+    width = 4.0
+    height = 1.0
+    g = 0.0
+
+
+    rho = np.ones((ny, nx), dtype=np.float32)
+    u = np.zeros((ny, nx), dtype=np.float32)
+    v = np.zeros((ny, nx), dtype=np.float32)
+    E = np.zeros((ny, nx), dtype=np.float32)
+    p = np.ones((ny, nx), dtype=np.float32)
+
+    
+    x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid)
+    x = np.linspace(x0, x1, nx, dtype=np.float32)
+    y = np.linspace(y0, y1, ny, dtype=np.float32)
+    xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')
+       
+    #Bubble
+    radius = 0.25
+    x_center = 0.5
+    y_center = 0.5
+    bubble = np.sqrt((xv-x_center)**2+(yv-y_center)**2) <= radius
+    rho = np.where(bubble, 0.1, rho)
+    
+    #Left boundary
+    left = (xv < 0.1)
+    rho = np.where(left, 3.81250, rho)
+    u = np.where(left, 2.57669, u)
+    
+    #Energy
+    p = np.where(left, 10.0, p)
+    E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0)
+    
+
+    bc = BoundaryCondition({
+        'north': BoundaryCondition.Type.Reflective,
+        'south': BoundaryCondition.Type.Reflective,
+        'east': BoundaryCondition.Type.Periodic,
+        'west': BoundaryCondition.Type.Periodic
+    })
+    
+    #Construct simulator
+    arguments = {
+        'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E,
+        'nx': nx, 'ny': ny,
+        'dx': dx, 'dy': dy, 
+        'g': g,
+        'gamma': gamma,
+        'boundary_conditions': bc
+    } 
+    return arguments
+
+    
+    
+    
+    
+    
+    
+def genKelvinHelmholtz(nx, ny, gamma, roughness=0.125, grid=None, index=None):
+    """
+    Roughness parameter in (0, 1.0] determines how "squiggly" 
+    the interface betweeen the zones is
+    """
+    
+    def genZones(nx, ny, n):
+        """
+        Generates the zones of the two fluids of K-H
+        """
+        zone = np.zeros((ny, nx), dtype=np.int32)
+
+
+        def genSmoothRandom(nx, n):
+            n = max(1, min(n, nx))
+            
+            if n == nx:
+                return np.random.random(nx)-0.5
+            else:
+                from scipy.interpolate import interp1d
+
+                #Control points and interpolator
+                xp = np.linspace(0.0, 1.0, n)
+                yp = np.random.random(n) - 0.5
+                
+                if (n == 1):
+                    kind = 'nearest'
+                elif (n == 2):
+                    kind = 'linear'
+                elif (n == 3):
+                    kind = 'quadratic'
+                else:
+                    kind = 'cubic'
+                    
+                f = interp1d(xp, yp, kind=kind)
+
+                #Interpolation points
+                x = np.linspace(0.0, 1.0, nx)
+                return f(x)
+
+
+
+        x0, x1, y0, y1, _, dy = getExtent(1.0, 1.0, nx, ny, grid, index)
+        x = np.linspace(x0, x1, nx)
+        y = np.linspace(y0, y1, ny)
+        _, y = np.meshgrid(x, y)
+
+        #print(y+a[0])
+
+        a = genSmoothRandom(nx, n)*dy
+        zone = np.where(y > 0.25+a, zone, 1)
+
+        a = genSmoothRandom(nx, n)*dy
+        zone = np.where(y < 0.75+a, zone, 1)
+        
+        return zone
+        
+    width = 2.0
+    height = 1.0
+    g = 0.0
+    gamma = 1.4
+
+    rho = np.empty((ny, nx), dtype=np.float32)
+    u = np.empty((ny, nx), dtype=np.float32)
+    v = np.zeros((ny, nx), dtype=np.float32)
+    p = 2.5*np.ones((ny, nx), dtype=np.float32)
+
+    #Generate the different zones    
+    zones = genZones(nx, ny, max(1, min(nx, int(nx*roughness))))
+    
+    #Zone 0
+    zone0 = zones == 0
+    rho = np.where(zone0, 1.0, rho)
+    u = np.where(zone0, 0.5, u)
+    
+    #Zone 1
+    zone1 = zones == 1
+    rho = np.where(zone1, 2.0, rho)
+    u = np.where(zone1, -0.5, u)
+    
+    E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0)
+    
+    _, _, _, _, dx, dy = getExtent(width, height, nx, ny, grid, index)
+    
+    
+    bc = BoundaryCondition({
+        'north': BoundaryCondition.Type.Periodic,
+        'south': BoundaryCondition.Type.Periodic,
+        'east': BoundaryCondition.Type.Periodic,
+        'west': BoundaryCondition.Type.Periodic
+    })
+    
+    #Construct simulator
+    arguments = {
+        'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E,
+        'nx': nx, 'ny': ny,
+        'dx': dx, 'dy': dy, 
+        'g': g,
+        'gamma': gamma,
+        'boundary_conditions': bc
+    } 
+    
+    return arguments
+    
+    
+    
+def genRayleighTaylor(nx, ny, gamma, version=0, grid=None):
+    """
+    Generates Rayleigh-Taylor instability case
+    """
+    width = 0.5
+    height = 1.5
+    g = 0.1
+
+    rho = np.zeros((ny, nx), dtype=np.float32)
+    u = np.zeros((ny, nx), dtype=np.float32)
+    v = np.zeros((ny, nx), dtype=np.float32)
+    p = np.zeros((ny, nx), dtype=np.float32)
+    
+    
+    x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid)
+    x = np.linspace(x0, x1, nx, dtype=np.float32)-width*0.5
+    y = np.linspace(y0, y1, ny, dtype=np.float32)-height*0.5
+    xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy')
+    
+    #This gives a squigly interfact
+    if (version == 0):
+        y_threshold = 0.01*np.cos(2*np.pi*np.abs(x)/0.5)
+        rho = np.where(yv <= y_threshold, 1.0, rho)
+        rho = np.where(yv > y_threshold, 2.0, rho)
+    elif (version == 1):
+        rho = np.where(yv <= 0.0, 1.0, rho)
+        rho = np.where(yv > 0.0, 2.0, rho)
+        v = 0.01*(1.0 + np.cos(2*np.pi*xv/0.5))/4
+    else:
+        assert False, "Invalid version"
+    
+    p = 2.5 - rho*g*yv
+    E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0)
+    
+    bc = BoundaryCondition({
+        'north': BoundaryCondition.Type.Reflective,
+        'south': BoundaryCondition.Type.Reflective,
+        'east': BoundaryCondition.Type.Reflective,
+        'west': BoundaryCondition.Type.Reflective
+    })
+    
+    #Construct simulator
+    arguments = {
+        'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E,
+        'nx': nx, 'ny': ny,
+        'dx': dx, 'dy': dy, 
+        'g': g,
+        'gamma': gamma,
+        'boundary_conditions': bc
+    } 
+
+    return arguments

GPUSimulators/helpers/Visualization.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements visualization techniques/modes
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+
+import numpy as np
+
+from matplotlib.colors import Normalize
+
+
+
+def genSchlieren(rho):
+    #Compute length of z-component of normalized gradient vector 
+    normal = np.gradient(rho) #[x, y, 1]
+    length = 1.0 / np.sqrt(normal[0]**2 + normal[1]**2 + 1.0)
+    schlieren = np.power(length, 128)
+    return schlieren
+
+
+def genVorticity(rho, rho_u, rho_v):
+    u = rho_u / rho
+    v = rho_v / rho
+    u = np.sqrt(u**2 + v**2)
+    u_max = u.max()
+    
+    du_dy, _ = np.gradient(u)
+    _, dv_dx = np.gradient(v)
+    
+    #Length of curl
+    curl = dv_dx - du_dy
+    return curl
+
+
+def genColors(rho, rho_u, rho_v, cmap, vmax, vmin):
+    schlieren = genSchlieren(rho)
+    curl = genVorticity(rho, rho_u, rho_v)
+
+    colors = Normalize(vmin, vmax, clip=True)(curl)
+    colors = cmap(colors)
+    for k in range(3):
+        colors[:,:,k] = colors[:,:,k]*schlieren
+
+    return colors

Jobs/job_lumi.slrum

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+#SBATCH --job-name=lumi
+#SBATCH --account=project_4650000xx
+#SBATCH --time=00:10:00
+#SBATCH --partition=dev-g
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --gpus=8
+#SBATCH --gpus-per-node=8
+#SBATCH -o %x-%j.out
+#SBATCH --exclusive
+#
+
+N=$SLURM_JOB_NUM_NODES
+echo "--nbr of nodes:", $N
+echo "--total nbr of gpus:", $SLURM_NTASKS
+
+Mydir=/project/project_4650000xx
+Myapplication=${Mydir}/FiniteVolumeGPU_hip/mpiTesting.py
+
+#modules
+ml LUMI/23.03 partition/G
+ml lumi-container-wrapper
+ml cray-python/3.9.13.1
+ml rocm/5.2.3
+
+ml craype-accel-amd-gfx90a
+ml cray-mpich/8.1.27
+
+export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH"
+
+#missing library
+export LD_LIBRARY_PATH=/opt/cray/pe/mpich/8.1.27/ofi/cray/14.0/lib-abi-mpich:$LD_LIBRARY_PATH
+
+#Binding mask
+bind_mask="0x${fe}000000000000,0x${fe}00000000000000,0x${fe}0000,0x${fe}000000,0x${fe},0x${fe}00,0x${fe}00000000,0x${fe}0000000000"
+
+srun --cpu-bind=mask_cpu:$bind_mask \
+     python ${Myapplication} -nx 1024 -ny 1024 --profile 	

README.md

@@ -0,0 +1,46 @@
+# FiniteVolumeGPU
+
+This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. 
+
+## Setup on LUMI-G
+Here is a step-by-step guide on installing packages on LUMI-G
+
+### Step 0: load modules
+```
+ml LUMI/23.03 partition/G
+ml lumi-container-wrapper
+ml cray-python/3.9.13.1
+```
+
+### Step 1: run conda-container
+Installation via conda can be done as:
+```
+conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml
+```
+where the file `conda_environment_lumi.yml` contains packages to be installed.
+
+### Step 2: Set the env. variable to search for binaries
+```
+export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH"
+```
+### An alternative: Convert to a singularity container with cotainr
+```
+cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml
+```
+
+### Error when running MPI.
+```
+`MPI startup(): PMI server not found. Please set I_MPI_PMI_LIBRARY variable if it is not a singleton case.
+```
+This can be resolved by exporting this:
+```
+export I_MPI_PMI_LIBRARY=/opt/cray/pe/mpich/8.1.27/ofi/cray/14.0/lib/libmpi.so
+```
+### Install hip-python
+```
+python -m pip install -i https://test.pypi.org/simple/ hip-python==5.4.3.470.16
+```
+
+The testing was done with this specific version `hip-python==5.4.3.470.16`
+
+ 

SYSTEMS.md

@@ -0,0 +1,62 @@
+$ANSIBLE_VAULT;1.1;AES256
+61316265663939333638336466323036663861343233316466646432313138653633623662353937
+3232313165656633346432376237383566363537366534310a303231343936663438653835373161
+35616161323432653062323164623861353065333761663136313137333732313230626665386336
+6166656538396463370a356166316363326133313864386536323236346634323537393639653038
+66336337336132613061353964613638326233356336323962366531333932366539366339623563
+36333365326463616634323939333062363263636663373635653064626138363464666233316561
+63393735393233616437386537393739393433663631313864646535636262616336333631396166
+38643636323530386565396338623366393232313838356536303537393338393634666632656234
+65353930303762333639376638336364303439306132626531326132376264623063376464636430
+32333536386134333136313139313861306364333037323363393463333664633764653937623866
+34313064346261313330373132353563343761323435393930303136353865303163373937623831
+64343038373162333039653161643233353764633337366434396638376530636261323362373434
+38393630613065356632663533333331633039663935663732353234643131306665343339373265
+64356563653838613337663132663234356462343333623139626662363764656239326637653832
+35396636643937336431623531306133643137623831333936313839333738333730373136666336
+35623965643664343164373630313362656663386638376237616134343631386366313336626138
+62376436383837376539663438346431633138383363633862356366376537393932626262383637
+31323365333139653736623233636233323162343039663035346135326638633430303134396337
+36353264313835346130643736663665386364343835643166383361316631373338663731373335
+30313530326662663937666330643565363565616566633333363535656539656531666266613638
+30306264613438363265646332386535383238373433396337633636616532626161343236336533
+36366362653137333739353737386563613136653164383437316237643533633133313735633363
+64326433356266363133343339626333633063326533383632353639613163663966376465396231
+36663034363534396430316463386564663465323036613636343136643262666566303533346439
+63396466656639623836613130363835346435633437666463363333356231343038356434343861
+66643636633739336666316566653136363862346336353862653130346335363334616430366435
+30376365383262326438306266366265363030353764633630333034663037643037343132303631
+39316364366234363339363130333765616432306331373566393530653963356539636437383062
+34633938643563656363633864656361643539663833356638356365373061663964363530393535
+37646533386235613763396638393539303062326239633238373763326561313634313265613135
+64646138313562313732393732303133343234323438616165326530333234626363393735636530
+62353735313231353662353533636134306530623339383730306332613636663366653566313935
+32343935353566656130393533323639353863666436333839386463396337336635356663373136
+61323734613239396236393266363631313465363630306565636663396235626132336339623938
+62383435643661623938393662363262376566613365613465323432343534356433323330666133
+30303963656635303734316539333038663962626331313366666337663165323230646564623935
+61316630353739386365323339626166323562616630383538393733353864396565353039656333
+30343038636231363531383061613836653038373937616163643963393231356235626531366239
+62343333326434636665363931376235313535343135626261336439636663323233383565633964
+65333830613131396630336337646230393038386536336365313738316335386261393838383961
+64656331363738616539346663613261386639353437316231636533353031336464383432623939
+65386164396231393735643563663337643563633233373338643630313739373861356166616463
+35306263333963663434376263396464323135346663376334356134393066653439376263376231
+30333730383163366636323533393334336331633234306536376634313735613263366537346536
+62366564383861656662353738366665396639313833323038356661306135393338333466333563
+32653861346166663163383036386432343833333137663462343030363762663139366534326466
+66313864623438336164333430613766373430656536323964633863333931643036656563353639
+30313835666366383035343031643265386263316165323537613636656533376239633964393866
+61646163343032313036303738643763383364663134356634373262633361383035306231636364
+39333232636538643033313438396332383962656131363365666566633239366532326336363133
+38393064643030333538333562643435663434343863383834663266373337336433313663646164
+36343334343965623830613736393231666361643239663062393239613233376335383362666161
+66383035653330373736613234303631386163656561383138613363613539396332376162316131
+61313532653531653836343731636535623066383231613635316432323331623761383833623333
+39343632623961613561373261653939636363366531303839336237383166363733303538363237
+36373362636263666334316163633766303334373033636539353464393536356466636664333665
+32643135626366666137626464393961366165383334343063356334373534633764326162363837
+38643662326266313464343464646166643235663663303761313639376537306337353863336264
+66376335333738366265343636376363366365306137336665623466626261653937656461303332
+32613561616662383032393562613831626666373134303032626134313262363830326530643632
+61366133663564313933366430396430353762386133396436633839303766653765

conda_environment_lumi.yml

@@ -0,0 +1,35 @@
+# Assumes that conda, pip, build-essentials and cuda are installed
+---
+name: ShallowWaterGPU_HPC
+channels:
+- conda-forge
+
+dependencies:
+- python=3.9.13
+- numpy
+- mpi4py
+- six
+- pytools
+- netcdf4
+- scipy
+- pip:
+  - hip-python==5.4.3.470.16
+  - -i https://test.pypi.org/simple/ 
+
+
+#On LUMI-G
+#module load LUMI/23.03
+#module load lumi-container-wrapper
+#ml cray-python/3.9.13.1
+#conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml
+# export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH"
+#
+#
+#
+# Install conda environment (one-time operation):
+# $ conda env create -f conda_environment_hpc.yml
+# Activate environment and install the following packages using pip:
+# $ conda activate ShallowWaterGPU_HPC
+#  - pycuda: $ pip3 install --no-deps -U pycuda
+# on Windows: make sure your visual studio c++ compiler is available in PATH
+# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\

mpiTesting.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements MPI simulations for benchmarking
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+import numpy as np
+import gc
+import time
+import json
+import logging
+import os
+
+#GPU-aware MPI
+"""
+from os import environ
+if environ.get("MPICH_GPU_SUPPORT_ENABLED", False):
+    from ctypes import CDLL, RTLD_GLOBAL
+    CDLL(f"{environ.get('CRAY_MPICH_ROOTDIR')}/gtl/lib/libmpi_gtl_hsa.so", mode=RTLD_GLOBAL)
+"""
+
+# MPI
+from mpi4py import MPI
+
+# CUDA
+#import pycuda.driver as cuda
+from hip import hip,hiprtc
+
+# Simulator engine etc
+from GPUSimulators import MPISimulator, Common, CudaContext
+from GPUSimulators import EE2D_KP07_dimsplit
+from GPUSimulators.helpers import InitialConditions as IC
+from GPUSimulators.Simulator import BoundaryCondition as BC
+
+import argparse
+parser = argparse.ArgumentParser(description='Strong and weak scaling experiments.')
+parser.add_argument('-nx', type=int, default=128)
+parser.add_argument('-ny', type=int, default=128)
+parser.add_argument('--profile', action='store_true') # default: False
+
+def hip_check(call_result):
+    err = call_result[0]
+    result = call_result[1:]
+    if len(result) == 1:
+        result = result[0]
+    if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess:
+        raise RuntimeError(str(err))
+    elif (
+        isinstance(err, hiprtc.hiprtcResult)
+        and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS
+    ):
+        raise RuntimeError(str(err))
+    return result
+
+args = parser.parse_args()
+
+if(args.profile):
+    profiling_data = {}
+    # profiling: total run time
+    t_total_start = time.time()
+    t_init_start = time.time()
+
+
+# Get MPI COMM to use
+comm = MPI.COMM_WORLD
+
+
+####
+# Initialize logging
+####
+log_level_console = 20
+log_level_file = 10
+log_filename = 'mpi_' + str(comm.rank) + '.log'
+logger = logging.getLogger('GPUSimulators')
+logger.setLevel(min(log_level_console, log_level_file))
+
+ch = logging.StreamHandler()
+ch.setLevel(log_level_console)
+logger.addHandler(ch)
+logger.info("Console logger using level %s",
+            logging.getLevelName(log_level_console))
+
+fh = logging.FileHandler(log_filename)
+formatter = logging.Formatter(
+    '%(asctime)s:%(name)s:%(levelname)s: %(message)s')
+fh.setFormatter(formatter)
+fh.setLevel(log_level_file)
+logger.addHandler(fh)
+logger.info("File logger using level %s to %s",
+            logging.getLevelName(log_level_file), log_filename)
+
+
+####
+# Initialize MPI grid etc
+####
+logger.info("Creating MPI grid")
+grid = MPISimulator.MPIGrid(MPI.COMM_WORLD)
+
+"""
+job_id = int(os.environ["SLURM_JOB_ID"])
+allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"])
+allocated_gpus = int(os.environ["ROCR_VISIBLE_DEVICES"].count(",") + 1)
+
+print("job_id:", job_id)
+print("allocated_nodes", allocated_nodes)
+print("allocated_gpus", allocated_gpus)
+"""
+
+####
+# Initialize CUDA
+####
+#cuda.init(flags=0)
+#logger.info("Initializing CUDA")
+local_rank = grid.getLocalRank()
+#num_cuda_devices = cuda.Device.count()
+num_cuda_devices = hip_check(hip.hipGetDeviceCount())
+cuda_device = local_rank % num_cuda_devices
+logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device))
+cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False)
+
+####
+# Set initial conditions
+####
+
+# DEBUGGING - setting random seed
+np.random.seed(42)
+
+logger.info("Generating initial conditions")
+nx = args.nx
+ny = args.ny
+
+dt = 0.000001
+
+gamma = 1.4
+#save_times = np.linspace(0, 0.000009, 2)
+#save_times = np.linspace(0, 0.000099, 11)
+#save_times = np.linspace(0, 0.000099, 2)
+save_times = np.linspace(0, 0.0000999, 2)
+outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc"
+save_var_names = ['rho', 'rho_u', 'rho_v', 'E']
+
+arguments = IC.genKelvinHelmholtz(nx, ny, gamma, grid=grid)
+arguments['context'] = cuda_context
+arguments['theta'] = 1.2
+arguments['grid'] = grid
+
+if(args.profile):
+    t_init_end = time.time()
+    t_init = t_init_end - t_init_start
+    profiling_data["t_init"] = t_init
+
+####
+# Run simulation
+####
+logger.info("Running simulation")
+# Helper function to create MPI simulator
+
+
+def genSim(grid, **kwargs):
+    local_sim = EE2D_KP07_dimsplit.EE2D_KP07_dimsplit(**kwargs)
+    sim = MPISimulator.MPISimulator(local_sim, grid)
+    return sim
+
+
+outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation(
+    genSim, arguments, outfile, save_times, save_var_names, dt)
+
+if(args.profile):
+    t_total_end = time.time()
+    t_total = t_total_end - t_total_start
+    profiling_data["t_total"] = t_total
+    print("Total run time on rank " + str(MPI.COMM_WORLD.rank) + " is " + str(t_total) + " s")
+
+# write profiling to json file
+if(args.profile and MPI.COMM_WORLD.rank == 0):
+    job_id = ""
+    if "SLURM_JOB_ID" in os.environ:
+        job_id = int(os.environ["SLURM_JOB_ID"])
+        allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"])
+        allocated_gpus = int(os.environ["ROCR_VISIBLE_DEVICES"].count(",") + 1)
+     #   allocated_gpus = int(os.environ["CUDA_VISIBLE_DEVICES"].count(",") + 1)
+        profiling_file = "MPI_jobid_" + \
+            str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json"
+        profiling_data["outfile"] = outfile
+    else:
+        profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json"
+
+    for stage in sim_runner_profiling_data["start"].keys():
+        profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage]
+
+    for stage in sim_profiling_data["start"].keys():
+        profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage]
+
+    profiling_data["nx"] = nx
+    profiling_data["ny"] = ny
+    profiling_data["dt"] = dt
+    profiling_data["n_time_steps"] = sim_profiling_data["n_time_steps"]
+
+    profiling_data["slurm_job_id"] = job_id
+    profiling_data["n_cuda_devices"] = str(num_cuda_devices)
+    profiling_data["n_processes"] = str(MPI.COMM_WORLD.size)
+    profiling_data["git_hash"] = Common.getGitHash()
+    profiling_data["git_status"] = Common.getGitStatus()
+
+    with open(profiling_file, "w") as write_file:
+        json.dump(profiling_data, write_file)
+
+####
+# Clean shutdown
+####
+sim = None
+local_sim = None
+cuda_context = None
+arguments = None
+logging.shutdown()
+gc.collect()
+
+
+
+####
+# Print completion and exit
+####
+print("Completed!")
+exit(0)

reference/swashes_1_nx=128.csv

@@ -0,0 +1,146 @@
+##############################################################################
+# Generated by SWASHES version 1.03.00, 2016-01-29
+##############################################################################
+# Dimension: 1
+# Type: 3 (=Dam break)
+# Domain: 1
+# Choice: 1 (=on a wet domain without friction (Stoker's solution))
+##############################################################################
+# PARAMETERS OF THE SOLUTION
+# 
+# Length of the domain: 10 meters
+# Space step: 0.078125 meters
+# Number of cells: 128
+# Position of the dam: x=5 meters
+# Time value: 6 seconds
+##############################################################################
+# 
+#(i-0.5)*dx 	    h[i] 	    u[i] 	 topo[i] 	    q[i] 	 topo[i]+h[i] 	Fr[i]=Froude	 topo[i]+hc[i] 	
+0.0390625	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.117188	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.195312	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.273438	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.351562	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.429688	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.507812	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.585938	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.664062	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.742188	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.820312	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.898438	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.976562	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.05469	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.13281	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.21094	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.28906	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.36719	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.44531	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.52344	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.60156	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.67969	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.75781	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.83594	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.91406	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.99219	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.07031	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.14844	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.22656	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.30469	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.38281	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.46094	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.53906	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.61719	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.69531	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.77344	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.85156	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.92969	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.00781	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.08594	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.16406	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.24219	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.32031	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.39844	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.47656	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.55469	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.63281	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.71094	0.00490073	0.00441906	        0	2.16566e-005	0.00490073	0.0201542	0.000362943	
+3.78906	0.00470863	0.0130996	        0	6.16813e-005	0.00470863	0.0609504	0.000729255	
+3.86719	0.00452038	0.0217802	        0	9.84546e-005	0.00452038	 0.103428	0.000996019	
+3.94531	0.00433596	0.0304607	        0	0.000132076	0.00433596	 0.147694	0.00121151	
+4.02344	0.00415538	0.0391413	        0	0.000162647	0.00415538	 0.193863	0.0013919	
+4.10156	0.00397865	0.0478218	        0	0.000190266	0.00397865	 0.242061	0.00154532	
+4.17969	0.00380575	0.0565024	        0	0.000215034	0.00380575	 0.292423	0.00167667	
+4.25781	0.0036367	 0.065183	        0	0.000237051	0.0036367	 0.345101	0.00178925	
+4.33594	0.00347148	0.0738635	        0	0.000256416	0.00347148	 0.400256	0.00188541	
+4.41406	0.00331011	0.0825441	        0	0.00027323	0.00331011	 0.458068	0.00196696	
+4.49219	0.00315257	0.0912246	        0	0.000287592	0.00315257	 0.518734	0.0020353	
+4.57031	0.00299888	0.0999052	        0	0.000299604	0.00299888	  0.58247	0.00209158	
+4.64844	0.00284903	 0.108586	        0	0.000309364	0.00284903	 0.649516	0.00213677	
+4.72656	0.00270302	 0.117266	        0	0.000316973	0.00270302	 0.720135	0.00217166	
+4.80469	0.00256085	 0.125947	        0	0.000322531	0.00256085	 0.794623	0.00219697	
+4.88281	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.96094	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.03906	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.11719	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.19531	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.27344	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.35156	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.42969	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.50781	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.58594	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.66406	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.74219	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.82031	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.89844	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.97656	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.05469	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.13281	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.21094	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.28906	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.36719	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.44531	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.52344	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.60156	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.67969	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.75781	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.83594	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.91406	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.99219	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.07031	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.14844	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.22656	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.30469	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.38281	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.46094	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.53906	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.61719	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.69531	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.77344	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.85156	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.92969	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.00781	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.08594	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.16406	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.24219	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.32031	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.39844	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.47656	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.55469	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.63281	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.71094	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.78906	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.86719	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.94531	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.02344	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.10156	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.17969	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.25781	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.33594	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.41406	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.49219	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.57031	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.64844	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.72656	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.80469	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.88281	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.96094	    0.001	        0	        0	        0	    0.001	        0	        0	

reference/swashes_1_nx=256.csv

@@ -0,0 +1,274 @@
+##############################################################################
+# Generated by SWASHES version 1.03.00, 2016-01-29
+##############################################################################
+# Dimension: 1
+# Type: 3 (=Dam break)
+# Domain: 1
+# Choice: 1 (=on a wet domain without friction (Stoker's solution))
+##############################################################################
+# PARAMETERS OF THE SOLUTION
+# 
+# Length of the domain: 10 meters
+# Space step: 0.0390625 meters
+# Number of cells: 256
+# Position of the dam: x=5 meters
+# Time value: 6 seconds
+##############################################################################
+# 
+#(i-0.5)*dx 	    h[i] 	    u[i] 	 topo[i] 	    q[i] 	 topo[i]+h[i] 	Fr[i]=Froude	 topo[i]+hc[i] 	
+0.0195312	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0585938	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0976562	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.136719	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.175781	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.214844	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.253906	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.292969	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.332031	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.371094	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.410156	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.449219	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.488281	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.527344	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.566406	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.605469	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.644531	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.683594	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.722656	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.761719	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.800781	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.839844	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.878906	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.917969	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.957031	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.996094	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.03516	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.07422	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.11328	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.15234	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.19141	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.23047	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.26953	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.30859	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.34766	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.38672	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.42578	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.46484	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.50391	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.54297	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.58203	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.62109	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.66016	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.69922	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.73828	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.77734	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.81641	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.85547	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.89453	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.93359	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.97266	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.01172	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.05078	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.08984	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.12891	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.16797	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.20703	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.24609	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.28516	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.32422	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.36328	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.40234	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.44141	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.48047	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.51953	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.55859	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.59766	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.63672	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.67578	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.71484	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.75391	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.79297	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.83203	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.87109	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.91016	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.94922	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.98828	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.02734	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.06641	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.10547	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.14453	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.18359	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.22266	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.26172	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.30078	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.33984	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.37891	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.41797	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.45703	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.49609	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.53516	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.57422	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.61328	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.65234	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.69141	0.00494936	0.00224893	        0	1.11307e-005	0.00494936	0.0102062	0.000232877	
+3.73047	0.00485235	0.0065892	        0	3.19731e-005	0.00485235	0.0302011	0.00047058	
+3.76953	0.0047563	0.0109295	        0	5.19839e-005	0.0047563	0.0505977	0.000650663	
+3.80859	0.00466121	0.0152698	        0	7.11755e-005	0.00466121	0.0714082	0.000802289	
+3.84766	0.00456708	  0.01961	        0	8.95606e-005	0.00456708	0.0926456	0.000935093	
+3.88672	0.00447391	0.0239503	        0	0.000107152	0.00447391	 0.114323	0.00105384	
+3.92578	0.0043817	0.0282906	        0	0.000123961	0.0043817	 0.136454	0.00116136	
+3.96484	0.00429045	0.0326309	        0	0.000140001	0.00429045	 0.159053	0.0012595	
+4.00391	0.00420017	0.0369711	        0	0.000155285	0.00420017	 0.182136	0.00134957	
+4.04297	0.00411084	0.0413114	        0	0.000169825	0.00411084	 0.205717	0.00143255	
+4.08203	0.00402247	0.0456517	        0	0.000183633	0.00402247	 0.229814	0.00150919	
+4.12109	0.00393506	 0.049992	        0	0.000196722	0.00393506	 0.254443	0.00158008	
+4.16016	0.00384861	0.0543323	        0	0.000209104	0.00384861	 0.279622	0.0016457	
+4.19922	0.00376313	0.0586725	        0	0.000220792	0.00376313	  0.30537	0.00170647	
+4.23828	0.0036786	0.0630128	        0	0.000231799	0.0036786	 0.331706	0.00176272	
+4.27734	0.00359503	0.0673531	        0	0.000242137	0.00359503	 0.358651	0.00181475	
+4.31641	0.00351242	0.0716934	        0	0.000251818	0.00351242	 0.386226	0.00186281	
+4.35547	0.00343078	0.0760336	        0	0.000260855	0.00343078	 0.414453	0.00190711	
+4.39453	0.00335009	0.0803739	        0	0.00026926	0.00335009	 0.443356	0.00194786	
+4.43359	0.00327036	0.0847142	        0	0.000277046	0.00327036	 0.472959	0.00198523	
+4.47266	0.0031916	0.0890545	        0	0.000284226	0.0031916	 0.503289	0.00201939	
+4.51172	0.00311379	0.0933948	        0	0.000290812	0.00311379	 0.534371	0.00205046	
+4.55078	0.00303694	 0.097735	        0	0.000296816	0.00303694	 0.566236	0.00207859	
+4.58984	0.00296106	 0.102075	        0	0.000302251	0.00296106	 0.598912	0.00210389	
+4.62891	0.00288613	 0.106416	        0	0.000307129	0.00288613	  0.63243	0.00212646	
+4.66797	0.00281217	 0.110756	        0	0.000311464	0.00281217	 0.666825	0.00214642	
+4.70703	0.00273916	 0.115096	        0	0.000315267	0.00273916	  0.70213	0.00216386	
+4.74609	0.00266712	 0.119436	        0	0.000318551	0.00266712	 0.738383	0.00217886	
+4.78516	0.00259603	 0.123777	        0	0.000321328	0.00259603	 0.775621	0.00219151	
+4.82422	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.86328	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.90234	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.94141	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.98047	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.01953	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.05859	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.09766	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.13672	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.17578	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.21484	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.25391	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.29297	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.33203	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.37109	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.41016	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.44922	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.48828	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.52734	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.56641	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.60547	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.64453	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.68359	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.72266	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.76172	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.80078	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.83984	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.87891	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.91797	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.95703	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.99609	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.03516	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.07422	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.11328	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.15234	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.19141	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.23047	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.26953	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.30859	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.34766	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.38672	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.42578	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.46484	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.50391	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.54297	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.58203	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.62109	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.66016	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.69922	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.73828	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.77734	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.81641	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.85547	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.89453	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.93359	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.97266	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.01172	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.05078	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.08984	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.12891	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.16797	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.20703	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.24609	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.28516	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.32422	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.36328	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.40234	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.44141	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.48047	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.51953	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.55859	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.59766	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.63672	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.67578	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.71484	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.75391	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.79297	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.83203	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.87109	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.91016	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.94922	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.98828	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.02734	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.06641	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.10547	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.14453	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.18359	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.22266	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.26172	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.30078	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.33984	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.37891	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.41797	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.45703	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.49609	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.53516	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.57422	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.61328	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.65234	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.69141	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.73047	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.76953	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.80859	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.84766	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.88672	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.92578	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.96484	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.00391	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.04297	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.08203	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.12109	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.16016	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.19922	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.23828	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.27734	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.31641	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.35547	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.39453	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.43359	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.47266	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.51172	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.55078	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.58984	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.62891	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.66797	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.70703	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.74609	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.78516	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.82422	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.86328	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.90234	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.94141	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.98047	    0.001	        0	        0	        0	    0.001	        0	        0	

reference/swashes_1_nx=512.csv

@@ -0,0 +1,530 @@
+##############################################################################
+# Generated by SWASHES version 1.03.00, 2016-01-29
+##############################################################################
+# Dimension: 1
+# Type: 3 (=Dam break)
+# Domain: 1
+# Choice: 1 (=on a wet domain without friction (Stoker's solution))
+##############################################################################
+# PARAMETERS OF THE SOLUTION
+# 
+# Length of the domain: 10 meters
+# Space step: 0.0195312 meters
+# Number of cells: 512
+# Position of the dam: x=5 meters
+# Time value: 6 seconds
+##############################################################################
+# 
+#(i-0.5)*dx 	    h[i] 	    u[i] 	 topo[i] 	    q[i] 	 topo[i]+h[i] 	Fr[i]=Froude	 topo[i]+hc[i] 	
+0.00976562	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0292969	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0488281	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0683594	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.0878906	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.107422	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.126953	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.146484	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.166016	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.185547	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.205078	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.224609	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.244141	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.263672	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.283203	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.302734	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.322266	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.341797	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.361328	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.380859	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.400391	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.419922	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.439453	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.458984	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.478516	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.498047	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.517578	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.537109	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.556641	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.576172	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.595703	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.615234	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.634766	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.654297	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.673828	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.693359	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.712891	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.732422	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.751953	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.771484	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.791016	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.810547	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.830078	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.849609	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.869141	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.888672	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.908203	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.927734	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.947266	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.966797	    0.005	        0	        0	        0	    0.005	        0	        0	
+0.986328	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.00586	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.02539	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.04492	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.06445	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.08398	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.10352	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.12305	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.14258	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.16211	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.18164	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.20117	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.2207	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.24023	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.25977	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.2793	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.29883	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.31836	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.33789	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.35742	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.37695	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.39648	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.41602	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.43555	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.45508	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.47461	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.49414	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.51367	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.5332	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.55273	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.57227	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.5918	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.61133	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.63086	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.65039	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.66992	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.68945	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.70898	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.72852	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.74805	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.76758	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.78711	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.80664	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.82617	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.8457	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.86523	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.88477	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.9043	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.92383	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.94336	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.96289	    0.005	        0	        0	        0	    0.005	        0	        0	
+1.98242	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.00195	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.02148	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.04102	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.06055	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.08008	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.09961	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.11914	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.13867	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.1582	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.17773	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.19727	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.2168	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.23633	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.25586	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.27539	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.29492	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.31445	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.33398	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.35352	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.37305	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.39258	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.41211	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.43164	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.45117	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.4707	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.49023	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.50977	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.5293	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.54883	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.56836	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.58789	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.60742	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.62695	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.64648	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.66602	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.68555	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.70508	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.72461	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.74414	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.76367	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.7832	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.80273	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.82227	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.8418	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.86133	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.88086	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.90039	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.91992	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.93945	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.95898	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.97852	    0.005	        0	        0	        0	    0.005	        0	        0	
+2.99805	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.01758	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.03711	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.05664	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.07617	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.0957	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.11523	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.13477	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.1543	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.17383	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.19336	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.21289	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.23242	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.25195	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.27148	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.29102	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.31055	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.33008	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.34961	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.36914	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.38867	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.4082	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.42773	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.44727	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.4668	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.48633	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.50586	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.52539	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.54492	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.56445	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.58398	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.60352	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.62305	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.64258	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.66211	    0.005	        0	        0	        0	    0.005	        0	        0	
+3.68164	0.00497376	0.00116386	        0	5.78874e-006	0.00497376	0.00526893	0.000150603	
+3.70117	0.00492501	0.00333399	        0	1.642e-005	0.00492501	0.0151679	0.000301781	
+3.7207	0.00487651	0.00550413	        0	2.6841e-005	0.00487651	0.0251652	0.00041877	
+3.74023	0.00482825	0.00767427	        0	3.70533e-005	0.00482825	0.0352621	0.000519192	
+3.75977	0.00478022	0.00984441	        0	4.70585e-005	0.00478022	0.0454602	0.000608885	
+3.7793	0.00473244	0.0120146	        0	5.68581e-005	0.00473244	 0.055761	0.000690725	
+3.79883	0.00468489	0.0141847	        0	6.64537e-005	0.00468489	0.0661661	0.000766402	
+3.81836	0.00463759	0.0163548	        0	7.58469e-005	0.00463759	0.0766771	0.00083702	
+3.83789	0.00459052	 0.018525	        0	8.50393e-005	0.00459052	0.0872955	0.000903351	
+3.85742	0.0045437	0.0206951	        0	9.40323e-005	0.0045437	0.0980231	0.000965966	
+3.87695	0.00449711	0.0228652	        0	0.000102828	0.00449711	 0.108862	0.0010253	
+3.89648	0.00445077	0.0250354	        0	0.000111427	0.00445077	 0.119813	0.00108169	
+3.91602	0.00440467	0.0272055	        0	0.000119831	0.00440467	 0.130878	0.00113542	
+3.93555	0.0043588	0.0293757	        0	0.000128043	0.0043588	 0.142059	0.00118672	
+3.95508	0.00431318	0.0315458	        0	0.000136063	0.00431318	 0.153359	0.00123577	
+3.97461	0.00426779	0.0337159	        0	0.000143893	0.00426779	 0.164778	0.00128273	
+3.99414	0.00422265	0.0358861	        0	0.000151534	0.00422265	 0.176319	0.00132775	
+4.01367	0.00417774	0.0380562	        0	0.000158989	0.00417774	 0.187984	0.00137095	
+4.0332	0.00413308	0.0402264	        0	0.000166259	0.00413308	 0.199774	0.00141243	
+4.05273	0.00408866	0.0423965	        0	0.000173345	0.00408866	 0.211692	0.00145228	
+4.07227	0.00404447	0.0445666	        0	0.000180248	0.00404447	  0.22374	0.00149059	
+4.0918	0.00400053	0.0467368	        0	0.000186972	0.00400053	  0.23592	0.00152743	
+4.11133	0.00395682	0.0489069	        0	0.000193516	0.00395682	 0.248235	0.00156287	
+4.13086	0.00391336	0.0510771	        0	0.000199883	0.00391336	 0.260685	0.00159696	
+4.15039	0.00387014	0.0532472	        0	0.000206074	0.00387014	 0.273274	0.00162977	
+4.16992	0.00382715	0.0554173	        0	0.000212091	0.00382715	 0.286005	0.00166134	
+4.18945	0.00378441	0.0575875	        0	0.000217935	0.00378441	 0.298878	0.00169172	
+4.20898	0.0037419	0.0597576	        0	0.000223607	0.0037419	 0.311898	0.00172095	
+4.22852	0.00369964	0.0619277	        0	0.00022911	0.00369964	 0.325066	0.00174907	
+4.24805	0.00365762	0.0640979	        0	0.000234446	0.00365762	 0.338384	0.00177612	
+4.26758	0.00361583	 0.066268	        0	0.000239614	0.00361583	 0.351856	0.00180213	
+4.28711	0.00357429	0.0684382	        0	0.000244618	0.00357429	 0.365484	0.00182713	
+4.30664	0.00353299	0.0706083	        0	0.000249458	0.00353299	 0.379272	0.00185115	
+4.32617	0.00349192	0.0727784	        0	0.000254137	0.00349192	  0.39322	0.00187423	
+4.3457	0.0034511	0.0749486	        0	0.000258655	0.0034511	 0.407334	0.00189638	
+4.36523	0.00341052	0.0771187	        0	0.000263015	0.00341052	 0.421614	0.00191762	
+4.38477	0.00337017	0.0792889	        0	0.000267217	0.00337017	 0.436065	 0.001938	
+4.4043	0.00333007	 0.081459	        0	0.000271264	0.00333007	  0.45069	0.00195752	
+4.42383	0.00329021	0.0836291	        0	0.000275157	0.00329021	 0.465491	0.0019762	
+4.44336	0.00325058	0.0857993	        0	0.000278898	0.00325058	 0.480472	0.00199407	
+4.46289	0.0032112	0.0879694	        0	0.000282487	0.0032112	 0.495637	0.00201114	
+4.48242	0.00317206	0.0901396	        0	0.000285928	0.00317206	 0.510988	0.00202744	
+4.50195	0.00313315	0.0923097	        0	0.00028922	0.00313315	 0.526529	0.00204297	
+4.52148	0.00309449	0.0944798	        0	0.000292367	0.00309449	 0.542263	0.00205776	
+4.54102	0.00305607	  0.09665	        0	0.000295369	0.00305607	 0.558195	0.00207183	
+4.56055	0.00301788	0.0988201	        0	0.000298228	0.00301788	 0.574327	0.00208517	
+4.58008	0.00297994	  0.10099	        0	0.000300945	0.00297994	 0.590665	0.00209782	
+4.59961	0.00294224	  0.10316	        0	0.000303522	0.00294224	 0.607211	0.00210978	
+4.61914	0.00290477	 0.105331	        0	0.000305961	0.00290477	  0.62397	0.00212107	
+4.63867	0.00286755	 0.107501	        0	0.000308264	0.00286755	 0.640945	0.0021317	
+4.6582	0.00283057	 0.109671	        0	0.000310431	0.00283057	 0.658142	0.00214167	
+4.67773	0.00279383	 0.111841	        0	0.000312464	0.00279383	 0.675564	0.00215102	
+4.69727	0.00275732	 0.114011	        0	0.000314365	0.00275732	 0.693216	0.00215973	
+4.7168	0.00272106	 0.116181	        0	0.000316136	0.00272106	 0.711103	0.00216784	
+4.73633	0.00268504	 0.118351	        0	0.000317778	0.00268504	 0.729228	0.00217533	
+4.75586	0.00264925	 0.120521	        0	0.000319292	0.00264925	 0.747598	0.00218224	
+4.77539	0.00261371	 0.122692	        0	0.00032068	0.00261371	 0.766217	0.00218856	
+4.79492	0.00257841	 0.124862	        0	0.000321945	0.00257841	 0.785089	0.00219431	
+4.81445	0.00254335	 0.127032	        0	0.000323086	0.00254335	 0.804221	0.00219949	
+4.83398	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.85352	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.87305	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.89258	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.91211	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.93164	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.95117	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.9707	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+4.99023	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.00977	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.0293	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.04883	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.06836	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.08789	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.10742	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.12695	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.14648	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.16602	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.18555	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.20508	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.22461	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.24414	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.26367	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.2832	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.30273	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.32227	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.3418	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.36133	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.38086	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.40039	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.41992	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.43945	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.45898	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.47852	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.49805	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.51758	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.53711	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.55664	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.57617	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.5957	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.61523	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.63477	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.6543	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.67383	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.69336	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.71289	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.73242	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.75195	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.77148	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.79102	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.81055	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.83008	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.84961	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.86914	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.88867	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.9082	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.92773	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.94727	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.9668	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+5.98633	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.00586	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.02539	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.04492	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.06445	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.08398	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.10352	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.12305	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.14258	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.16211	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.18164	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.20117	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.2207	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.24023	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.25977	0.00253936	 0.127279	        0	0.000323208	0.00253936	 0.806419	0.00220005	
+6.2793	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.29883	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.31836	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.33789	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.35742	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.37695	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.39648	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.41602	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.43555	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.45508	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.47461	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.49414	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.51367	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.5332	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.55273	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.57227	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.5918	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.61133	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.63086	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.65039	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.66992	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.68945	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.70898	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.72852	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.74805	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.76758	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.78711	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.80664	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.82617	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.8457	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.86523	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.88477	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.9043	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.92383	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.94336	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.96289	    0.001	        0	        0	        0	    0.001	        0	        0	
+6.98242	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.00195	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.02148	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.04102	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.06055	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.08008	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.09961	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.11914	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.13867	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.1582	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.17773	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.19727	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.2168	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.23633	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.25586	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.27539	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.29492	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.31445	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.33398	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.35352	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.37305	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.39258	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.41211	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.43164	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.45117	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.4707	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.49023	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.50977	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.5293	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.54883	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.56836	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.58789	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.60742	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.62695	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.64648	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.66602	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.68555	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.70508	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.72461	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.74414	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.76367	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.7832	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.80273	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.82227	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.8418	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.86133	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.88086	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.90039	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.91992	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.93945	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.95898	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.97852	    0.001	        0	        0	        0	    0.001	        0	        0	
+7.99805	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.01758	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.03711	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.05664	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.07617	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.0957	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.11523	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.13477	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.1543	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.17383	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.19336	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.21289	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.23242	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.25195	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.27148	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.29102	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.31055	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.33008	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.34961	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.36914	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.38867	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.4082	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.42773	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.44727	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.4668	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.48633	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.50586	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.52539	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.54492	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.56445	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.58398	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.60352	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.62305	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.64258	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.66211	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.68164	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.70117	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.7207	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.74023	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.75977	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.7793	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.79883	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.81836	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.83789	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.85742	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.87695	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.89648	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.91602	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.93555	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.95508	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.97461	    0.001	        0	        0	        0	    0.001	        0	        0	
+8.99414	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.01367	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.0332	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.05273	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.07227	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.0918	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.11133	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.13086	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.15039	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.16992	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.18945	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.20898	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.22852	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.24805	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.26758	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.28711	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.30664	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.32617	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.3457	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.36523	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.38477	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.4043	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.42383	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.44336	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.46289	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.48242	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.50195	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.52148	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.54102	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.56055	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.58008	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.59961	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.61914	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.63867	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.6582	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.67773	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.69727	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.7168	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.73633	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.75586	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.77539	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.79492	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.81445	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.83398	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.85352	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.87305	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.89258	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.91211	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.93164	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.95117	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.9707	    0.001	        0	        0	        0	    0.001	        0	        0	
+9.99023	    0.001	        0	        0	        0	    0.001	        0	        0