From fea9c28b1674ca7fea5fcab94a63f284ad767764 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 14:55:51 +0100 Subject: [PATCH 01/55] convert cuda-codes located in /cuda to hip --- GPUSimulators/Autotuner.py | 279 +++++++ GPUSimulators/Common.py | 780 ++++++++++++++++++ GPUSimulators/CudaContext.py | 295 +++++++ GPUSimulators/CudaContext_cu.py | 272 ++++++ GPUSimulators/EE2D_KP07_dimsplit.py | 282 +++++++ GPUSimulators/FORCE.py | 129 +++ GPUSimulators/HLL.py | 124 +++ GPUSimulators/HLL2.py | 133 +++ GPUSimulators/IPythonMagic.py | 193 +++++ GPUSimulators/KP07.py | 138 ++++ GPUSimulators/KP07_dimsplit.py | 137 +++ GPUSimulators/LxF.py | 125 +++ GPUSimulators/MPISimulator.py | 480 +++++++++++ GPUSimulators/SHMEMSimulator.py | 263 ++++++ GPUSimulators/SHMEMSimulatorGroup.py | 394 +++++++++ GPUSimulators/Simulator.py | 288 +++++++ GPUSimulators/WAF.py | 127 +++ GPUSimulators/__init__.py | 5 + .../__pycache__/MPISimulator.cpython-39.pyc | Bin 0 -> 11058 bytes .../__pycache__/Simulator.cpython-39.pyc | Bin 0 -> 8433 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 182 bytes GPUSimulators/cuda/EE2D_KP07_dimsplit.cu | 250 ++++++ GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip | 251 ++++++ GPUSimulators/cuda/EulerCommon.h | 187 +++++ GPUSimulators/cuda/SWE2D_FORCE.cu | 143 ++++ GPUSimulators/cuda/SWE2D_FORCE.cu.hip | 144 ++++ GPUSimulators/cuda/SWE2D_HLL.cu | 161 ++++ GPUSimulators/cuda/SWE2D_HLL.cu.hip | 162 ++++ GPUSimulators/cuda/SWE2D_HLL2.cu | 216 +++++ GPUSimulators/cuda/SWE2D_HLL2.cu.hip | 217 +++++ GPUSimulators/cuda/SWE2D_KP07.cu | 233 ++++++ GPUSimulators/cuda/SWE2D_KP07.cu.hip | 234 ++++++ GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu | 216 +++++ GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip | 217 +++++ GPUSimulators/cuda/SWE2D_LxF.cu | 168 ++++ GPUSimulators/cuda/SWE2D_LxF.cu.hip | 169 ++++ GPUSimulators/cuda/SWE2D_WAF.cu | 178 ++++ GPUSimulators/cuda/SWE2D_WAF.cu.hip | 179 ++++ GPUSimulators/cuda/SWECommon.h | 533 ++++++++++++ GPUSimulators/cuda/common.h | 557 +++++++++++++ GPUSimulators/cuda/limiters.h | 118 +++ GPUSimulators/helpers/InitialConditions.py | 355 ++++++++ GPUSimulators/helpers/Visualization.py | 61 ++ 43 files changed, 9393 insertions(+) create mode 100644 GPUSimulators/Autotuner.py create mode 100644 GPUSimulators/Common.py create mode 100644 GPUSimulators/CudaContext.py create mode 100644 GPUSimulators/CudaContext_cu.py create mode 100644 GPUSimulators/EE2D_KP07_dimsplit.py create mode 100644 GPUSimulators/FORCE.py create mode 100644 GPUSimulators/HLL.py create mode 100644 GPUSimulators/HLL2.py create mode 100644 GPUSimulators/IPythonMagic.py create mode 100644 GPUSimulators/KP07.py create mode 100644 GPUSimulators/KP07_dimsplit.py create mode 100644 GPUSimulators/LxF.py create mode 100644 GPUSimulators/MPISimulator.py create mode 100644 GPUSimulators/SHMEMSimulator.py create mode 100644 GPUSimulators/SHMEMSimulatorGroup.py create mode 100644 GPUSimulators/Simulator.py create mode 100644 GPUSimulators/WAF.py create mode 100644 GPUSimulators/__init__.py create mode 100644 GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc create mode 100644 GPUSimulators/__pycache__/Simulator.cpython-39.pyc create mode 100644 GPUSimulators/__pycache__/__init__.cpython-39.pyc create mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu create mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip create mode 100644 GPUSimulators/cuda/EulerCommon.h create mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu create mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu create mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu create mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu create mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu create mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu create mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu create mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu.hip create mode 100644 GPUSimulators/cuda/SWECommon.h create mode 100644 GPUSimulators/cuda/common.h create mode 100644 GPUSimulators/cuda/limiters.h create mode 100644 GPUSimulators/helpers/InitialConditions.py create mode 100644 GPUSimulators/helpers/Visualization.py diff --git a/GPUSimulators/Autotuner.py b/GPUSimulators/Autotuner.py new file mode 100644 index 0000000..8dc38e4 --- /dev/null +++ b/GPUSimulators/Autotuner.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the different helper functions and +classes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import os +import gc +import numpy as np +import logging +from socket import gethostname + +#import pycuda.driver as cuda +import pyhip as cuda + +from GPUSimulators import Common, Simulator, CudaContext + +class Autotuner: + def __init__(self, + nx=2048, ny=2048, + block_widths=range(8, 32, 1), + block_heights=range(8, 32, 1)): + logger = logging.getLogger(__name__) + self.filename = "autotuning_data_" + gethostname() + ".npz" + self.nx = nx + self.ny = ny + self.block_widths = block_widths + self.block_heights = block_heights + self.performance = {} + + + def benchmark(self, simulator, force=False): + logger = logging.getLogger(__name__) + + #Run through simulators and benchmark + key = str(simulator.__name__) + logger.info("Benchmarking %s to %s", key, self.filename) + + #If this simulator has been benchmarked already, skip it + if (force==False and os.path.isfile(self.filename)): + with np.load(self.filename) as data: + if key in data["simulators"]: + logger.info("%s already benchmarked - skipping", key) + return + + # Set arguments to send to the simulators during construction + context = CudaContext.CudaContext(autotuning=False) + g = 9.81 + h0, hu0, hv0, dx, dy, dt = Autotuner.gen_test_data(nx=self.nx, ny=self.ny, g=g) + arguments = { + 'context': context, + 'h0': h0, 'hu0': hu0, 'hv0': hv0, + 'nx': self.nx, 'ny': self.ny, + 'dx': dx, 'dy': dy, 'dt': 0.9*dt, + 'g': g + } + + # Load existing data into memory + benchmark_data = { + "simulators": [], + } + if (os.path.isfile(self.filename)): + with np.load(self.filename) as data: + for k, v in data.items(): + benchmark_data[k] = v + + # Run benchmark + benchmark_data[key + "_megacells"] = Autotuner.benchmark_single_simulator(simulator, arguments, self.block_widths, self.block_heights) + benchmark_data[key + "_block_widths"] = self.block_widths + benchmark_data[key + "_block_heights"] = self.block_heights + benchmark_data[key + "_arguments"] = str(arguments) + + existing_sims = benchmark_data["simulators"] + if (isinstance(existing_sims, np.ndarray)): + existing_sims = existing_sims.tolist() + if (key not in existing_sims): + benchmark_data["simulators"] = existing_sims + [key] + + # Save to file + np.savez_compressed(self.filename, **benchmark_data) + + + + """ + Function which reads a numpy file with autotuning data + and reports the maximum performance and block size + """ + def get_peak_performance(self, simulator): + logger = logging.getLogger(__name__) + + assert issubclass(simulator, Simulator.BaseSimulator) + key = simulator.__name__ + + if (key in self.performance): + return self.performance[key] + else: + #Run simulation if required + if (not os.path.isfile(self.filename)): + logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) + self.benchmark(simulator) + + with np.load(self.filename) as data: + if key not in data['simulators']: + logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) + data.close() + self.benchmark(simulator) + data = np.load(self.filename) + + def find_max_index(megacells): + max_index = np.nanargmax(megacells) + return np.unravel_index(max_index, megacells.shape) + + megacells = data[key + '_megacells'] + block_widths = data[key + '_block_widths'] + block_heights = data[key + '_block_heights'] + j, i = find_max_index(megacells) + + self.performance[key] = { "block_width": block_widths[i], + "block_height": block_heights[j], + "megacells": megacells[j, i] } + logger.debug("Returning %s as peak performance parameters", self.performance[key]) + return self.performance[key] + + #This should never happen + raise "Something wrong: Could not get autotuning data!" + return None + + + + """ + Runs a set of benchmarks for a single simulator + """ + def benchmark_single_simulator(simulator, arguments, block_widths, block_heights): + logger = logging.getLogger(__name__) + + megacells = np.empty((len(block_heights), len(block_widths))) + megacells.fill(np.nan) + + logger.debug("Running %d benchmarks with %s", len(block_heights)*len(block_widths), simulator.__name__) + + sim_arguments = arguments.copy() + + with Common.Timer(simulator.__name__) as t: + for j, block_height in enumerate(block_heights): + sim_arguments.update({'block_height': block_height}) + for i, block_width in enumerate(block_widths): + sim_arguments.update({'block_width': block_width}) + megacells[j, i] = Autotuner.run_benchmark(simulator, sim_arguments) + + + logger.debug("Completed %s in %f seconds", simulator.__name__, t.secs) + + return megacells + + + """ + Runs a benchmark, and returns the number of megacells achieved + """ + def run_benchmark(simulator, arguments, timesteps=10, warmup_timesteps=2): + logger = logging.getLogger(__name__) + + #Initialize simulator + try: + sim = simulator(**arguments) + except: + #An exception raised - not possible to continue + logger.debug("Failed creating %s with arguments %s", simulator.__name__, str(arguments)) + return np.nan + + #Create timer events + start = cuda.Event() + end = cuda.Event() + + #Warmup + for i in range(warmup_timesteps): + sim.stepEuler(sim.dt) + + #Run simulation with timer + start.record(sim.stream) + for i in range(timesteps): + sim.stepEuler(sim.dt) + end.record(sim.stream) + + #Synchronize end event + end.synchronize() + + #Compute megacells + gpu_elapsed = end.time_since(start)*1.0e-3 + megacells = (sim.nx*sim.ny*timesteps / (1000*1000)) / gpu_elapsed + + #Sanity check solution + h, hu, hv = sim.download() + sane = True + sane = sane and Autotuner.sanity_check(h, 0.3, 0.7) + sane = sane and Autotuner.sanity_check(hu, -0.2, 0.2) + sane = sane and Autotuner.sanity_check(hv, -0.2, 0.2) + + if (sane): + logger.debug("%s [%d x %d] succeeded: %f megacells, gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], megacells, gpu_elapsed) + return megacells + else: + logger.debug("%s [%d x %d] failed: gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], gpu_elapsed) + return np.nan + + + + """ + Generates test dataset + """ + def gen_test_data(nx, ny, g): + width = 100.0 + height = 100.0 + dx = width / float(nx) + dy = height / float(ny) + + x_center = dx*nx/2.0 + y_center = dy*ny/2.0 + + #Create a gaussian "dam break" that will not form shocks + size = width / 5.0 + dt = 10**10 + + h = np.zeros((ny, nx), dtype=np.float32); + hu = np.zeros((ny, nx), dtype=np.float32); + hv = np.zeros((ny, nx), dtype=np.float32); + + extent = 1.0/np.sqrt(2.0) + x = (dx*(np.arange(0, nx, dtype=np.float32)+0.5) - x_center) / size + y = (dy*(np.arange(0, ny, dtype=np.float32)+0.5) - y_center) / size + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + r = np.minimum(1.0, np.sqrt(xv**2 + yv**2)) + xv = None + yv = None + gc.collect() + + #Generate highres + cos = np.cos(np.pi*r) + h = 0.5 + 0.1*0.5*(1.0 + cos) + hu = 0.1*0.5*(1.0 + cos) + hv = hu.copy() + + scale = 0.7 + max_h_estimate = 0.6 + max_u_estimate = 0.1*np.sqrt(2.0) + dx = width/nx + dy = height/ny + dt = scale * min(dx, dy) / (max_u_estimate + np.sqrt(g*max_h_estimate)) + + return h, hu, hv, dx, dy, dt + + """ + Checks that a variable is "sane" + """ + def sanity_check(variable, bound_min, bound_max): + maxval = np.amax(variable) + minval = np.amin(variable) + if (np.isnan(maxval) + or np.isnan(minval) + or maxval > bound_max + or minval < bound_min): + return False + else: + return True diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py new file mode 100644 index 0000000..76902c5 --- /dev/null +++ b/GPUSimulators/Common.py @@ -0,0 +1,780 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the different helper functions and +classes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import os + +import numpy as np +import time +import signal +import subprocess +import tempfile +import re +import io +import hashlib +import logging +import gc +import netCDF4 +import json + +import pycuda.compiler as cuda_compiler +import pycuda.gpuarray +import pycuda.driver as cuda +from pycuda.tools import PageLockedMemoryPool + + + + + + +def safeCall(cmd): + logger = logging.getLogger(__name__) + try: + #git rev-parse HEAD + current_dir = os.path.dirname(os.path.realpath(__file__)) + params = dict() + params['stderr'] = subprocess.STDOUT + params['cwd'] = current_dir + params['universal_newlines'] = True #text=True in more recent python + params['shell'] = False + if os.name == 'nt': + params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + stdout = subprocess.check_output(cmd, **params) + except subprocess.CalledProcessError as e: + output = e.output + logger.error("Git failed, \nReturn code: " + str(e.returncode) + "\nOutput: " + output) + raise e + + return stdout + +def getGitHash(): + return safeCall(["git", "rev-parse", "HEAD"]) + +def getGitStatus(): + return safeCall(["git", "status", "--porcelain", "-uno"]) + +def toJson(in_dict, compressed=True): + """ + Creates JSON string from a dictionary + """ + logger = logging.getLogger(__name__) + out_dict = in_dict.copy() + for key in out_dict: + if isinstance(out_dict[key], np.ndarray): + out_dict[key] = out_dict[key].tolist() + else: + try: + json.dumps(out_dict[key]) + except: + value = str(out_dict[key]) + logger.warning("JSON: Converting {:s} to string ({:s})".format(key, value)) + out_dict[key] = value + return json.dumps(out_dict) + +def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names=[], dt=None): + """ + Runs a simulation, and stores output in netcdf file. Stores the times given in + save_times, and saves all of the variables in list save_var_names. Elements in + save_var_names can be set to None if you do not want to save them + """ + profiling_data_sim_runner = { 'start': {}, 'end': {} } + profiling_data_sim_runner["start"]["t_sim_init"] = 0 + profiling_data_sim_runner["end"]["t_sim_init"] = 0 + profiling_data_sim_runner["start"]["t_nc_write"] = 0 + profiling_data_sim_runner["end"]["t_nc_write"] = 0 + profiling_data_sim_runner["start"]["t_full_step"] = 0 + profiling_data_sim_runner["end"]["t_full_step"] = 0 + + profiling_data_sim_runner["start"]["t_sim_init"] = time.time() + + logger = logging.getLogger(__name__) + + assert len(save_times) > 0, "Need to specify which times to save" + + with Timer("construct") as t: + sim = simulator(**simulator_args) + logger.info("Constructed in " + str(t.secs) + " seconds") + + #Create netcdf file and simulate + with DataDumper(outfile, mode='w', clobber=False) as outdata: + + #Create attributes (metadata) + outdata.ncfile.created = time.ctime(time.time()) + outdata.ncfile.git_hash = getGitHash() + outdata.ncfile.git_status = getGitStatus() + outdata.ncfile.simulator = str(simulator) + + # do not write fields to attributes (they are to large) + simulator_args_for_ncfile = simulator_args.copy() + del simulator_args_for_ncfile["rho"] + del simulator_args_for_ncfile["rho_u"] + del simulator_args_for_ncfile["rho_v"] + del simulator_args_for_ncfile["E"] + outdata.ncfile.sim_args = toJson(simulator_args_for_ncfile) + + #Create dimensions + outdata.ncfile.createDimension('time', len(save_times)) + outdata.ncfile.createDimension('x', simulator_args['nx']) + outdata.ncfile.createDimension('y', simulator_args['ny']) + + #Create variables for dimensions + ncvars = {} + ncvars['time'] = outdata.ncfile.createVariable('time', np.dtype('float32').char, 'time') + ncvars['x'] = outdata.ncfile.createVariable( 'x', np.dtype('float32').char, 'x') + ncvars['y'] = outdata.ncfile.createVariable( 'y', np.dtype('float32').char, 'y') + + #Fill variables with proper values + ncvars['time'][:] = save_times + extent = sim.getExtent() + ncvars['x'][:] = np.linspace(extent[0], extent[1], simulator_args['nx']) + ncvars['y'][:] = np.linspace(extent[2], extent[3], simulator_args['ny']) + + #Choose which variables to download (prune None from list, but keep the index) + download_vars = [] + for i, var_name in enumerate(save_var_names): + if var_name is not None: + download_vars += [i] + save_var_names = list(save_var_names[i] for i in download_vars) + + #Create variables + for var_name in save_var_names: + ncvars[var_name] = outdata.ncfile.createVariable(var_name, np.dtype('float32').char, ('time', 'y', 'x'), zlib=True, least_significant_digit=3) + + #Create step sizes between each save + t_steps = np.empty_like(save_times) + t_steps[0] = save_times[0] + t_steps[1:] = save_times[1:] - save_times[0:-1] + + profiling_data_sim_runner["end"]["t_sim_init"] = time.time() + + #Start simulation loop + progress_printer = ProgressPrinter(save_times[-1], print_every=10) + for k in range(len(save_times)): + #Get target time and step size there + t_step = t_steps[k] + t_end = save_times[k] + + #Sanity check simulator + try: + sim.check() + except AssertionError as e: + logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e))) + return outdata.filename + + profiling_data_sim_runner["start"]["t_full_step"] += time.time() + + #Simulate + if (t_step > 0.0): + sim.simulate(t_step, dt) + + profiling_data_sim_runner["end"]["t_full_step"] += time.time() + + profiling_data_sim_runner["start"]["t_nc_write"] += time.time() + + #Download + save_vars = sim.download(download_vars) + + #Save to file + for i, var_name in enumerate(save_var_names): + ncvars[var_name][k, :] = save_vars[i] + + profiling_data_sim_runner["end"]["t_nc_write"] += time.time() + + #Write progress to screen + print_string = progress_printer.getPrintString(t_end) + if (print_string): + logger.debug(print_string) + + logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps())) + + return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi + + + + + + +class Timer(object): + """ + Class which keeps track of time spent for a section of code + """ + def __init__(self, tag, log_level=logging.DEBUG): + self.tag = tag + self.log_level = log_level + self.logger = logging.getLogger(__name__) + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, *args): + self.end = time.time() + self.secs = self.end - self.start + self.msecs = self.secs * 1000 # millisecs + self.logger.log(self.log_level, "%s: %f ms", self.tag, self.msecs) + + def elapsed(self): + return time.time() - self.start + + + + + +class PopenFileBuffer(object): + """ + Simple class for holding a set of tempfiles + for communicating with a subprocess + """ + def __init__(self): + self.stdout = tempfile.TemporaryFile(mode='w+t') + self.stderr = tempfile.TemporaryFile(mode='w+t') + + def __del__(self): + self.stdout.close() + self.stderr.close() + + def read(self): + self.stdout.seek(0) + cout = self.stdout.read() + self.stdout.seek(0, 2) + + self.stderr.seek(0) + cerr = self.stderr.read() + self.stderr.seek(0, 2) + + return cout, cerr + +class IPEngine(object): + """ + Class for starting IPEngines for MPI processing in IPython + """ + def __init__(self, n_engines): + self.logger = logging.getLogger(__name__) + + #Start ipcontroller + self.logger.info("Starting IPController") + self.c_buff = PopenFileBuffer() + c_cmd = ["ipcontroller", "--ip='*'"] + c_params = dict() + c_params['stderr'] = self.c_buff.stderr + c_params['stdout'] = self.c_buff.stdout + c_params['shell'] = False + if os.name == 'nt': + c_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + self.c = subprocess.Popen(c_cmd, **c_params) + + #Wait until controller is running + time.sleep(3) + + #Start engines + self.logger.info("Starting IPEngines") + self.e_buff = PopenFileBuffer() + e_cmd = ["mpiexec", "-n", str(n_engines), "ipengine", "--mpi"] + e_params = dict() + e_params['stderr'] = self.e_buff.stderr + e_params['stdout'] = self.e_buff.stdout + e_params['shell'] = False + if os.name == 'nt': + e_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + self.e = subprocess.Popen(e_cmd, **e_params) + + # attach to a running cluster + import ipyparallel + self.cluster = ipyparallel.Client()#profile='mpi') + time.sleep(3) + while(len(self.cluster.ids) != n_engines): + time.sleep(0.5) + self.logger.info("Waiting for cluster...") + self.cluster = ipyparallel.Client()#profile='mpi') + + self.logger.info("Done") + + def __del__(self): + self.shutdown() + + def shutdown(self): + if (self.e is not None): + if (os.name == 'nt'): + self.logger.warn("Sending CTRL+C to IPEngine") + self.e.send_signal(signal.CTRL_C_EVENT) + + try: + self.e.communicate(timeout=3) + self.e.kill() + except subprocess.TimeoutExpired: + self.logger.warn("Killing IPEngine") + self.e.kill() + self.e.communicate() + self.e = None + + cout, cerr = self.e_buff.read() + self.logger.info("IPEngine cout: {:s}".format(cout)) + self.logger.info("IPEngine cerr: {:s}".format(cerr)) + self.e_buff = None + + gc.collect() + + if (self.c is not None): + if (os.name == 'nt'): + self.logger.warn("Sending CTRL+C to IPController") + self.c.send_signal(signal.CTRL_C_EVENT) + + try: + self.c.communicate(timeout=3) + self.c.kill() + except subprocess.TimeoutExpired: + self.logger.warn("Killing IPController") + self.c.kill() + self.c.communicate() + self.c = None + + cout, cerr = self.c_buff.read() + self.logger.info("IPController cout: {:s}".format(cout)) + self.logger.info("IPController cerr: {:s}".format(cerr)) + self.c_buff = None + + gc.collect() + + + + + + +class DataDumper(object): + """ + Simple class for holding a netCDF4 object + (handles opening and closing in a nice way) + Use as + with DataDumper("filename") as data: + ... + """ + def __init__(self, filename, *args, **kwargs): + self.logger = logging.getLogger(__name__) + + #Create directory if needed + filename = os.path.abspath(filename) + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + self.logger.info("Creating directory " + dirname) + os.makedirs(dirname) + + #Get mode of file if we have that + mode = None + if (args): + mode = args[0] + elif (kwargs and 'mode' in kwargs.keys()): + mode = kwargs['mode'] + + #Create new unique file if writing + if (mode): + if (("w" in mode) or ("+" in mode) or ("a" in mode)): + i = 0 + stem, ext = os.path.splitext(filename) + while (os.path.isfile(filename)): + filename = "{:s}_{:04d}{:s}".format(stem, i, ext) + i = i+1 + self.filename = os.path.abspath(filename) + + #Save arguments + self.args = args + self.kwargs = kwargs + + #Log output + self.logger.info("Initialized " + self.filename) + + + def __enter__(self): + self.logger.info("Opening " + self.filename) + if (self.args): + self.logger.info("Arguments: " + str(self.args)) + if (self.kwargs): + self.logger.info("Keyword arguments: " + str(self.kwargs)) + self.ncfile = netCDF4.Dataset(self.filename, *self.args, **self.kwargs) + return self + + def __exit__(self, *args): + self.logger.info("Closing " + self.filename) + self.ncfile.close() + + + def toJson(in_dict): + out_dict = in_dict.copy() + + for key in out_dict: + if isinstance(out_dict[key], np.ndarray): + out_dict[key] = out_dict[key].tolist() + else: + try: + json.dumps(out_dict[key]) + except: + out_dict[key] = str(out_dict[key]) + + return json.dumps(out_dict) + + + + + +class ProgressPrinter(object): + """ + Small helper class for + """ + def __init__(self, total_steps, print_every=5): + self.logger = logging.getLogger(__name__) + self.start = time.time() + self.total_steps = total_steps + self.print_every = print_every + self.next_print_time = self.print_every + self.last_step = 0 + self.secs_per_iter = None + + def getPrintString(self, step): + elapsed = time.time() - self.start + if (elapsed > self.next_print_time): + dt = elapsed - (self.next_print_time - self.print_every) + dsteps = step - self.last_step + steps_remaining = self.total_steps - step + + if (dsteps == 0): + return + + self.last_step = step + self.next_print_time = elapsed + self.print_every + + if not self.secs_per_iter: + self.secs_per_iter = dt / dsteps + self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps) + + remaining_time = steps_remaining * self.secs_per_iter + + return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format( + ProgressPrinter.progressBar(step, self.total_steps), + ProgressPrinter.timeString(elapsed + remaining_time), + ProgressPrinter.timeString(elapsed), + ProgressPrinter.timeString(remaining_time)) + + def timeString(seconds): + seconds = int(max(seconds, 1)) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + periods = [('h', hours), ('m', minutes), ('s', seconds)] + time_string = ' '.join('{}{}'.format(value, name) + for name, value in periods + if value) + return time_string + + def progressBar(step, total_steps, width=30): + progress = np.round(width * step / total_steps).astype(np.int32) + progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%" + return progressbar + + + + + + + +""" +Class that holds 2D data +""" +class CudaArray2D: + """ + Uploads initial data to the CUDA device + """ + def __init__(self, stream, nx, ny, x_halo, y_halo, cpu_data=None, dtype=np.float32): + self.logger = logging.getLogger(__name__) + self.nx = nx + self.ny = ny + self.x_halo = x_halo + self.y_halo = y_halo + + nx_halo = nx + 2*x_halo + ny_halo = ny + 2*y_halo + + #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny) + #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here + self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) + + #For returning to download + self.memorypool = PageLockedMemoryPool() + + #If we don't have any data, just allocate and return + if cpu_data is None: + return + + #Make sure data is in proper format + assert cpu_data.shape == (ny_halo, nx_halo) or cpu_data.shape == (self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.ny, self.nx)), str((ny_halo, nx_halo))) + assert cpu_data.itemsize == 4, "Wrong size of data type" + assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" + + #Create copy object from host to device + x = (nx_halo - cpu_data.shape[1]) // 2 + y = (ny_halo - cpu_data.shape[0]) // 2 + self.upload(stream, cpu_data, extent=[x, y, cpu_data.shape[1], cpu_data.shape[0]]) + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) + + + def __del__(self, *args): + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) + self.data.gpudata.free() + self.data = None + + """ + Enables downloading data from GPU to Python + """ + def download(self, stream, cpu_data=None, asynch=False, extent=None): + if (extent is None): + x = self.x_halo + y = self.y_halo + nx = self.nx + ny = self.ny + else: + x, y, nx, ny = extent + + if (cpu_data is None): + #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) + #Allocate host memory + #The following fails, don't know why (crashes python) + cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) + #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32) + #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32) + + assert nx == cpu_data.shape[1] + assert ny == cpu_data.shape[0] + assert x+nx <= self.nx + 2*self.x_halo + assert y+ny <= self.ny + 2*self.y_halo + + #Create copy object from device to host + copy = cuda.Memcpy2D() + copy.set_src_device(self.data.gpudata) + copy.set_dst_host(cpu_data) + + #Set offsets and pitch of source + copy.src_x_in_bytes = int(x)*self.data.strides[1] + copy.src_y = int(y) + copy.src_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + copy.width_in_bytes = int(nx)*cpu_data.itemsize + copy.height = int(ny) + + copy(stream) + if asynch==False: + stream.synchronize() + + return cpu_data + + + def upload(self, stream, cpu_data, extent=None): + if (extent is None): + x = self.x_halo + y = self.y_halo + nx = self.nx + ny = self.ny + else: + x, y, nx, ny = extent + + assert(nx == cpu_data.shape[1]) + assert(ny == cpu_data.shape[0]) + assert(x+nx <= self.nx + 2*self.x_halo) + assert(y+ny <= self.ny + 2*self.y_halo) + + #Create copy object from device to host + copy = cuda.Memcpy2D() + copy.set_dst_device(self.data.gpudata) + copy.set_src_host(cpu_data) + + #Set offsets and pitch of source + copy.dst_x_in_bytes = int(x)*self.data.strides[1] + copy.dst_y = int(y) + copy.dst_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + copy.width_in_bytes = int(nx)*cpu_data.itemsize + copy.height = int(ny) + + copy(stream) + + + + + + + + +""" +Class that holds 2D data +""" +class CudaArray3D: + """ + Uploads initial data to the CL device + """ + def __init__(self, stream, nx, ny, nz, x_halo, y_halo, z_halo, cpu_data=None, dtype=np.float32): + self.logger = logging.getLogger(__name__) + self.nx = nx + self.ny = ny + self.nz = nz + self.x_halo = x_halo + self.y_halo = y_halo + self.z_halo = z_halo + + nx_halo = nx + 2*x_halo + ny_halo = ny + 2*y_halo + nz_halo = nz + 2*z_halo + + #self.logger.debug("Allocating [%dx%dx%d] buffer", self.nx, self.ny, self.nz) + #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here + self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) + + #For returning to download + self.memorypool = PageLockedMemoryPool() + + #If we don't have any data, just allocate and return + if cpu_data is None: + return + + #Make sure data is in proper format + assert cpu_data.shape == (nz_halo, ny_halo, nx_halo) or cpu_data.shape == (self.nz, self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.nz, self.ny, self.nx)), str((nz_halo, ny_halo, nx_halo))) + assert cpu_data.itemsize == 4, "Wrong size of data type" + assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" + + #Create copy object from host to device + copy = cuda.Memcpy3D() + copy.set_src_host(cpu_data) + copy.set_dst_device(self.data.gpudata) + + #Set offsets of destination + x_offset = (nx_halo - cpu_data.shape[2]) // 2 + y_offset = (ny_halo - cpu_data.shape[1]) // 2 + z_offset = (nz_halo - cpu_data.shape[0]) // 2 + copy.dst_x_in_bytes = x_offset*self.data.strides[1] + copy.dst_y = y_offset + copy.dst_z = z_offset + + #Set pitch of destination + copy.dst_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + width = max(self.nx, cpu_data.shape[2]) + height = max(self.ny, cpu_data.shape[1]) + depth = max(self.nz, cpu-data.shape[0]) + copy.width_in_bytes = width*cpu_data.itemsize + copy.height = height + copy.depth = depth + + #Perform the copy + copy(stream) + + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) + + + def __del__(self, *args): + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) + self.data.gpudata.free() + self.data = None + + """ + Enables downloading data from GPU to Python + """ + def download(self, stream, asynch=False): + #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) + #Allocate host memory + #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32) + #cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) + cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) + + #Create copy object from device to host + copy = cuda.Memcpy2D() + copy.set_src_device(self.data.gpudata) + copy.set_dst_host(cpu_data) + + #Set offsets and pitch of source + copy.src_x_in_bytes = self.x_halo*self.data.strides[1] + copy.src_y = self.y_halo + copy.src_z = self.z_halo + copy.src_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + copy.width_in_bytes = self.nx*cpu_data.itemsize + copy.height = self.ny + copy.depth = self.nz + + copy(stream) + if asynch==False: + stream.synchronize() + + return cpu_data + + + + + + + + + +""" +A class representing an Arakawa A type (unstaggered, logically Cartesian) grid +""" +class ArakawaA2D: + def __init__(self, stream, nx, ny, halo_x, halo_y, cpu_variables): + """ + Uploads initial data to the GPU device + """ + self.logger = logging.getLogger(__name__) + self.gpu_variables = [] + for cpu_variable in cpu_variables: + self.gpu_variables += [CudaArray2D(stream, nx, ny, halo_x, halo_y, cpu_variable)] + + def __getitem__(self, key): + assert type(key) == int, "Indexing is int based" + if (key > len(self.gpu_variables) or key < 0): + raise IndexError("Out of bounds") + return self.gpu_variables[key] + + def download(self, stream, variables=None): + """ + Enables downloading data from the GPU device to Python + """ + if variables is None: + variables=range(len(self.gpu_variables)) + + cpu_variables = [] + for i in variables: + assert i < len(self.gpu_variables), "Variable {:d} is out of range".format(i) + cpu_variables += [self.gpu_variables[i].download(stream, asynch=True)] + + #stream.synchronize() + return cpu_variables + + def check(self): + """ + Checks that data is still sane + """ + for i, gpu_variable in enumerate(self.gpu_variables): + var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() + self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny)) + assert np.isnan(var_sum) == False, "Data contains NaN values!" + \ No newline at end of file diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py new file mode 100644 index 0000000..b4a2490 --- /dev/null +++ b/GPUSimulators/CudaContext.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements Cuda context handling + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import os + +import numpy as np +import time +import re +import io +import hashlib +import logging +import gc + +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda + +from hip import hip,hiprtc +from hip import rccl + +from GPUSimulators import Autotuner, Common + + + +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): + """ + Create a new CUDA context + Set device to an id or pci_bus_id to select a specific GPU + Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context + """ + self.use_cache = use_cache + self.logger = logging.getLogger(__name__) + self.modules = {} + + self.module_path = os.path.dirname(os.path.realpath(__file__)) + + #Initialize cuda (must be first call to PyCUDA) + ##cuda.init(flags=0) + + ##self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) + + #Print some info about CUDA + ##self.logger.info("CUDA version %s", str(cuda.get_version())) + #self.logger.info("Driver version %s", str(cuda.get_driver_version())) + self.logger.info("Driver version %s", str(hip_check(hip.hipDriverGetVersion()))) + + if device is None: + device = 0 + + self.cuda_device = hip.Device(device) + #self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) + self.logger.info("Using device %d/%d '%s' (%s) GPU", device, hip_check(hip.hipGetDeviceCount())) + #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) + self.logger.debug(" => compute capability: %s", str(self.hip.hipDeviceComputeCapability(device))) + + # Create the CUDA context + #In HIP there is no need to specify a scheduling policy (it is abstracted). Here the HIP runtime system manages the workload to fit a specifc target architecture + #if context_flags is None: + # context_flags=cuda.ctx_flags.SCHED_AUTO + + #self.cuda_context = self.cuda_device.make_context(flags=context_flags) + + #free, total = cuda.mem_get_info() + total = hip_check(hip.hipDeviceTotalMem(device)) + #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) + self.logger.debug(" => memory: %d / %d MB available", int(total/(1024*1024))) + + #self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + + #Create cache dir for cubin files + self.cache_path = os.path.join(self.module_path, "cuda_cache") + if (self.use_cache): + if not os.path.isdir(self.cache_path): + os.mkdir(self.cache_path) + self.logger.info("Using CUDA cache dir %s", self.cache_path) + + self.autotuner = None + if (autotuning): + self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") + self.autotuner = Autotuner.Autotuner() + + +# def __del__(self, *args): +# self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + + # Loop over all contexts in stack, and remove "this" +# other_contexts = [] +# while (cuda.Context.get_current() != None): +# context = cuda.Context.get_current() +# if (context.handle != self.cuda_context.handle): +# self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) +# other_contexts = [context] + other_contexts +# cuda.Context.pop() +# else: +# self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) +# cuda.Context.pop() + + # Add all the contexts we popped that were not our own +# for context in other_contexts: +# self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) +# cuda.Context.push(context) + +# self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) +# self.cuda_context.detach() + + +# def __str__(self): +# return "CudaContext id " + str(self.cuda_context.handle) + + + def hash_kernel(kernel_filename, include_dirs): + # Generate a kernel ID for our caches + num_includes = 0 + max_includes = 100 + kernel_hasher = hashlib.md5() + logger = logging.getLogger(__name__) + + # Loop over file and includes, and check if something has changed + files = [kernel_filename] + while len(files): + + if (num_includes > max_includes): + raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) + + filename = files.pop() + + #logger.debug("Hashing %s", filename) + + modified = os.path.getmtime(filename) + + # Open the file + with io.open(filename, "r") as file: + + # Search for #inclue and also hash the file + file_str = file.read() + kernel_hasher.update(file_str.encode('utf-8')) + kernel_hasher.update(str(modified).encode('utf-8')) + + #Find all includes + includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) + + # Loop over everything that looks like an include + for include_file in includes: + + #Search through include directories for the file + file_path = os.path.dirname(filename) + for include_path in [file_path] + include_dirs: + + # If we find it, add it to list of files to check + temp_path = os.path.join(include_path, include_file) + if (os.path.isfile(temp_path)): + files = files + [temp_path] + num_includes = num_includes + 1 #For circular includes... + break + + return kernel_hasher.hexdigest() + + + """ + Reads a text file and creates an OpenCL kernel from that + """ + def get_module(self, kernel_filename, + include_dirs=[], \ + defines={}, \ + compile_args={'no_extern_c', True}, jit_compile_args={}): + """ + Helper function to print compilation output + """ + def cuda_compile_message_handler(compile_success_bool, info_str, error_str): + self.logger.debug("Compilation returned %s", str(compile_success_bool)) + if info_str: + self.logger.debug("Info: %s", info_str) + if error_str: + self.logger.debug("Error: %s", error_str) + + kernel_filename = os.path.normpath(kernel_filename) + kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) + #self.logger.debug("Getting %s", kernel_filename) + + # Create a hash of the kernel options + options_hasher = hashlib.md5() + options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); + options_hash = options_hasher.hexdigest() + + # Create hash of kernel souce + source_hash = CudaContext.hash_kernel( \ + kernel_path, \ + include_dirs=[self.module_path] + include_dirs) + + # Create final hash + root, ext = os.path.splitext(kernel_filename) + kernel_hash = root \ + + "_" + source_hash \ + + "_" + options_hash \ + + ext + cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) + + # If we have the kernel in our hashmap, return it + if (kernel_hash in self.modules.keys()): + self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) + return self.modules[kernel_hash] + + # If we have it on disk, return it + elif (self.use_cache and os.path.isfile(cached_kernel_filename)): + self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) + + with io.open(cached_kernel_filename, "rb") as file: + file_str = file.read() + module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) + + self.modules[kernel_hash] = module + return module + + # Otherwise, compile it from source + else: + self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) + + #Create kernel string + kernel_string = "" + for key, value in defines.items(): + kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) + kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) + if (self.use_cache): + cached_kernel_dir = os.path.dirname(cached_kernel_filename) + if not os.path.isdir(cached_kernel_dir): + os.mkdir(cached_kernel_dir) + with io.open(cached_kernel_filename + ".txt", "w") as file: + file.write(kernel_string) + + + with Common.Timer("compiler") as timer: + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) + cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + if (self.use_cache): + with io.open(cached_kernel_filename, "wb") as file: + file.write(cubin) + + self.modules[kernel_hash] = module + return module + + """ + Clears the kernel cache (useful for debugging & development) + """ + def clear_kernel_cache(self): + self.logger.debug("Clearing cache") + self.modules = {} + gc.collect() + + """ + Synchronizes all streams etc + """ + def synchronize(self): + self.cuda_context.synchronize() diff --git a/GPUSimulators/CudaContext_cu.py b/GPUSimulators/CudaContext_cu.py new file mode 100644 index 0000000..6c90636 --- /dev/null +++ b/GPUSimulators/CudaContext_cu.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements Cuda context handling + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import os + +import numpy as np +import time +import re +import io +import hashlib +import logging +import gc + +import pycuda.compiler as cuda_compiler +import pycuda.gpuarray +import pycuda.driver as cuda + +from GPUSimulators import Autotuner, Common + + + +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): + """ + Create a new CUDA context + Set device to an id or pci_bus_id to select a specific GPU + Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context + """ + self.use_cache = use_cache + self.logger = logging.getLogger(__name__) + self.modules = {} + + self.module_path = os.path.dirname(os.path.realpath(__file__)) + + #Initialize cuda (must be first call to PyCUDA) + cuda.init(flags=0) + + self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) + + #Print some info about CUDA + self.logger.info("CUDA version %s", str(cuda.get_version())) + self.logger.info("Driver version %s", str(cuda.get_driver_version())) + + if device is None: + device = 0 + + self.cuda_device = cuda.Device(device) + self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) + self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) + + # Create the CUDA context + if context_flags is None: + context_flags=cuda.ctx_flags.SCHED_AUTO + + self.cuda_context = self.cuda_device.make_context(flags=context_flags) + + free, total = cuda.mem_get_info() + self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) + + self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + + #Create cache dir for cubin files + self.cache_path = os.path.join(self.module_path, "cuda_cache") + if (self.use_cache): + if not os.path.isdir(self.cache_path): + os.mkdir(self.cache_path) + self.logger.info("Using CUDA cache dir %s", self.cache_path) + + self.autotuner = None + if (autotuning): + self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") + self.autotuner = Autotuner.Autotuner() + + + def __del__(self, *args): + self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + + # Loop over all contexts in stack, and remove "this" + other_contexts = [] + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + if (context.handle != self.cuda_context.handle): + self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) + other_contexts = [context] + other_contexts + cuda.Context.pop() + else: + self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) + cuda.Context.pop() + + # Add all the contexts we popped that were not our own + for context in other_contexts: + self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) + cuda.Context.push(context) + + self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) + self.cuda_context.detach() + + + def __str__(self): + return "CudaContext id " + str(self.cuda_context.handle) + + + def hash_kernel(kernel_filename, include_dirs): + # Generate a kernel ID for our caches + num_includes = 0 + max_includes = 100 + kernel_hasher = hashlib.md5() + logger = logging.getLogger(__name__) + + # Loop over file and includes, and check if something has changed + files = [kernel_filename] + while len(files): + + if (num_includes > max_includes): + raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) + + filename = files.pop() + + #logger.debug("Hashing %s", filename) + + modified = os.path.getmtime(filename) + + # Open the file + with io.open(filename, "r") as file: + + # Search for #inclue and also hash the file + file_str = file.read() + kernel_hasher.update(file_str.encode('utf-8')) + kernel_hasher.update(str(modified).encode('utf-8')) + + #Find all includes + includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) + + # Loop over everything that looks like an include + for include_file in includes: + + #Search through include directories for the file + file_path = os.path.dirname(filename) + for include_path in [file_path] + include_dirs: + + # If we find it, add it to list of files to check + temp_path = os.path.join(include_path, include_file) + if (os.path.isfile(temp_path)): + files = files + [temp_path] + num_includes = num_includes + 1 #For circular includes... + break + + return kernel_hasher.hexdigest() + + + """ + Reads a text file and creates an OpenCL kernel from that + """ + def get_module(self, kernel_filename, + include_dirs=[], \ + defines={}, \ + compile_args={'no_extern_c', True}, jit_compile_args={}): + """ + Helper function to print compilation output + """ + def cuda_compile_message_handler(compile_success_bool, info_str, error_str): + self.logger.debug("Compilation returned %s", str(compile_success_bool)) + if info_str: + self.logger.debug("Info: %s", info_str) + if error_str: + self.logger.debug("Error: %s", error_str) + + kernel_filename = os.path.normpath(kernel_filename) + kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) + #self.logger.debug("Getting %s", kernel_filename) + + # Create a hash of the kernel options + options_hasher = hashlib.md5() + options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); + options_hash = options_hasher.hexdigest() + + # Create hash of kernel souce + source_hash = CudaContext.hash_kernel( \ + kernel_path, \ + include_dirs=[self.module_path] + include_dirs) + + # Create final hash + root, ext = os.path.splitext(kernel_filename) + kernel_hash = root \ + + "_" + source_hash \ + + "_" + options_hash \ + + ext + cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) + + # If we have the kernel in our hashmap, return it + if (kernel_hash in self.modules.keys()): + self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) + return self.modules[kernel_hash] + + # If we have it on disk, return it + elif (self.use_cache and os.path.isfile(cached_kernel_filename)): + self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) + + with io.open(cached_kernel_filename, "rb") as file: + file_str = file.read() + module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) + + self.modules[kernel_hash] = module + return module + + # Otherwise, compile it from source + else: + self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) + + #Create kernel string + kernel_string = "" + for key, value in defines.items(): + kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) + kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) + if (self.use_cache): + cached_kernel_dir = os.path.dirname(cached_kernel_filename) + if not os.path.isdir(cached_kernel_dir): + os.mkdir(cached_kernel_dir) + with io.open(cached_kernel_filename + ".txt", "w") as file: + file.write(kernel_string) + + + with Common.Timer("compiler") as timer: + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) + cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + if (self.use_cache): + with io.open(cached_kernel_filename, "wb") as file: + file.write(cubin) + + self.modules[kernel_hash] = module + return module + + """ + Clears the kernel cache (useful for debugging & development) + """ + def clear_kernel_cache(self): + self.logger.debug("Clearing cache") + self.modules = {} + gc.collect() + + """ + Synchronizes all streams etc + """ + def synchronize(self): + self.cuda_context.synchronize() \ No newline at end of file diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py new file mode 100644 index 0000000..2c3f810 --- /dev/null +++ b/GPUSimulators/EE2D_KP07_dimsplit.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the 2nd order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class EE2D_KP07_dimsplit (BaseSimulator): + + """ + Initialization routine + rho: Density + rho_u: Momentum along x-axis + rho_v: Momentum along y-axis + E: energy + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis + dy: Grid cell spacing along y-axis + dt: Size of each timestep + g: Gravitational constant + gamma: Gas constant + p: pressure + """ + def __init__(self, + context, + rho, rho_u, rho_v, E, + nx, ny, + dx, dy, + g, + gamma, + theta=1.3, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=8): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height) + self.g = np.float32(g) + self.gamma = np.float32(gamma) + self.theta = np.float32(theta) + + #Get kernels + module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("KP07DimsplitKernel") + self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") + + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [rho, rho_u, rho_v, E]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho))) + dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho))) + self.dt = min(dt_x, dt_y) + self.cfl_data.fill(self.dt, stream=self.stream) + + + def substep(self, dt, step_number, external=True, internal=True): + self.substepDimsplit(0.5*dt, step_number, external, internal) + + def substepDimsplit(self, dt, substep, external, internal): + if external and internal: + #print("COMPLETE DOMAIN (dt=" + str(dt) + ")") + + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + self.nx, self.ny) + return + + if external and not internal: + ################################### + # XXX: Corners are treated twice! # + ################################### + + ns_grid_size = (self.grid_size[0], 1) + + # NORTH + # (x0, y0) x (x1, y1) + # (0, ny-y_halo) x (nx, ny) + self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, self.ny - int(self.u0[0].y_halo), + self.nx, self.ny) + + # SOUTH + # (x0, y0) x (x1, y1) + # (0, 0) x (nx, y_halo) + self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + self.nx, int(self.u0[0].y_halo)) + + we_grid_size = (1, self.grid_size[1]) + + # WEST + # (x0, y0) x (x1, y1) + # (0, 0) x (x_halo, ny) + self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + int(self.u0[0].x_halo), self.ny) + + # EAST + # (x0, y0) x (x1, y1) + # (nx-x_halo, 0) x (nx, ny) + self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + self.nx - int(self.u0[0].x_halo), 0, + self.nx, self.ny) + return + + if internal and not external: + + # INTERNAL DOMAIN + # (x0, y0) x (x1, y1) + # (x_halo, y_halo) x (nx - x_halo, ny - y_halo) + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.internal_stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + int(self.u0[0].x_halo), int(self.u0[0].y_halo), + self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo)) + return + + def swapBuffers(self): + self.u0, self.u1 = self.u1, self.u0 + return + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + return + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py new file mode 100644 index 0000000..2e60be6 --- /dev/null +++ b/GPUSimulators/FORCE.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the FORCE flux +for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + + + + + + +""" +Class that solves the SW equations +""" +class FORCE (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height) + self.g = np.float32(g) + + #Get kernels + module = context.get_module("cuda/SWE2D_FORCE.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("FORCEKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt \ No newline at end of file diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py new file mode 100644 index 0000000..9911f9b --- /dev/null +++ b/GPUSimulators/HLL.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + + +""" +Class that solves the SW equations using the Harten-Lax -van Leer approximate Riemann solver +""" +class HLL (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height); + self.g = np.float32(g) + + #Get kernels + module = context.get_module("cuda/SWE2D_HLL.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("HLLKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py new file mode 100644 index 0000000..64fa765 --- /dev/null +++ b/GPUSimulators/HLL2.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the 2nd order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class HLL2 (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.8, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height); + self.g = np.float32(g) + self.theta = np.float32(theta) + + #Get kernels + module = context.get_module("cuda/SWE2D_HLL2.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("HLL2Kernel") + self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/IPythonMagic.py b/GPUSimulators/IPythonMagic.py new file mode 100644 index 0000000..fa452df --- /dev/null +++ b/GPUSimulators/IPythonMagic.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements helpers for IPython / Jupyter and CUDA + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging +import gc + +from IPython.core import magic_arguments +from IPython.core.magic import line_magic, Magics, magics_class +import pycuda.driver as cuda + +from GPUSimulators import Common, CudaContext + + +@magics_class +class MagicCudaContext(Magics): + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--blocking', '-b', action="store_true", help='Enable blocking context') + @magic_arguments.argument( + '--no_cache', '-nc', action="store_true", help='Disable caching of kernels') + @magic_arguments.argument( + '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels') + def cuda_context_handler(self, line): + args = magic_arguments.parse_argstring(self.cuda_context_handler, line) + self.logger = logging.getLogger(__name__) + + self.logger.info("Registering %s in user workspace", args.name) + + context_flags = None + if (args.blocking): + context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC + + if args.name in self.shell.user_ns.keys(): + self.logger.debug("Context already registered! Ignoring") + return + else: + self.logger.debug("Creating context") + use_cache = False if args.no_cache else True + use_autotuning = False if args.no_autotuning else True + self.shell.user_ns[args.name] = CudaContext.CudaContext(context_flags=context_flags, use_cache=use_cache, autotuning=use_autotuning) + + # this function will be called on exceptions in any cell + def custom_exc(shell, etype, evalue, tb, tb_offset=None): + self.logger.exception("Exception caught: Resetting to CUDA context %s", args.name) + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + self.logger.info("Popping <%s>", str(context.handle)) + cuda.Context.pop() + + if args.name in self.shell.user_ns.keys(): + self.logger.info("Pushing <%s>", str(self.shell.user_ns[args.name].cuda_context.handle)) + self.shell.user_ns[args.name].cuda_context.push() + else: + self.logger.error("No CUDA context called %s found (something is wrong)", args.name) + self.logger.error("CUDA will not work now") + + self.logger.debug("==================================================================") + + # still show the error within the notebook, don't just swallow it + shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset) + + # this registers a custom exception handler for the whole current notebook + get_ipython().set_custom_exc((Exception,), custom_exc) + + + # Handle CUDA context when exiting python + import atexit + def exitfunc(): + self.logger.info("Exitfunc: Resetting CUDA context stack") + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + self.logger.info("`-> Popping <%s>", str(context.handle)) + cuda.Context.pop() + self.logger.debug("==================================================================") + atexit.register(exitfunc) + + + + + + + + +@magics_class +class MagicLogger(Magics): + logger_initialized = False + + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--out', '-o', type=str, default='output.log', help='The filename to store the log to') + @magic_arguments.argument( + '--level', '-l', type=int, default=20, help='The level of logging to screen [0, 50]') + @magic_arguments.argument( + '--file_level', '-f', type=int, default=10, help='The level of logging to file [0, 50]') + def setup_logging(self, line): + if (self.logger_initialized): + logging.getLogger('GPUSimulators').info("Global logger already initialized!") + return; + else: + self.logger_initialized = True + + args = magic_arguments.parse_argstring(self.setup_logging, line) + import sys + + #Get root logger + logger = logging.getLogger('GPUSimulators') + logger.setLevel(min(args.level, args.file_level)) + + #Add log to screen + ch = logging.StreamHandler() + ch.setLevel(args.level) + logger.addHandler(ch) + logger.log(args.level, "Console logger using level %s", logging.getLevelName(args.level)) + + #Get the outfilename (try to evaluate if Python expression...) + try: + outfile = eval(args.out, self.shell.user_global_ns, self.shell.user_ns) + except: + outfile = args.out + + #Add log to file + logger.log(args.level, "File logger using level %s to %s", logging.getLevelName(args.file_level), outfile) + + fh = logging.FileHandler(outfile) + formatter = logging.Formatter('%(asctime)s:%(name)s:%(levelname)s: %(message)s') + fh.setFormatter(formatter) + fh.setLevel(args.file_level) + logger.addHandler(fh) + + logger.info("Python version %s", sys.version) + self.shell.user_ns[args.name] = logger + + + + + + +@magics_class +class MagicMPI(Magics): + + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--num_engines', '-n', type=int, default=4, help='Number of engines to start') + def setup_mpi(self, line): + args = magic_arguments.parse_argstring(self.setup_mpi, line) + logger = logging.getLogger('GPUSimulators') + if args.name in self.shell.user_ns.keys(): + logger.warning("MPI alreay set up, resetting") + self.shell.user_ns[args.name].shutdown() + self.shell.user_ns[args.name] = None + gc.collect() + self.shell.user_ns[args.name] = Common.IPEngine(args.num_engines) + + + + + + + + +# Register +ip = get_ipython() +ip.register_magics(MagicCudaContext) +ip.register_magics(MagicLogger) +ip.register_magics(MagicMPI) + diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py new file mode 100644 index 0000000..8e2af50 --- /dev/null +++ b/GPUSimulators/KP07.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class KP07 (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.3, + cfl_scale=0.9, + order=2, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + order, + block_width, block_height); + self.g = np.float32(g) + self.theta = np.float32(theta) + self.order = np.int32(order) + + #Get kernels + module = context.get_module("cuda/SWE2D_KP07.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("KP07Kernel") + self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + + def substep(self, dt, step_number): + self.substepRK(dt, step_number) + + + def substepRK(self, dt, substep): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + Simulator.stepOrderToCodedInt(step=substep, order=self.order), + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5**(self.order-1) \ No newline at end of file diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py new file mode 100644 index 0000000..c90cf07 --- /dev/null +++ b/GPUSimulators/KP07_dimsplit.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + +""" +Class that solves the SW equations using the dimentionally split KP07 scheme +""" +class KP07_dimsplit(Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.3, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height) + self.gc_x = 2 + self.gc_y = 2 + self.g = np.float32(g) + self.theta = np.float32(theta) + + #Get kernels + module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("KP07DimsplitKernel") + self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + self.gc_x, self.gc_y, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + self.gc_x, self.gc_y, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py new file mode 100644 index 0000000..48d9127 --- /dev/null +++ b/GPUSimulators/LxF.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the classical Lax-Friedrichs numerical +scheme for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + + + +""" +Class that solves the SW equations using the Lax Friedrichs scheme +""" +class LxF (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height); + self.g = np.float32(g) + + # Get kernels + module = context.get_module("cuda/SWE2D_LxF.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("LxFKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py new file mode 100644 index 0000000..31f62e8 --- /dev/null +++ b/GPUSimulators/MPISimulator.py @@ -0,0 +1,480 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements MPI simulator class + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator +import numpy as np +from mpi4py import MPI +import time + +import pycuda.driver as cuda +#import nvtx + + + +class MPIGrid(object): + """ + Class which represents an MPI grid of nodes. Facilitates easy communication between + neighboring nodes + """ + def __init__(self, comm, ndims=2): + self.logger = logging.getLogger(__name__) + + assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" + assert comm.size >= 1, "Must have at least one node" + + self.grid = MPIGrid.getGrid(comm.size, ndims) + self.comm = comm + + self.logger.debug("Created MPI grid: {:}. Rank {:d} has coordinate {:}".format( + self.grid, self.comm.rank, self.getCoordinate())) + + def getCoordinate(self, rank=None): + if (rank is None): + rank = self.comm.rank + i = (rank % self.grid[0]) + j = (rank // self.grid[0]) + return i, j + + def getRank(self, i, j): + return j*self.grid[0] + i + + def getEast(self): + i, j = self.getCoordinate(self.comm.rank) + i = (i+1) % self.grid[0] + return self.getRank(i, j) + + def getWest(self): + i, j = self.getCoordinate(self.comm.rank) + i = (i+self.grid[0]-1) % self.grid[0] + return self.getRank(i, j) + + def getNorth(self): + i, j = self.getCoordinate(self.comm.rank) + j = (j+1) % self.grid[1] + return self.getRank(i, j) + + def getSouth(self): + i, j = self.getCoordinate(self.comm.rank) + j = (j+self.grid[1]-1) % self.grid[1] + return self.getRank(i, j) + + def getGrid(num_nodes, num_dims): + assert(isinstance(num_nodes, int)) + assert(isinstance(num_dims, int)) + + # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors + # Original code by https://stackoverflow.com/users/3928385/ishamael + # Factorizes a number into n roughly equal factors + + #Dictionary to remember already computed permutations + memo = {} + def dp(n, left): # returns tuple (cost, [factors]) + """ + Recursively searches through all factorizations + """ + + #Already tried: return existing result + if (n, left) in memo: + return memo[(n, left)] + + #Spent all factors: return number itself + if left == 1: + return (n, [n]) + + #Find new factor + i = 2 + best = n + bestTuple = [n] + while i * i < n: + #If factor found + if n % i == 0: + #Factorize remainder + rem = dp(n // i, left - 1) + + #If new permutation better, save it + if rem[0] + i < best: + best = rem[0] + i + bestTuple = [i] + rem[1] + i += 1 + + #Store calculation + memo[(n, left)] = (best, bestTuple) + return memo[(n, left)] + + + grid = dp(num_nodes, num_dims)[1] + + if (len(grid) < num_dims): + #Split problematic 4 + if (4 in grid): + grid.remove(4) + grid.append(2) + grid.append(2) + + #Pad with ones to guarantee num_dims + grid = grid + [1]*(num_dims - len(grid)) + + #Sort in descending order + grid = np.sort(grid) + grid = grid[::-1] + + # XXX: We only use vertical (north-south) partitioning for now + grid[0] = 1 + grid[1] = num_nodes + + return grid + + + def gather(self, data, root=0): + out_data = None + if (self.comm.rank == root): + out_data = np.empty([self.comm.size] + list(data.shape), dtype=data.dtype) + self.comm.Gather(data, out_data, root) + return out_data + + def getLocalRank(self): + """ + Returns the local rank on this node for this MPI process + """ + + # This function has been adapted from + # https://github.com/SheffieldML/PyDeepGP/blob/master/deepgp/util/parallel.py + # by Zhenwen Dai released under BSD 3-Clause "New" or "Revised" License: + # + # Copyright (c) 2016, Zhenwen Dai + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # * Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. + # + # * Redistributions in binary form must reproduce the above copyright notice, + # this list of conditions and the following disclaimer in the documentation + # and/or other materials provided with the distribution. + # + # * Neither the name of DGP nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + #Get this ranks unique (physical) node name + node_name = MPI.Get_processor_name() + + #Gather the list of all node names on all nodes + node_names = self.comm.allgather(node_name) + + #Loop over all node names up until our rank + #and count how many duplicates of our nodename we find + local_rank = len([0 for name in node_names[:self.comm.rank] if name==node_name]) + + return local_rank + + +class MPISimulator(Simulator.BaseSimulator): + """ + Class which handles communication between simulators on different MPI nodes + """ + def __init__(self, sim, grid): + self.profiling_data_mpi = { 'start': {}, 'end': {} } + self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] = 0 + self.profiling_data_mpi["start"]["t_mpi_step"] = 0 + self.profiling_data_mpi["end"]["t_mpi_step"] = 0 + self.profiling_data_mpi["n_time_steps"] = 0 + self.logger = logging.getLogger(__name__) + + autotuner = sim.context.autotuner + sim.context.autotuner = None; + boundary_conditions = sim.getBoundaryConditions() + super().__init__(sim.context, + sim.nx, sim.ny, + sim.dx, sim.dy, + boundary_conditions, + sim.cfl_scale, + sim.num_substeps, + sim.block_size[0], sim.block_size[1]) + sim.context.autotuner = autotuner + + self.sim = sim + self.grid = grid + + #Get neighbor node ids + self.east = grid.getEast() + self.west = grid.getWest() + self.north = grid.getNorth() + self.south = grid.getSouth() + + #Get coordinate of this node + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate() + #print("gi: " + str(gi) + ", gj: " + str(gj)) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars = len(self.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(self.sim.getOutput()[0].x_halo) + gc_y = int(self.sim.getOutput()[0].y_halo) + nx = int(self.sim.nx) + ny = int(self.sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n = np.array([gc_x, ny, nx, gc_y]) + self.read_s = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e = self.read_e + np.array([gc_x, 0, 0, 0]) + self.write_w = self.read_w - np.array([gc_x, 0, 0, 0]) + self.write_n = self.read_n + np.array([0, gc_y, 0, 0]) + self.write_s = self.read_s - np.array([0, gc_y, 0, 0]) + + #Allocate data for receiving + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) + self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) + self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) + self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) + + #Allocate data for sending + self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) + self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) + self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) + self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) + + self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) + + self.full_exchange() + sim.context.synchronize() + + def substep(self, dt, step_number): + + #nvtx.mark("substep start", color="yellow") + + self.profiling_data_mpi["start"]["t_mpi_step"] += time.time() + + #nvtx.mark("substep external", color="blue") + self.sim.substep(dt, step_number, external=True, internal=False) # only "internal ghost cells" + + #nvtx.mark("substep internal", color="red") + self.sim.substep(dt, step_number, internal=True, external=False) # "internal ghost cells" excluded + + #nvtx.mark("substep full", color="blue") + #self.sim.substep(dt, step_number, external=True, internal=True) + + self.sim.swapBuffers() + + self.profiling_data_mpi["end"]["t_mpi_step"] += time.time() + + #nvtx.mark("exchange", color="blue") + self.full_exchange() + + #nvtx.mark("sync start", color="blue") + self.sim.stream.synchronize() + self.sim.internal_stream.synchronize() + #nvtx.mark("sync end", color="blue") + + self.profiling_data_mpi["n_time_steps"] += 1 + + def getOutput(self): + return self.sim.getOutput() + + def synchronize(self): + self.sim.synchronize() + + def check(self): + return self.sim.check() + + def computeDt(self): + local_dt = np.array([np.float32(self.sim.computeDt())]); + global_dt = np.empty(1, dtype=np.float32) + self.grid.comm.Allreduce(local_dt, global_dt, op=MPI.MIN) + self.logger.debug("Local dt: {:f}, global dt: {:f}".format(local_dt[0], global_dt[0])) + return global_dt[0] + + + def getExtent(self): + """ + Function which returns the extent of node with rank + rank in the grid + """ + width = self.sim.nx*self.sim.dx + height = self.sim.ny*self.sim.dy + i, j = self.grid.getCoordinate() + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def full_exchange(self): + #### + # First transfer internal cells north-south + #### + + #Download from the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() + + if self.north is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_n[k,:,:], asynch=True, extent=self.read_n) + if self.south is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_s[k,:,:], asynch=True, extent=self.read_s) + self.sim.stream.synchronize() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() + + #Send/receive to north/south neighbours + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + comm_send = [] + comm_recv = [] + if self.north is not None: + comm_send += [self.grid.comm.Isend(self.out_n, dest=self.north, tag=4*self.nt + 0)] + comm_recv += [self.grid.comm.Irecv(self.in_n, source=self.north, tag=4*self.nt + 1)] + if self.south is not None: + comm_send += [self.grid.comm.Isend(self.out_s, dest=self.south, tag=4*self.nt + 1)] + comm_recv += [self.grid.comm.Irecv(self.in_s, source=self.south, tag=4*self.nt + 0)] + + #Wait for incoming transfers to complete + for comm in comm_recv: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #Upload to the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() + + if self.north is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_n[k,:,:], extent=self.write_n) + if self.south is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_s[k,:,:], extent=self.write_s) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() + + #Wait for sending to complete + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + for comm in comm_send: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + + #Download from the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() + + if self.east is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_e[k,:,:], asynch=True, extent=self.read_e) + if self.west is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_w[k,:,:], asynch=True, extent=self.read_w) + self.sim.stream.synchronize() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() + + #Send/receive to east/west neighbours + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + comm_send = [] + comm_recv = [] + if self.east is not None: + comm_send += [self.grid.comm.Isend(self.out_e, dest=self.east, tag=4*self.nt + 2)] + comm_recv += [self.grid.comm.Irecv(self.in_e, source=self.east, tag=4*self.nt + 3)] + if self.west is not None: + comm_send += [self.grid.comm.Isend(self.out_w, dest=self.west, tag=4*self.nt + 3)] + comm_recv += [self.grid.comm.Irecv(self.in_w, source=self.west, tag=4*self.nt + 2)] + + #Wait for incoming transfers to complete + for comm in comm_recv: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #Upload to the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() + + if self.east is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_e[k,:,:], extent=self.write_e) + if self.west is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_w[k,:,:], extent=self.write_w) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() + + #Wait for sending to complete + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + for comm in comm_send: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py new file mode 100644 index 0000000..4385919 --- /dev/null +++ b/GPUSimulators/SHMEMSimulator.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements SHMEM simulator group class + +Copyright (C) 2020 Norwegian Meteorological Institute + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator, CudaContext +import numpy as np + +import pycuda.driver as cuda + +import time + +class SHMEMSimulator(Simulator.BaseSimulator): + """ + Class which handles communication and synchronization between simulators in different + contexts (presumably on different GPUs) + """ + def __init__(self, sims, grid): + self.logger = logging.getLogger(__name__) + + assert(len(sims) > 1) + + self.sims = sims + + # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? + # See also getOutput() and check(). + # + # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of + # SHMEMSimulators that have BaseSimulator as a superclass. + # + # This would also eliminate the need for all the array bookkeeping in this class. + autotuner = sims[0].context.autotuner + sims[0].context.autotuner = None + boundary_conditions = sims[0].getBoundaryConditions() + super().__init__(sims[0].context, + sims[0].nx, sims[0].ny, + sims[0].dx, sims[0].dy, + boundary_conditions, + sims[0].cfl_scale, + sims[0].num_substeps, + sims[0].block_size[0], sims[0].block_size[1]) + sims[0].context.autotuner = autotuner + + self.sims = sims + self.grid = grid + + self.east = [None] * len(self.sims) + self.west = [None] * len(self.sims) + self.north = [None] * len(self.sims) + self.south = [None] * len(self.sims) + + self.nvars = [None] * len(self.sims) + + self.read_e = [None] * len(self.sims) + self.read_w = [None] * len(self.sims) + self.read_n = [None] * len(self.sims) + self.read_s = [None] * len(self.sims) + + self.write_e = [None] * len(self.sims) + self.write_w = [None] * len(self.sims) + self.write_n = [None] * len(self.sims) + self.write_s = [None] * len(self.sims) + + self.e = [None] * len(self.sims) + self.w = [None] * len(self.sims) + self.n = [None] * len(self.sims) + self.s = [None] * len(self.sims) + + for i, sim in enumerate(self.sims): + #Get neighbor subdomain ids + self.east[i] = grid.getEast(i) + self.west[i] = grid.getWest(i) + self.north[i] = grid.getNorth(i) + self.south[i] = grid.getSouth(i) + + #Get coordinate of this subdomain + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate(i) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars[i] = len(sim.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(sim.getOutput()[0].x_halo) + gc_y = int(sim.getOutput()[0].y_halo) + nx = int(sim.nx) + ny = int(sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) + self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) + self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) + self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) + self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) + + #Allocate host data + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) + self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) + self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) + self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) + + self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) + + + def substep(self, dt, step_number): + self.exchange() + + for i, sim in enumerate(self.sims): + sim.substep(dt, step_number) + + def getOutput(self): + # XXX: Does not return what we would expect. + # Returns first subdomain, but we want the whole domain. + return self.sims[0].getOutput() + + def synchronize(self): + for sim in self.sims: + sim.synchronize() + + def check(self): + # XXX: Does not return what we would expect. + # Checks only first subdomain, but we want to check the whole domain. + return self.sims[0].check() + + def computeDt(self): + global_dt = float("inf") + + for sim in self.sims: + sim.context.synchronize() + + for sim in self.sims: + local_dt = sim.computeDt() + if local_dt < global_dt: + global_dt = local_dt + self.logger.debug("Local dt: {:f}".format(local_dt)) + + self.logger.debug("Global dt: {:f}".format(global_dt)) + return global_dt + + def getExtent(self, index=0): + """ + Function which returns the extent of the subdomain with index + index in the grid + """ + width = self.sims[index].nx*self.sims[index].dx + height = self.sims[index].ny*self.sims[index].dy + i, j = self.grid.getCoordinate(index) + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def exchange(self): + #### + # First transfer internal cells north-south + #### + for i in range(len(self.sims)): + self.ns_download(i) + + for i in range(len(self.sims)): + self.ns_upload(i) + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + for i in range(len(self.sims)): + self.ew_download(i) + + for i in range(len(self.sims)): + self.ew_upload(i) + + def ns_download(self, i): + #Download from the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) + self.sims[i].stream.synchronize() + + def ns_upload(self, i): + #Upload to the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) + + def ew_download(self, i): + #Download from the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) + self.sims[i].stream.synchronize() + + def ew_upload(self, i): + #Upload to the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) + #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) + #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/SHMEMSimulatorGroup.py b/GPUSimulators/SHMEMSimulatorGroup.py new file mode 100644 index 0000000..fc11d50 --- /dev/null +++ b/GPUSimulators/SHMEMSimulatorGroup.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements SHMEM simulator group class + +Copyright (C) 2020 Norwegian Meteorological Institute + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator, CudaContext +import numpy as np + +import pycuda.driver as cuda + +import time + +class SHMEMGrid(object): + """ + Class which represents an SHMEM grid of GPUs. Facilitates easy communication between + neighboring subdomains in the grid. Contains one CUDA context per subdomain. + """ + def __init__(self, ngpus=None, ndims=2): + self.logger = logging.getLogger(__name__) + + cuda.init(flags=0) + self.logger.info("Initializing CUDA") + num_cuda_devices = cuda.Device.count() + + if ngpus is None: + ngpus = num_cuda_devices + + # XXX: disabled for testing on single-GPU system + #assert ngpus <= num_cuda_devices, "Trying to allocate more GPUs than are available in the system." + #assert ngpus >= 2, "Must have at least two GPUs available to run multi-GPU simulations." + + assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" + + self.ngpus = ngpus + self.ndims = ndims + + self.grid = SHMEMGrid.getGrid(self.ngpus, self.ndims) + + self.logger.debug("Created {:}-dimensional SHMEM grid, using {:} GPUs".format( + self.ndims, self.ngpus)) + + # XXX: Is this a natural place to store the contexts? Consider moving contexts out of this + # class, into notebook / calling script (shmemTesting.py) + self.cuda_contexts = [] + + for i in range(self.ngpus): + # XXX: disabled for testing on single-GPU system + #self.cuda_contexts.append(CudaContext.CudaContext(device=i, autotuning=False)) + self.cuda_contexts.append(CudaContext.CudaContext(device=0, autotuning=False)) + + def getCoordinate(self, index): + i = (index % self.grid[0]) + j = (index // self.grid[0]) + return i, j + + def getIndex(self, i, j): + return j*self.grid[0] + i + + def getEast(self, index): + i, j = self.getCoordinate(index) + i = (i+1) % self.grid[0] + return self.getIndex(i, j) + + def getWest(self, index): + i, j = self.getCoordinate(index) + i = (i+self.grid[0]-1) % self.grid[0] + return self.getIndex(i, j) + + def getNorth(self, index): + i, j = self.getCoordinate(index) + j = (j+1) % self.grid[1] + return self.getIndex(i, j) + + def getSouth(self, index): + i, j = self.getCoordinate(index) + j = (j+self.grid[1]-1) % self.grid[1] + return self.getIndex(i, j) + + def getGrid(num_gpus, num_dims): + assert(isinstance(num_gpus, int)) + assert(isinstance(num_dims, int)) + + # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors + # Original code by https://stackoverflow.com/users/3928385/ishamael + # Factorizes a number into n roughly equal factors + + #Dictionary to remember already computed permutations + memo = {} + def dp(n, left): # returns tuple (cost, [factors]) + """ + Recursively searches through all factorizations + """ + + #Already tried: return existing result + if (n, left) in memo: + return memo[(n, left)] + + #Spent all factors: return number itself + if left == 1: + return (n, [n]) + + #Find new factor + i = 2 + best = n + bestTuple = [n] + while i * i < n: + #If factor found + if n % i == 0: + #Factorize remainder + rem = dp(n // i, left - 1) + + #If new permutation better, save it + if rem[0] + i < best: + best = rem[0] + i + bestTuple = [i] + rem[1] + i += 1 + + #Store calculation + memo[(n, left)] = (best, bestTuple) + return memo[(n, left)] + + + grid = dp(num_gpus, num_dims)[1] + + if (len(grid) < num_dims): + #Split problematic 4 + if (4 in grid): + grid.remove(4) + grid.append(2) + grid.append(2) + + #Pad with ones to guarantee num_dims + grid = grid + [1]*(num_dims - len(grid)) + + #Sort in descending order + grid = np.sort(grid) + grid = grid[::-1] + + return grid + +class SHMEMSimulatorGroup(object): + """ + Class which handles communication and synchronization between simulators in different + contexts (typically on different GPUs) + """ + def __init__(self, sims, grid): + self.logger = logging.getLogger(__name__) + + assert(len(sims) > 1) + + self.sims = sims + + # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? + # See also getOutput() and check(). + # + # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of + # SHMEMSimulators that have BaseSimulator as a superclass. + # + # This would also eliminate the need for all the array bookkeeping in this class. + # + CONT HERE! Model shmemTesting after mpiTesting and divide existing functionality between SHMEMSimulatorGroup and SHMEMSimulator + + autotuner = sims[0].context.autotuner + sims[0].context.autotuner = None + boundary_conditions = sims[0].getBoundaryConditions() + super().__init__(sims[0].context, + sims[0].nx, sims[0].ny, + sims[0].dx, sims[0].dy, + boundary_conditions, + sims[0].cfl_scale, + sims[0].num_substeps, + sims[0].block_size[0], sims[0].block_size[1]) + sims[0].context.autotuner = autotuner + + self.sims = sims + self.grid = grid + + self.east = [None] * len(self.sims) + self.west = [None] * len(self.sims) + self.north = [None] * len(self.sims) + self.south = [None] * len(self.sims) + + self.nvars = [None] * len(self.sims) + + self.read_e = [None] * len(self.sims) + self.read_w = [None] * len(self.sims) + self.read_n = [None] * len(self.sims) + self.read_s = [None] * len(self.sims) + + self.write_e = [None] * len(self.sims) + self.write_w = [None] * len(self.sims) + self.write_n = [None] * len(self.sims) + self.write_s = [None] * len(self.sims) + + self.e = [None] * len(self.sims) + self.w = [None] * len(self.sims) + self.n = [None] * len(self.sims) + self.s = [None] * len(self.sims) + + for i, sim in enumerate(self.sims): + #Get neighbor subdomain ids + self.east[i] = grid.getEast(i) + self.west[i] = grid.getWest(i) + self.north[i] = grid.getNorth(i) + self.south[i] = grid.getSouth(i) + + #Get coordinate of this subdomain + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate(i) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars[i] = len(sim.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(sim.getOutput()[0].x_halo) + gc_y = int(sim.getOutput()[0].y_halo) + nx = int(sim.nx) + ny = int(sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) + self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) + self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) + self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) + self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) + + #Allocate host data + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) + self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) + self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) + self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) + + self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) + + + def substep(self, dt, step_number): + self.exchange() + + for i, sim in enumerate(self.sims): + sim.substep(dt, step_number) + + def getOutput(self): + # XXX: Does not return what we would expect. + # Returns first subdomain, but we want the whole domain. + return self.sims[0].getOutput() + + def synchronize(self): + for sim in self.sims: + sim.synchronize() + + def check(self): + # XXX: Does not return what we would expect. + # Checks only first subdomain, but we want to check the whole domain. + return self.sims[0].check() + + def computeDt(self): + global_dt = float("inf") + + for sim in self.sims: + sim.context.synchronize() + + for sim in self.sims: + local_dt = sim.computeDt() + if local_dt < global_dt: + global_dt = local_dt + self.logger.debug("Local dt: {:f}".format(local_dt)) + + self.logger.debug("Global dt: {:f}".format(global_dt)) + return global_dt + + def getExtent(self, index=0): + """ + Function which returns the extent of the subdomain with index + index in the grid + """ + width = self.sims[index].nx*self.sims[index].dx + height = self.sims[index].ny*self.sims[index].dy + i, j = self.grid.getCoordinate(index) + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def exchange(self): + #### + # First transfer internal cells north-south + #### + for i in range(len(self.sims)): + self.ns_download(i) + + for i in range(len(self.sims)): + self.ns_upload(i) + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + for i in range(len(self.sims)): + self.ew_download(i) + + for i in range(len(self.sims)): + self.ew_upload(i) + + def ns_download(self, i): + #Download from the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) + self.sims[i].stream.synchronize() + + def ns_upload(self, i): + #Upload to the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) + + def ew_download(self, i): + #Download from the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) + self.sims[i].stream.synchronize() + + def ew_upload(self, i): + #Upload to the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) + #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) + #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/Simulator.py b/GPUSimulators/Simulator.py new file mode 100644 index 0000000..178183d --- /dev/null +++ b/GPUSimulators/Simulator.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the classical Lax-Friedrichs numerical +scheme for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +import numpy as np +import logging +from enum import IntEnum + +import pycuda.compiler as cuda_compiler +import pycuda.gpuarray +import pycuda.driver as cuda + +from GPUSimulators import Common + + + + + +class BoundaryCondition(object): + """ + Class for holding boundary conditions for global boundaries + """ + + + class Type(IntEnum): + """ + Enum that describes the different types of boundary conditions + WARNING: MUST MATCH THAT OF common.h IN CUDA + """ + Dirichlet = 0, + Neumann = 1, + Periodic = 2, + Reflective = 3 + + def __init__(self, types={ + 'north': Type.Reflective, + 'south': Type.Reflective, + 'east': Type.Reflective, + 'west': Type.Reflective + }): + """ + Constructor + """ + self.north = types['north'] + self.south = types['south'] + self.east = types['east'] + self.west = types['west'] + + if (self.north == BoundaryCondition.Type.Neumann \ + or self.south == BoundaryCondition.Type.Neumann \ + or self.east == BoundaryCondition.Type.Neumann \ + or self.west == BoundaryCondition.Type.Neumann): + raise(NotImplementedError("Neumann boundary condition not supported")) + + def __str__(self): + return '[north={:s}, south={:s}, east={:s}, west={:s}]'.format(str(self.north), str(self.south), str(self.east), str(self.west)) + + + def asCodedInt(self): + """ + Helper function which packs four boundary conditions into one integer + """ + bc = 0 + bc = bc | (self.north & 0x0000000F) << 24 + bc = bc | (self.south & 0x0000000F) << 16 + bc = bc | (self.east & 0x0000000F) << 8 + bc = bc | (self.west & 0x0000000F) << 0 + + #for t in types: + # print("{0:s}, {1:d}, {1:032b}, {1:08b}".format(t, types[t])) + #print("bc: {0:032b}".format(bc)) + + return np.int32(bc) + + def getTypes(bc): + types = {} + types['north'] = BoundaryCondition.Type((bc >> 24) & 0x0000000F) + types['south'] = BoundaryCondition.Type((bc >> 16) & 0x0000000F) + types['east'] = BoundaryCondition.Type((bc >> 8) & 0x0000000F) + types['west'] = BoundaryCondition.Type((bc >> 0) & 0x0000000F) + return types + + + + + + + + +class BaseSimulator(object): + + def __init__(self, + context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + num_substeps, + block_width, block_height): + """ + Initialization routine + context: GPU context to use + kernel_wrapper: wrapper function of GPU kernel + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + cfl_scale: Courant number + num_substeps: Number of substeps to perform for a full step + """ + #Get logger + self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__) + + #Save input parameters + #Notice that we need to specify them in the correct dataformat for the + #GPU kernel + self.context = context + self.nx = np.int32(nx) + self.ny = np.int32(ny) + self.dx = np.float32(dx) + self.dy = np.float32(dy) + self.setBoundaryConditions(boundary_conditions) + self.cfl_scale = cfl_scale + self.num_substeps = num_substeps + + #Handle autotuning block size + if (self.context.autotuner): + peak_configuration = self.context.autotuner.get_peak_performance(self.__class__) + block_width = int(peak_configuration["block_width"]) + block_height = int(peak_configuration["block_height"]) + self.logger.debug("Used autotuning to get block size [%d x %d]", block_width, block_height) + + #Compute kernel launch parameters + self.block_size = (block_width, block_height, 1) + self.grid_size = ( + int(np.ceil(self.nx / float(self.block_size[0]))), + int(np.ceil(self.ny / float(self.block_size[1]))) + ) + + #Create a CUDA stream + self.stream = cuda.Stream() + self.internal_stream = cuda.Stream() + + #Keep track of simulation time and number of timesteps + self.t = 0.0 + self.nt = 0 + + + def __str__(self): + return "{:s} [{:d}x{:d}]".format(self.__class__.__name__, self.nx, self.ny) + + + def simulate(self, t, dt=None): + """ + Function which simulates t_end seconds using the step function + Requires that the step() function is implemented in the subclasses + """ + + printer = Common.ProgressPrinter(t) + + t_start = self.simTime() + t_end = t_start + t + + update_dt = True + if (dt is not None): + update_dt = False + self.dt = dt + + while(self.simTime() < t_end): + # Update dt every 100 timesteps and cross your fingers it works + # for the next 100 + if (update_dt and (self.simSteps() % 100 == 0)): + self.dt = self.computeDt()*self.cfl_scale + + # Compute timestep for "this" iteration (i.e., shorten last timestep) + current_dt = np.float32(min(self.dt, t_end-self.simTime())) + + # Stop if end reached (should not happen) + if (current_dt <= 0.0): + self.logger.warning("Timestep size {:d} is less than or equal to zero!".format(self.simSteps())) + break + + # Step forward in time + self.step(current_dt) + + #Print info + print_string = printer.getPrintString(self.simTime() - t_start) + if (print_string): + self.logger.info("%s: %s", self, print_string) + try: + self.check() + except AssertionError as e: + e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),) + raise + + + def step(self, dt): + """ + Function which performs one single timestep of size dt + """ + for i in range(self.num_substeps): + self.substep(dt, i) + + self.t += dt + self.nt += 1 + + def download(self, variables=None): + return self.getOutput().download(self.stream, variables) + + def synchronize(self): + self.stream.synchronize() + + def simTime(self): + return self.t + + def simSteps(self): + return self.nt + + def getExtent(self): + return [0, 0, self.nx*self.dx, self.ny*self.dy] + + def setBoundaryConditions(self, boundary_conditions): + self.logger.debug("Boundary conditions set to {:s}".format(str(boundary_conditions))) + self.boundary_conditions = boundary_conditions.asCodedInt() + + def getBoundaryConditions(self): + return BoundaryCondition(BoundaryCondition.getTypes(self.boundary_conditions)) + + def substep(self, dt, step_number): + """ + Function which performs one single substep with stepsize dt + """ + raise(NotImplementedError("Needs to be implemented in subclass")) + + def getOutput(self): + raise(NotImplementedError("Needs to be implemented in subclass")) + + def check(self): + self.logger.warning("check() is not implemented - please implement") + #raise(NotImplementedError("Needs to be implemented in subclass")) + + def computeDt(self): + raise(NotImplementedError("Needs to be implemented in subclass")) + + + + + + + + + + + + + +def stepOrderToCodedInt(step, order): + """ + Helper function which packs the step and order into a single integer + """ + step_order = (step << 16) | (order & 0x0000ffff) + #print("Step: {0:032b}".format(step)) + #print("Order: {0:032b}".format(order)) + #print("Mix: {0:032b}".format(step_order)) + return np.int32(step_order) \ No newline at end of file diff --git a/GPUSimulators/WAF.py b/GPUSimulators/WAF.py new file mode 100644 index 0000000..22860fc --- /dev/null +++ b/GPUSimulators/WAF.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Weighted average flux (WAF) described in +E. Toro, Shock-Capturing methods for free-surface shallow flows, 2001 + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np + +from pycuda import gpuarray + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class WAF (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height); + self.g = np.float32(g) + + #Get kernels + module = context.get_module("cuda/SWE2D_WAF.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("WAFKernel") + self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + def computeDt(self): + max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5 \ No newline at end of file diff --git a/GPUSimulators/__init__.py b/GPUSimulators/__init__.py new file mode 100644 index 0000000..4e04e36 --- /dev/null +++ b/GPUSimulators/__init__.py @@ -0,0 +1,5 @@ +#!/bin/env python +# -*- coding: utf-8 -*- + + +# Nothing general to do diff --git a/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc b/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da4daacc64c69c69b772807cf18375b7c8cf127b GIT binary patch literal 11058 zcmb_iOK=-UdY%`6!4RYS*uv09;PGHib$-8;*6=r=nJH)=_{+;GFt zvX)xyE$LTlk$rZlXuop)!W;JL^2*wcoA!0T>PK$FN@ta=s&tz+7N|(iv%^*;+H|FN z$=+&p?6Mo!(i46dNq@Z)VcE!b1F?v80Eb`MvQW_pgeUE&=Gl=ao1xvRaN*X$jf zMA^RBUm%_BoZXVvSvMkprQK@Npdz5$vVoAjaSI1Gytgl5`vC)8YqhbB8fL@BHvLA! zUia)y=v6w6a~3dQzrVcp_Ivl%?5isu*zaGxd-v+f+6QlTqmMeD9iq6>~AiG?PqV}c5#m&vlg=)}QXvyj#PMS_RT)eVi zeFa`oR*GtzU+q&U3OY!(OfADd9fh~>GJkmr_(~~hLM`dyw9tj|SS=Z1PGm(6xr{JH z{;^WZ3QLUPn-c{wj;|>u#4&vHVp1H(*Ai3W1ioY9j5sM~FtQ*{iCNT)iwyW{KF$G@ zThbTXp=IN@MBHO<*8Fk}IBQEUWEKM(5SLZakx&=3gcmN@H{G(|053&eXnStB1=Kd1 zoxm?M;ev9aP0tH>-oOJduD7HQobqU{{R+&jlxgGrALUKFOL)T?l1S;Pk+!RL6-Vy^ z9rv|x{vAbVkJOLUu39&Qem@f~pvIt@DAUVQZCBgXFe2Bbab{Pk>%F|l^ep6Zc+L7) zSLqdYm0f*Tg@96kitLKaV7=`(?*(C}-EPSU)Dm=>>tG>lS@;k(#DCcLyPYs1zKAwk zHaLa2x7i|D^p#38oZ)sRJG3@{naFMh9`Aem;*#`$FfqXACHohbb{6crZg3wtu>*Py zfykB=et?eDS{#pajaIb^1$`PT?rQ( z*gZ(&!Xh}L?zYdG)m)pU3YfuPqY&EZ+13Kk|OoYIK_3tvmOen%`c;cfbhY zB4}z*T4-<4j%bYI{1D59lSmY`piQb%c(a)KObfGZ@fz6%)EP~&_L;8x$VjLsyX@-%p4})@^+RspO2)H-EDnCVANV6#F z6rK@~sr_h>X`_dM%-{lxryw?F6X28MKL9*oBiX?-lsgK{_q{N>h7JQT=SRV$_M^c( z0<)mW6tjKW9KtM7BxCkSxJ(e%3WQee)-Yhe-+*EI9gc?V2m}%t>O_V>pgMv;wCtU} zvU`6{> z1u(Opf+VSQ8q7|-c3+b(p+>2qZWZK<1xZe!yaVvN8U%xa&HM^|v>64uP?MkYmbEz^ zKdU^_b~4=zZNIC5zI62de6LJ058H#ePNz$A3iWktaN~`x-ZcoS1fI)EKSz)ijqPRD z-()e+@2*$wz&L&2HMU?Cxw2e?MG)0wt5dDnE{w>ETZV1!Z?j>LSQ^Pl+fL}3j+n&; zi-lq~R)g4Rc$G+!e2a~Buv95e=~@Sdf2>Qd`BaNF(Thu0oC92qK*uR$UU10#wJ?|W?tef&@tQOmayvp z_{e=IdvHwqWIqBWY3G4VARS)?d>>AdweqQg6EcMI7utpa7}Oyq0r@YSj}5W`pQ<1) ztjPx99xue!WkNn|zxI`X#P`^(bNmZ z3)=SdVZZ_nbf`rYo>jt-6EcsA*m9gCJ^-V*;5Zu{w~?C2ImZ#Lvg628=$BX~Y>GTf z#pft_o{|?RAqJFhQbOD^M2T#&Fptt{QQweCrfL{g!OEGsnMt1|E5~+t8OXQkfW*e9T(QUPQ#6&HZF zfo;8gkLhmMxvL?J(SVdb)MeWXry!aU!1t+CJPH4XofZ~hJl}|vA*7LwOdAZRaXdaA zp?(RC4p!iKG9>e(G#klf2S@FIJ;U1-F}e5Ca9REO8XJBGJ_cf3nuCh#7^J4yC1f5%QM zyB5hvlz1M@ew<8sX9@sqZFXo3YIw{n z)nm*o@cn$3jj!khSe4(~RqEe|sirtTh+YKk9H$zBqW(k3V{>2KS=8l|y1IqDmFCBg zAB&2;m%0{c$Lx-Y@ot_>xGHdmxqrlM=AmuplD6(x*Xrie{x7F>`JuW6KsT62XhNgD zWXU^=Xh~jy@>JT&6vLq@!DV8p`Eh=oH&{N%uhZ`;SE z10%j|A8#HQ@ooFKd|<@4?c=97DWi8qO!j^cxvBbF^{dgf-V$O;a12zOa_{=Cf-j^% z?*`>K%5;3CsKFM3eFKXT7Gk2#mmxDtCnBfW_MI96kB;}S3;~Uh&O2y{=MQOtHrosu zEmy>Ihqb~;rnct}YePs+r18+$N;-s**NzK;6CujwP$6X7w`d>&L{23@Cf6-CD3}!+ zo3O(ZlbEEvyohA`MQEEO0?(EZhq5S`;;&3BZVI$-kptoi2M!2-H`6aWCa4eN z+@?fK4jt0WWqbq zyO(@6lTVO*LV1jEHc1H?j!8(3iRUVc@i^`m8Uw12$HJ|k4EHg>(G&|woa>Ki7#SGy zeI%t5>nRmDppujfN~Z#E(>bg`tNJCaS}z&Zvh$GBE%^b>RYC$kChs_WZv^{r&&DH5 z@n7+Vze0jQBuDZnnm7Dda49LuN1hp_8r)1no&4I!nOauO!cXJJFt`;x{>#i3w0XFq zrkbQLaz07nlIqjyG}bB@^Jqt2pHunCzM}G@r(f+^tjlr5ftfdmiH;a%gk=gkEa45m zL;`WeQAMF;l+$Dc!6AAbcH3oYAy*J?A>BBL%oBqI*+^ZsU)M8OywO}aDEdR@>SfX% zT0Kj~R63VA$ir|W2t+bIy@M}9@vA~JE9br5%&*+6%r?W zLL~a=gy>WdiiV4$@jj%BhNo)645P7AZdxM{Xck4?o*6d&DYB6Wm7^CWp6oB0Xe#+j zRP~o8{K!9}WZ%Nn@cz)kvm*-+aq-bhkM4JP**JqZQu+R&3x9-#OG67!jV#RLMwTQd zARWUfUPp%Sm6RUrB-{;wly}vt%CP{Yj%Zi}q8A!sAJm?$=js&w(4u_Z68aOJ33;qn z5Jqnt@dEfGU>KyS=I1<|Fx)6{%$C$6F8#Dtd;2-|bZrsQtxsj=oLz0S)^R`Fujo6o zB+_M>k|rfg$9Zz&aR+@pl0T+OmW6b1asFzffm`ZM*-PAY{oUmiR-t=Rx9^mjkmzp1 z0ZHeRrIFo#sE;w{hlE=?8bzH&jAaJ0XdhJ#pc%n4dEw+6GoDGf5>+A2L@A<`h-gqQ zjW(f7_(d?4Js2JFM+hI(X5d>uXv&ylMZD$Zpx!DQIWC{$f zF89JSBtPJ~rq`CqjV_xggTY)%FTm-xY|v25C9KD)f?|rSFXYcD$LJ@PhsdFLN1WO8 z1x%!DjjlSAxFz4q#M;C2C2i|Gz87L`>%u;Cv(mr8FnWY^(veDL5=fuutBn1q2SX=?5wf5d^#xEKp_&4_hskD;x6)B%Euy2q)#-KR!#DVF-~@LuhObF8AbX__=a*vK zA9JMpW356?HI1@_LAqdr9H23t?bqL7Hn`4*&NiOT*2QdCw>L4+F68bXqFvIxW%ui_ zDu@a%2k{6hIIJadzw5{9ll9~H=J6s*k92Ap8VHvf`6r0+Q`j5im-IV<@AODN3;o8@ ze$R1k0Ui$QmRsnI z3;Xea7Cz%IG`Vu*xtv0;e$n*BoGAcy{$bjII& zhZ&zeV#enNGk$~_|G)Sk*{dPfM`-m7R+t;`!NifYYLE0gHbSfYT^sT@BXd8?_~`Qi zjX9hTo*(H)FdWSX#>PLP_y2|ulKp4o&xTHBRNC$Ffqn2~u<8pb-DWoC8Df$v8TmKg zVZP@No$q^t`5J>g{a^SXv+?hPc@FRa+trh)UH$ve5No)NGRyEU>7u2S#Xoo8ZN=F{ zQ6>iUyXYogq69h#b|5Zk;Pc0N}+u0!mtx)F|Wc|7857sr#3%MIGdO-gQ2a*7giZP>X;a8E|Jq&>;c zDPb3CJoTK*9vt}8duAu$A$4H~>22ix1ur{DxGT{M74sR~mC$3F6O!7ClgYAVqsQYT z{}M_fEg3jp_(vxh#&H%AqgPSTQDEXKjkmWbXt5w9E}DFvsTQM!QWO%hgX7W#!bN>yAR3TRY literal 0 HcmV?d00001 diff --git a/GPUSimulators/__pycache__/Simulator.cpython-39.pyc b/GPUSimulators/__pycache__/Simulator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc16706a34d26a4641268277e3f0d257f351d864 GIT binary patch literal 8433 zcmbVR+ix3JdY>DI_|HP)7M^Ga1=ngC*SX>AHAB74nu)jRTkTver>A3f zJ6|hg^NKbI- zHCho1x^01)>pM}Sg$1k`O0^rMuY3^%3QW`L_HgJxTX1VRFrN8JtIYq0xXLj;jvE{H%qQA=>P z(Q|0+3Z1@|oz7N5?5YsC%7lk;$zBN=0WtR3@PKT$t1OhxoEsCsl7$_@z5*zF%vTi2 z*3BPB_y8}?yg&in3Ohifh24PIw&(k7L$GcnnqB{@0}8OGtLyh3K3Qi=)o1MK(&NWV z)%9n8g_&R}M#U?U?n7J!>%bH#-5~A(X6M13$IJIHf9b2$`>X5E05)4$U9a9*TVpE^ zAG0O)XzB6#>hhEOOOM&3CyyUJT)Q(5j)5$|-}w{dBuQw8QZO!J*YhKw`V7pA0FuvH z%Y7xl#D?%*0U(z(AQnRszI{{2^}}FOA%^2gYEwZ0M5|2}gz;4t0kWH|IPTn7SlHg) zp5F|*^P${a0H=w|(ZZMW&KoG5R;}nsel>{iK!#DZm&0~D3>q{v{>!0p3s3Yj6hf=% z&{k+^&4gwd+=OPC!h)vdL~hF-f4KE)tEnq()NA=v3u}c;|B8Hqm!O9ZpM)MxC8xep zq9uFFVZc2Smi`@w;kQhhr?jdS`rHHZ8`(^dESs8+Z~EZ|ggP7cM5NXq!$%+D!;av? zjhe-Q2e)cA&jDf^kmdVlk3lxS-s_0|Kd9{qL?U1+om@l>sH8}Z1~joLBy0$au|R3y z39yg0f!tPCt9Ng(2T#`4*@LC^<$G-X-qJdIxB^s(H}fsFT4l>mZZC}%Rdh)RRrEL6 z3-(NVW1^^53Q3_}58Sq>*Au5+Pc0kjrFtFa$j?Ud^*Rq5=q}#&$fo%sPV!aJZM#8` z6du94!K5@2=doz|q7g$}ztdzH2dLurKVRrHqXoYUTO>Ny7a)&YZl~j`3Z~b8`1y|( z@%!TQg%&nnSn&cc7C#Gp*v-3-p43}jX94duuib@R2xYV|s?K+MiAAKoio+suCk5TI zES>)344M@y$pxW|TS+d0nzs^5xKW%~+aih^M2F#z>hvt*iF_0To$D90fmWxQp=vs6 zrm7jJS*m6Zv@I>RwjkIqjOZJ*^g}J^aIpEL4J0{OMI||jT5`G?#;Zf0CiopG zLs>BrD-wP)$tn3fV5&StOCUVNta|-(w4(DUw4$S{KVwQSeQy}0KN@D~e+;XC@wko* z7dJ>Qhd-+4@hC>jp~Ce&E!II;;}+Ic;anRG}dt&FUT&!`JP~O;8^33g#ki`AQqcqOd5$tZ>Fgr+Poo6e^WqF zu}>mlVg#KehfO~Fv|=b~CdNkNC`BDNS`NA3@HSO!6_rqEQ01BQ&+cD3PTqqh$0%%J zwW9DhsCb}ftnGouT>ooD<$a~gH?^$S8tcuCdhM~^{HV7u)?2&@>i}9iRmpIUe4h%X zs~=EJDeQ+-Q(8-8Q>qIx5PYeHG9-duOsI6-=e+bG}V1rV_4{uXjHGA^s|0G{i6eHHaH z39IV`Vl<2pEEYTQ4F(@h9*PZ5S#+>{qv_Wp z+>dc}U4}Elt&8r=z03v9c<_MYOGfj|}X{#TgfScUG(@XMAKXltnHC@GHUmXyW`loT^8gsja1l6ptDFHk7*xIsfCCY*7S<6@(`nK&sG z2*Jd{Rnt#$>ik-96LM0|`n%k%*=va3-F9LlrlI#V#RW2O!MwCXj{$#NDWwt9bwcb@ zDn6s)M^sRln4BI48+Fhiy~suKFKPG-Dn6#-6BM->1vn-(y-kP(MJ{9BNjWBRU?HA} zE&$s3^Eq2DWq*!t>$X|Y%c@;AOU5Pr(jRRb(Wzfcdw^qQq+D3@GNNSQJK8 z--5B+qR`*kH(?MFd6p^UWMBW{2H)MG;&~+}-@yvGK!tL}2ef#X`VRc8!i2o=Z}1GG z$umdcQ;}q~K|Xc(Bfn0Mg74$ZSyA}63)uFIeQnnoz)_vkc5?_a?E!+vt}Yk3Np+a> zY}O$6Li?32KN{F_1M94Pgq&xQBYR<}c8(mvmOV>u!Hn~Jg}A7&&!uOa*IvT`?&b%$ z4ZqOkKVi(-o8b99a(38jAMohd|K&$$PY&|R`I&oD>FR-X9y4|egMwUCE2lYJ;-Elo ztxr3Y_cdL6s=ZvawD{Do^xlWMwp+y5{Ge!QTgcT7&q?>&H=6p7ag(dh{1U64A=uk3 z{$26qV)|ax7O|7c>pA^jvyp9BIUE>A=~a{N=yefr`fnIt#`pP6}wyYO*(ByMt8X?Ksue%LLvJQ6y&D3*;iz@<^MJ^;EKt zt0NZHreK5A*03LB9W5_thDi>&(8h~oYAK3@B-K{;&BSu$CR{^mLP)OVJruQkCxxw+ zk6~?G8P{@(HML^5gKVRy^SI_Tx{^{UR4=LdWSHngT^!Lig%xEJ#BZ^CG><~F9iyNZ z3`<9RtJ}tuIjfh=i!dN(b<1?j8T6KQ1IDDRvt#CCC^3g}3PjzJ`M8Njtc=HN{kHbJ zybA@~LmmOqqoHdr5z|4O%|R->P)xKep0+F{ux9_>NYIYRYi8S#KEiE{WE9!yk+X&O zRr12)(Q@U#s+C-lgR_M>C+bxoSLqwq)h=qqvi6}(_GHCniat5oa?<42_onEb&%85Sg#*nzAz{BkTPLazc zEXXdAwc;yRdhQ0~|0rb7FzerFe>zM%#;oCPhc1nDIiu+ct|?StjQ|x{AFi!w@+rD- zG69fGM7^NVl3@U+DNi!yHO!&}>JWenM~HH;{1^-bhl0UIgA1U?`X6J* z9q}ERI{!IG5&KWXY!w6lId;?$H##&H$Icq*!6hulhZ&Xc0D3yPmDHWiYMW^qCrwvc zY4D)p1{Fb2@Ng1LN>c9Z#8Ca;jX`$qFwJ3v(mBVZkqE6Q{sUA}uEWn8F_iH&e3$Z9 zI(ahqzZ}T|`TO^W0og&)ErOp=Hya8CkP0D#C`s+pK!6ZWui1A{%Y;}`{iyi<7=8bR zCn7!3V1N3T594xTJa~-cBP}aqmpDsxY{6H*M#>_UXEV)1sC&PH)nSLDs@5K3s0v8jDN6J=G;BkP3Y$$zlGy4xlrt1G-=`y>Oe(@lCbJX1Z{&H0a zxNo>cd=on2f3aHm-YPPE=K#qeqc=g@)ziz!Q7%2fv>{dB z8CCfl`$O2SDJh7wM4J(T8a${8b2K|DdJ<8c~Y zWBAO8Soxq6va4kK{@Vb3hyi?)9RYg!u=uLOjsf{r!pLvvoVQZtrx-;*I6CYIRYpNe z1zrXJGe`4c2RX1cI)}WPBnKkNh)Xi9l8m1GnhH`)`Ex3W{fZ-!u$2VGn1m3aRSr+i zq1MI|QGj1C3O^}W&eekBn1$h=g_@1WQgaWPCt}=)c?2kBFuzn^ByLfb*iiXPsI=0W zs?wT2mFpdbr1U8m;UazgMaUO2cNjHL060wb$`5d$_x<8q z={0ZzohM#;K5oy2VXZbv1Gq@01EZOnoi2U9?bYmbX)>8@fp7Boten{CTi&9RCJ%jG zQbBQIhZ|e?HkH~(;-La|48;+LlfupPv)Px#sfezL))_>SddVoIrCq*T{(J4}{{j@a BH8B7H literal 0 HcmV?d00001 diff --git a/GPUSimulators/__pycache__/__init__.cpython-39.pyc b/GPUSimulators/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a966589566cf0036cb97a40facf080e9dc3106f1 GIT binary patch literal 182 zcmYe~<>g`kg7xe&sV+eJF^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5w0=Qav3^cz zaY<2XfuVjuQGQlpK|v0fk(yi*Z(?R@00fq1`WczY8TxLSd6^}tVfi_wxvA~}q461+ s1^PfbI5W32C$S{Is8~Nf9;75bUaz3?7Kcr4eoARhsvXGE&p^xo01!1T>Hq)$ literal 0 HcmV?d00001 diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu new file mode 100644 index 0000000..238718c --- /dev/null +++ b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu @@ -0,0 +1,250 @@ + /* +This kernel implements the Central Upwind flux function to +solve the Euler equations + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "EulerCommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float gamma_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + + + // Write to main memory for all internal cells + writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); + } +} + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip new file mode 100644 index 0000000..67b701b --- /dev/null +++ b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip @@ -0,0 +1,251 @@ +#include "hip/hip_runtime.h" + /* +This kernel implements the Central Upwind flux function to +solve the Euler equations + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "EulerCommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float gamma_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + + + // Write to main memory for all internal cells + writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); + } +} + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EulerCommon.h b/GPUSimulators/cuda/EulerCommon.h new file mode 100644 index 0000000..cb22a53 --- /dev/null +++ b/GPUSimulators/cuda/EulerCommon.h @@ -0,0 +1,187 @@ +/* +These CUDA functions implement different types of numerical flux +functions for the shallow water equations + +Copyright (C) 2016, 2017, 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once +#include "limiters.h" + + + +template +__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const float dx_, const float dy_, const float gamma_, + float* output_) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + tx; + const int tj = blockDim.y*blockIdx.y + ty; + + //Only internal cells + if (ti < nx_+gc_x && tj < ny_+gc_y) { + const float rho = Q[0][ty][tx]; + const float u = Q[1][ty][tx] / rho; + const float v = Q[2][ty][tx] / rho; + + const float max_u = dx_ / (fabsf(u) + sqrtf(gamma_*rho)); + const float max_v = dy_ / (fabsf(v) + sqrtf(gamma_*rho)); + + shmem[ty][tx] = fminf(max_u, max_v); + } + __syncthreads(); + + //One row of threads loop over all rows + if (ti < nx_+gc_x && tj < ny_+gc_y) { + if (ty == gc_y) { + float min_val = shmem[ty][tx]; + const int max_y = min(h, ny_+gc_y - tj); + for (int j=gc_y; j h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + + //Upwind selection + if (S_l >= 0.0f) { + return F_func(Q_l, P_l); + } + else if (S_r <= 0.0f) { + return F_func(Q_r, P_r); + } + //Or estimate flux in the star region + else { + const float4 F_l = F_func(Q_l, P_l); + const float4 F_r = F_func(Q_r, P_r); + const float4 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); + return flux; + } +} + + + + + + + +/** + * Central upwind flux function + */ +__device__ float4 CentralUpwindFlux(const float4 Qm, const float4 Qp, const float gamma) { + + const float Pp = pressure(Qp, gamma); + const float4 Fp = F_func(Qp, Pp); + const float up = Qp.y / Qp.x; // rho*u / rho + const float cp = sqrt(gamma*Pp/Qp.x); // sqrt(gamma*P/rho) + + const float Pm = pressure(Qm, gamma); + const float4 Fm = F_func(Qm, Pm); + const float um = Qm.y / Qm.x; // rho*u / rho + const float cm = sqrt(gamma*Pm/Qm.x); // sqrt(gamma*P/rho) + + const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed + const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed + + return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); +} \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu b/GPUSimulators/cuda/SWE2D_FORCE.cu new file mode 100644 index 0000000..dac46be --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_FORCE.cu @@ -0,0 +1,143 @@ +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_, const float dx_, const float dt_) { + //Compute fluxes along the x axis + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + //Compute flux along x, and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute flux along y, and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu.hip b/GPUSimulators/cuda/SWE2D_FORCE.cu.hip new file mode 100644 index 0000000..aa4e968 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_FORCE.cu.hip @@ -0,0 +1,144 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_, const float dx_, const float dt_) { + //Compute fluxes along the x axis + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + //Compute flux along x, and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute flux along y, and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu b/GPUSimulators/cuda/SWE2D_HLL.cu new file mode 100644 index 0000000..3ed6b35 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL.cu @@ -0,0 +1,161 @@ +/* +This GPU kernel implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute F flux + computeFluxF(Q, F, g_); + __syncthreads(); + + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute G flux + computeFluxG(Q, F, g_); + __syncthreads(); + + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu.hip b/GPUSimulators/cuda/SWE2D_HLL.cu.hip new file mode 100644 index 0000000..c2f449d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL.cu.hip @@ -0,0 +1,162 @@ +#include "hip/hip_runtime.h" +/* +This GPU kernel implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute F flux + computeFluxF(Q, F, g_); + __syncthreads(); + + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute G flux + computeFluxG(Q, F, g_); + __syncthreads(); + + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu b/GPUSimulators/cuda/SWE2D_HLL2.cu new file mode 100644 index 0000000..94f92b5 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL2.cu @@ -0,0 +1,216 @@ +/* +This OpenCL kernel implements the second order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu.hip b/GPUSimulators/cuda/SWE2D_HLL2.cu.hip new file mode 100644 index 0000000..c0bc9d1 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL2.cu.hip @@ -0,0 +1,217 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the second order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu b/GPUSimulators/cuda/SWE2D_KP07.cu new file mode 100644 index 0000000..6fa6154 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07.cu @@ -0,0 +1,233 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], + const float g_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + int j=ty; + const int l = j + 2; //Skip ghost cells + for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + + //Reconstruct slopes along x and axis + minmodSlopeX(Q, Qx, theta_); + minmodSlopeY(Q, Qy, theta_); + __syncthreads(); + + + //Compute fluxes along the x and y axis + computeFluxF(Q, Qx, F, g_); + computeFluxG(Q, Qy, G, g_); + __syncthreads(); + + + //Sum fluxes and advance in time for all internal cells + if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { + const int i = tx + 2; //Skip local ghost cells, i.e., +2 + const int j = ty + 2; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + + float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); + float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); + float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); + + if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { + //Write to main memory + h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); + hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); + hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); + } + else { + h_row[ti] = Q[0][j][i]; + hu_row[ti] = Q[1][j][i]; + hv_row[ti] = Q[2][j][i]; + } + } + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} +} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu.hip b/GPUSimulators/cuda/SWE2D_KP07.cu.hip new file mode 100644 index 0000000..fd9ef0d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07.cu.hip @@ -0,0 +1,234 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], + const float g_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + int j=ty; + const int l = j + 2; //Skip ghost cells + for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + + //Reconstruct slopes along x and axis + minmodSlopeX(Q, Qx, theta_); + minmodSlopeY(Q, Qy, theta_); + __syncthreads(); + + + //Compute fluxes along the x and y axis + computeFluxF(Q, Qx, F, g_); + computeFluxG(Q, Qy, G, g_); + __syncthreads(); + + + //Sum fluxes and advance in time for all internal cells + if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { + const int i = tx + 2; //Skip local ghost cells, i.e., +2 + const int j = ty + 2; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + + float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); + float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); + float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); + + if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { + //Write to main memory + h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); + hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); + hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); + } + else { + h_row[ti] = Q[0][j][i]; + hu_row[ti] = Q[1][j][i]; + hv_row[ti] = Q[2][j][i]; + } + } + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} +} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu new file mode 100644 index 0000000..ac256e3 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu @@ -0,0 +1,216 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +template +__device__ +void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], + float Qx[3][h+2*gc_y][w+2*gc_x], + float F[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j +__device__ +void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], + float Qy[3][h+2*gc_y][w+2*gc_x], + float G[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dy_, const float dt_) { + for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + if (step_ == 0) { + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + else { + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + + + + + + + + + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip new file mode 100644 index 0000000..f366b0a --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip @@ -0,0 +1,217 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +template +__device__ +void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], + float Qx[3][h+2*gc_y][w+2*gc_x], + float F[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j +__device__ +void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], + float Qy[3][h+2*gc_y][w+2*gc_x], + float G[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dy_, const float dt_) { + for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + if (step_ == 0) { + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + else { + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + + + + + + + + + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu b/GPUSimulators/cuda/SWE2D_LxF.cu new file mode 100644 index 0000000..1f197fd --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_LxF.cu @@ -0,0 +1,168 @@ +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +template +__device__ +void computeFluxF(float Q[3][block_height+2][block_width+2], + float F[3][block_height][block_width+1], + const float g_, const float dx_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + const int j=ty; + const int l = j + 1; //Skip ghost cells + for (int i=tx; i +__device__ +void computeFluxG(float Q[3][block_height+2][block_width+2], + float G[3][block_height+1][block_width], + const float g_, const float dy_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute fluxes along the x and y axis + computeFluxF(Q, F, g_, dx_, dt_); + computeFluxG(Q, G, g_, dy_, dt_); + __syncthreads(); + + //Evolve for all cells + const int tx = threadIdx.x; + const int ty = threadIdx.y; + const int i = tx + 1; //Skip local ghost cells, i.e., +1 + const int j = ty + 1; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" + diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu.hip b/GPUSimulators/cuda/SWE2D_LxF.cu.hip new file mode 100644 index 0000000..588d691 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_LxF.cu.hip @@ -0,0 +1,169 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +template +__device__ +void computeFluxF(float Q[3][block_height+2][block_width+2], + float F[3][block_height][block_width+1], + const float g_, const float dx_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + const int j=ty; + const int l = j + 1; //Skip ghost cells + for (int i=tx; i +__device__ +void computeFluxG(float Q[3][block_height+2][block_width+2], + float G[3][block_height+1][block_width], + const float g_, const float dy_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute fluxes along the x and y axis + computeFluxF(Q, F, g_, dx_, dt_); + computeFluxG(Q, G, g_, dy_, dt_); + __syncthreads(); + + //Evolve for all cells + const int tx = threadIdx.x; + const int ty = threadIdx.y; + const int i = tx + 1; //Skip local ghost cells, i.e., +1 + const int j = ty + 1; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" + diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu b/GPUSimulators/cuda/SWE2D_WAF.cu new file mode 100644 index 0000000..2c38cdf --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_WAF.cu @@ -0,0 +1,178 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu.hip b/GPUSimulators/cuda/SWE2D_WAF.cu.hip new file mode 100644 index 0000000..ddfad9d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_WAF.cu.hip @@ -0,0 +1,179 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWECommon.h b/GPUSimulators/cuda/SWECommon.h new file mode 100644 index 0000000..52f8b31 --- /dev/null +++ b/GPUSimulators/cuda/SWECommon.h @@ -0,0 +1,533 @@ +/* +These CUDA functions implement different types of numerical flux +functions for the shallow water equations + +Copyright (C) 2016, 2017, 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once +#include "limiters.h" + + + + + + +__device__ float3 F_func(const float3 Q, const float g) { + float3 F; + + F.x = Q.y; //hu + F.y = Q.y*Q.y / Q.x + 0.5f*g*Q.x*Q.x; //hu*hu/h + 0.5f*g*h*h; + F.z = Q.y*Q.z / Q.x; //hu*hv/h; + + return F; +} + + + + + + + + + + + + + +/** + * Superbee flux limiter for WAF. + * Related to superbee limiter so that WAF_superbee(r, c) = 1 - (1-|c|)*superbee(r) + * @param r_ the ratio of upwind change (see Toro 2001, p. 203/204) + * @param c_ the courant number for wave k, dt*S_k/dx + */ +__device__ float WAF_superbee(float r_, float c_) { + // r <= 0.0 + if (r_ <= 0.0f) { + return 1.0f; + } + // 0.0 <= r <= 1/2 + else if (r_ <= 0.5f) { + return 1.0f - 2.0f*(1.0f - fabsf(c_))*r_; + } + // 1/2 <= r <= 1 + else if (r_ <= 1.0f) { + return fabs(c_); + } + // 1 <= r <= 2 + else if (r_ <= 2.0f) { + return 1.0f - (1.0f - fabsf(c_))*r_; + } + // r >= 2 + else { + return 2.0f*fabsf(c_) - 1.0f; + } +} + + + + +__device__ float WAF_albada(float r_, float c_) { + if (r_ <= 0.0f) { + return 1.0f; + } + else { + return 1.0f - (1.0f - fabsf(c_)) * r_ * (1.0f + r_) / (1.0f + r_*r_); + } +} + +__device__ float WAF_minbee(float r_, float c_) { + r_ = fmaxf(-1.0f, fminf(2.0f, r_)); + if (r_ <= 0.0f) { + return 1.0f; + } + if (r_ >= 0.0f && r_ <= 1.0f) { + return 1.0f - (1.0f - fabsf(c_)) * r_; + } + else { + return fabsf(c_); + } +} + +__device__ float WAF_minmod(float r_, float c_) { + return 1.0f - (1.0f - fabsf(c_)) * fmaxf(0.0f, fminf(1.0f, r_)); +} + +__device__ float limiterToWAFLimiter(float r_, float c_) { + return 1.0f - (1.0f - fabsf(c_))*r_; +} + +// Compute h in the "star region", h^dagger +__device__ __inline__ float computeHStar(float h_l, float h_r, float u_l, float u_r, float c_l, float c_r, float g_) { + + //This estimate for the h* gives rise to spurious oscillations. + //return 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float h_tmp = 0.5f * (c_l + c_r) + 0.25f * (u_l - u_r); + return h_tmp*h_tmp / g_; +} + +/** + * Weighted average flux (Toro 2001, p 200) for interface {i+1/2} + * @param r_ The flux limiter parameter (see Toro 2001, p. 203) + * @param Q_l2 Q_{i-1} + * @param Q_l1 Q_{i} + * @param Q_r1 Q_{i+1} + * @param Q_r2 Q_{i+2} + */ +__device__ float3 WAF_1D_flux(const float3 Q_l2, const float3 Q_l1, const float3 Q_r1, const float3 Q_r2, const float g_, const float dx_, const float dt_) { + const float h_l = Q_l1.x; + const float h_r = Q_r1.x; + + const float h_l2 = Q_l2.x; + const float h_r2 = Q_r2.x; + + // Calculate velocities + const float u_l = Q_l1.y / h_l; + const float u_r = Q_r1.y / h_r; + + const float u_l2 = Q_l2.y / h_l2; + const float u_r2 = Q_r2.y / h_r2; + + const float v_l = Q_l1.z / h_l; + const float v_r = Q_r1.z / h_r; + + const float v_l2 = Q_l2.z / h_l2; + const float v_r2 = Q_r2.z / h_r2; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + const float c_l2 = sqrt(g_*h_l2); + const float c_r2 = sqrt(g_*h_r2); + + // Compute h in the "star region", h^dagger + const float h_dag_l = computeHStar(h_l2, h_l, u_l2, u_l, c_l2, c_l, g_); + const float h_dag = computeHStar( h_l, h_r, u_l, u_r, c_l, c_r, g_); + const float h_dag_r = computeHStar( h_r, h_r2, u_r, u_r2, c_r, c_r2, g_); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag ) ) / h_l; + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag ) ) / h_r; + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); + + const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1.0, S_star, v_l); + const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1.0, S_star, v_r); + + // Estimate the fluxes in the four regions + const float3 F_1 = F_func(Q_l1, g_); + const float3 F_4 = F_func(Q_r1, g_); + + const float3 F_2 = F_1 + S_l*(Q_star_l - Q_l1); + const float3 F_3 = F_4 + S_r*(Q_star_r - Q_r1); + //const float3 F_2 = F_func(Q_star_l, g_); + //const float3 F_3 = F_func(Q_star_r, g_); + + // Compute the courant numbers for the waves + const float c_1 = S_l * dt_ / dx_; + const float c_2 = S_star * dt_ / dx_; + const float c_3 = S_r * dt_ / dx_; + + // Compute the "upwind change" vectors for the i-3/2 and i+3/2 interfaces + const float eps = 1.0e-6f; + const float r_1 = desingularize( (c_1 > 0.0f) ? (h_dag_l - h_l2) : (h_dag_r - h_r), eps) / desingularize((h_dag - h_l), eps); + const float r_2 = desingularize( (c_2 > 0.0f) ? (v_l - v_l2) : (v_r2 - v_r), eps ) / desingularize((v_r - v_l), eps); + const float r_3 = desingularize( (c_3 > 0.0f) ? (h_l - h_dag_l) : (h_r2 - h_dag_r), eps ) / desingularize((h_r - h_dag), eps); + + // Compute the limiter + // We use h for the nonlinear waves, and v for the middle shear wave + const float A_1 = copysign(1.0f, c_1) * limiterToWAFLimiter(generalized_minmod(r_1, 1.9f), c_1); + const float A_2 = copysign(1.0f, c_2) * limiterToWAFLimiter(generalized_minmod(r_2, 1.9f), c_2); + const float A_3 = copysign(1.0f, c_3) * limiterToWAFLimiter(generalized_minmod(r_3, 1.9f), c_3); + + //Average the fluxes + const float3 flux = 0.5f*( F_1 + F_4 ) + - 0.5f*( A_1 * (F_2 - F_1) + + A_2 * (F_3 - F_2) + + A_3 * (F_4 - F_3) ); + + return flux; +} + + + + + + + + + + + + + + +/** + * Central upwind flux function + */ +__device__ float3 CentralUpwindFlux(const float3 Qm, float3 Qp, const float g) { + const float3 Fp = F_func(Qp, g); + const float up = Qp.y / Qp.x; // hu / h + const float cp = sqrt(g*Qp.x); // sqrt(g*h) + + const float3 Fm = F_func(Qm, g); + const float um = Qm.y / Qm.x; // hu / h + const float cm = sqrt(g*Qm.x); // sqrt(g*h) + + const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed + const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed + + return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); +} + + + + + + + + + +/** + * Godunovs centered scheme (Toro 2001, p 165) + */ +__device__ float3 GodC_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + const float3 Q_godc = 0.5f*(Q_l + Q_r) + (dt_/dx_)*(F_l - F_r); + + return F_func(Q_godc, g_); +} + + + + + + + + + +/** + * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 180) + */ +__device__ float3 HLL_flux(const float3 Q_l, const float3 Q_r, const float g_) { + const float h_l = Q_l.x; + const float h_r = Q_r.x; + + // Calculate velocities + const float u_l = Q_l.y / h_l; + const float u_r = Q_r.y / h_r; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + // Compute h in the "star region", h^dagger + const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + + //Upwind selection + if (S_l >= 0.0f) { + return F_func(Q_l, g_); + } + else if (S_r <= 0.0f) { + return F_func(Q_r, g_); + } + //Or estimate flux in the star region + else { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + const float3 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); + return flux; + } +} + + + + + + + + + + + + + + + + + +/** + * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 181) + */ +__device__ float3 HLLC_flux(const float3 Q_l, const float3 Q_r, const float g_) { + const float h_l = Q_l.x; + const float h_r = Q_r.x; + + // Calculate velocities + const float u_l = Q_l.y / h_l; + const float u_r = Q_r.y / h_r; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + // Compute h in the "star region", h^dagger + const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); + + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + //Upwind selection + if (S_l >= 0.0f) { + return F_l; + } + else if (S_r <= 0.0f) { + return F_r; + } + //Or estimate flux in the "left star" region + else if (S_l <= 0.0f && 0.0f <=S_star) { + const float v_l = Q_l.z / h_l; + const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1, S_star, v_l); + const float3 flux = F_l + S_l*(Q_star_l - Q_l); + return flux; + } + //Or estimate flux in the "righ star" region + else if (S_star <= 0.0f && 0.0f <=S_r) { + const float v_r = Q_r.z / h_r; + const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1, S_star, v_r); + const float3 flux = F_r + S_r*(Q_star_r - Q_r); + return flux; + } + else { + return make_float3(-99999.9f, -99999.9f, -99999.9f); //Something wrong here + } +} + + + + + + + + + + + + + +/** + * Lax-Friedrichs flux (Toro 2001, p 163) + */ +__device__ float3 LxF_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + return 0.5f*(F_l + F_r) + (dx_/(2.0f*dt_))*(Q_l - Q_r); +} + + + + + + + + +/** + * Lax-Friedrichs extended to 2D + */ +__device__ float3 LxF_2D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + //Note numerical diffusion for 2D here (0.25) + return 0.5f*(F_l + F_r) + (dx_/(4.0f*dt_))*(Q_l - Q_r); +} + + + + + + + + + + + + +/** + * Richtmeyer / Two-step Lax-Wendroff flux (Toro 2001, p 164) + */ +__device__ float3 LxW2_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + const float3 Q_lw2 = 0.5f*(Q_l + Q_r) + (dt_/(2.0f*dx_))*(F_l - F_r); + + return F_func(Q_lw2, g_); +} + + + + + + + + + + + + +/** + * First Ordered Centered (Toro 2001, p.163) + */ +__device__ float3 FORCE_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_lf = LxF_1D_flux(Q_l, Q_r, g_, dx_, dt_); + const float3 F_lw2 = LxW2_1D_flux(Q_l, Q_r, g_, dx_, dt_); + return 0.5f*(F_lf + F_lw2); +} + + + + + + + + + + +template +__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const float dx_, const float dy_, const float g_, + float* output_) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + tx; + const int tj = blockDim.y*blockIdx.y + ty; + + //Only internal cells + if (ti < nx_+gc_x && tj < ny_+gc_y) { + const float h = Q[0][ty][tx]; + const float u = Q[1][ty][tx] / h; + const float v = Q[2][ty][tx] / h; + + const float max_u = dx_ / (fabsf(u) + sqrtf(g_*h)); + const float max_v = dy_ / (fabsf(v) + sqrtf(g_*h)); + + shmem[ty][tx] = fminf(max_u, max_v); + } + __syncthreads(); + + //One row of threads loop over all rows + if (ti < nx_+gc_x && tj < ny_+gc_y) { + if (ty == gc_y) { + float min_val = shmem[ty][tx]; + const int max_y = min(h, ny_+gc_y - tj); + for (int j=gc_y; j. +*/ + +#pragma once + + +/** + * Float3 operators + */ +inline __device__ float3 operator*(const float a, const float3 b) { + return make_float3(a*b.x, a*b.y, a*b.z); +} + +inline __device__ float3 operator/(const float3 a, const float b) { + return make_float3(a.x/b, a.y/b, a.z/b); +} + +inline __device__ float3 operator-(const float3 a, const float3 b) { + return make_float3(a.x-b.x, a.y-b.y, a.z-b.z); +} + +inline __device__ float3 operator+(const float3 a, const float3 b) { + return make_float3(a.x+b.x, a.y+b.y, a.z+b.z); +} + +/** + * Float4 operators + */ +inline __device__ float4 operator*(const float a, const float4 b) { + return make_float4(a*b.x, a*b.y, a*b.z, a*b.w); +} + +inline __device__ float4 operator/(const float4 a, const float b) { + return make_float4(a.x/b, a.y/b, a.z/b, a.w/b); +} + +inline __device__ float4 operator-(const float4 a, const float4 b) { + return make_float4(a.x-b.x, a.y-b.y, a.z-b.z, a.w-b.w); +} + +inline __device__ float4 operator+(const float4 a, const float4 b) { + return make_float4(a.x+b.x, a.y+b.y, a.z+b.z, a.w+b.w); +} + + + + +inline __device__ __host__ float clamp(const float f, const float a, const float b) { + return fmaxf(a, fminf(f, b)); +} + +inline __device__ __host__ int clamp(const int f, const int a, const int b) { + return (f < b) ? ( (f > a) ? f : a) : b; +} + + + + + +__device__ float desingularize(float x_, float eps_) { + return copysign(1.0f, x_)*fmaxf(fabsf(x_), fminf(x_*x_/(2.0f*eps_)+0.5f*eps_, eps_)); +} + + + + + + + + +/** + * Returns the step stored in the leftmost 16 bits + * of the 32 bit step-order integer + */ +inline __device__ int getStep(int step_order_) { + return step_order_ >> 16; +} + +/** + * Returns the order stored in the rightmost 16 bits + * of the 32 bit step-order integer + */ +inline __device__ int getOrder(int step_order_) { + return step_order_ & 0x0000FFFF; +} + + +enum BoundaryCondition { + Dirichlet = 0, + Neumann = 1, + Periodic = 2, + Reflective = 3 +}; + +inline __device__ BoundaryCondition getBCNorth(int bc_) { + return static_cast((bc_ >> 24) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCSouth(int bc_) { + return static_cast((bc_ >> 16) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCEast(int bc_) { + return static_cast((bc_ >> 8) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCWest(int bc_) { + return static_cast((bc_ >> 0) & 0x0000000F); +} + + +// West boundary +template +__device__ void bcWestReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int j=threadIdx.y; j= 1 && ti == gc_x) { + Q[j][i-1] = sign*Q[j][i]; + } + if (gc_x >= 2 && ti == gc_x + 1) { + Q[j][i-3] = sign*Q[j][i]; + } + if (gc_x >= 3 && ti == gc_x + 2) { + Q[j][i-5] = sign*Q[j][i]; + } + if (gc_x >= 4 && ti == gc_x + 3) { + Q[j][i-7] = sign*Q[j][i]; + } + if (gc_x >= 5 && ti == gc_x + 4) { + Q[j][i-9] = sign*Q[j][i]; + } + } +} + + +// East boundary +template +__device__ void bcEastReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int j=threadIdx.y; j= 1 && ti == nx_ + gc_x - 1) { + Q[j][i+1] = sign*Q[j][i]; + } + if (gc_x >= 2 && ti == nx_ + gc_x - 2) { + Q[j][i+3] = sign*Q[j][i]; + } + if (gc_x >= 3 && ti == nx_ + gc_x - 3) { + Q[j][i+5] = sign*Q[j][i]; + } + if (gc_x >= 4 && ti == nx_ + gc_x - 4) { + Q[j][i+7] = sign*Q[j][i]; + } + if (gc_x >= 5 && ti == nx_ + gc_x - 5) { + Q[j][i+9] = sign*Q[j][i]; + } + } +} + + +// South boundary +template +__device__ void bcSouthReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int i=threadIdx.x; i= 1 && tj == gc_y) { + Q[j-1][i] = sign*Q[j][i]; + } + if (gc_y >= 2 && tj == gc_y + 1) { + Q[j-3][i] = sign*Q[j][i]; + } + if (gc_y >= 3 && tj == gc_y + 2) { + Q[j-5][i] = sign*Q[j][i]; + } + if (gc_y >= 4 && tj == gc_y + 3) { + Q[j-7][i] = sign*Q[j][i]; + } + if (gc_y >= 5 && tj == gc_y + 4) { + Q[j-9][i] = sign*Q[j][i]; + } + } +} + + + + +// North boundary +template +__device__ void bcNorthReflective(float Q[h+2*gc_y][w+2*gc_x], const int nx_, const int ny_) { + for (int i=threadIdx.x; i= 1 && tj == ny_ + gc_y - 1) { + Q[j+1][i] = sign*Q[j][i]; + } + if (gc_y >= 2 && tj == ny_ + gc_y - 2) { + Q[j+3][i] = sign*Q[j][i]; + } + if (gc_y >= 3 && tj == ny_ + gc_y - 3) { + Q[j+5][i] = sign*Q[j][i]; + } + if (gc_y >= 4 && tj == ny_ + gc_y - 4) { + Q[j+7][i] = sign*Q[j][i]; + } + if (gc_y >= 5 && tj == ny_ + gc_y - 5) { + Q[j+9][i] = sign*Q[j][i]; + } + } +} + + + + +/** + * Alter the index l so that it gives periodic boundary conditions when reading + */ +template +inline __device__ int handlePeriodicBoundaryX(int k, int nx_, int boundary_conditions_) { + const int gc_pad = gc_x; + + //West boundary: add an offset to read from east of domain + if (gc_x > 0) { + if ((k < gc_pad) + && getBCWest(boundary_conditions_) == Periodic) { + k += (nx_+2*gc_x - 2*gc_pad); + } + //East boundary: subtract an offset to read from west of domain + else if ((k >= nx_+2*gc_x-gc_pad) + && getBCEast(boundary_conditions_) == Periodic) { + k -= (nx_+2*gc_x - 2*gc_pad); + } + } + + return k; +} + +/** + * Alter the index l so that it gives periodic boundary conditions when reading + */ +template +inline __device__ int handlePeriodicBoundaryY(int l, int ny_, int boundary_conditions_) { + const int gc_pad = gc_y; + + //South boundary: add an offset to read from north of domain + if (gc_y > 0) { + if ((l < gc_pad) + && getBCSouth(boundary_conditions_) == Periodic) { + l += (ny_+2*gc_y - 2*gc_pad); + } + //North boundary: subtract an offset to read from south of domain + else if ((l >= ny_+2*gc_y-gc_pad) + && getBCNorth(boundary_conditions_) == Periodic) { + l -= (ny_+2*gc_y - 2*gc_pad); + } + } + + return l; +} + + +template +inline __device__ +void handleReflectiveBoundary( + float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const int boundary_conditions_) { + + //Handle reflective boundary conditions + if (getBCNorth(boundary_conditions_) == Reflective) { + bcNorthReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCSouth(boundary_conditions_) == Reflective) { + bcSouthReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCEast(boundary_conditions_) == Reflective) { + bcEastReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCWest(boundary_conditions_) == Reflective) { + bcWestReflective(Q, nx_, ny_); + __syncthreads(); + } +} + +/** + * Reads a block of data with ghost cells + */ +template +inline __device__ void readBlock(float* ptr_, int pitch_, + float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const int boundary_conditions_, + int x0, int y0, + int x1, int y1) { + //Index of block within domain + const int bx = blockDim.x * blockIdx.x; + const int by = blockDim.y * blockIdx.y; + + //Read into shared memory + //Loop over all variables + for (int j=threadIdx.y; j(by + j + y0, ny_, boundary_conditions_); + l = min(l, min(ny_+2*gc_y-1, y1+2*gc_y-1)); + float* row = (float*) ((char*) ptr_ + pitch_*l); + + for (int i=threadIdx.x; i(bx + i + x0, nx_, boundary_conditions_); + k = min(k, min(nx_+2*gc_x-1, x1+2*gc_x-1)); + + //Read from global memory + Q[j][i] = row[k]; + } + } + __syncthreads(); + + handleReflectiveBoundary(Q, nx_, ny_, boundary_conditions_); +} + + + + +/** + * Writes a block of data to global memory for the shallow water equations. + */ +template +inline __device__ void writeBlock(float* ptr_, int pitch_, + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + int rk_step_, int rk_order_, + int x0, int y0, + int x1, int y1) { + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + threadIdx.x + gc_x + x0; + const int tj = blockDim.y*blockIdx.y + threadIdx.y + gc_y + y0; + + //In case we are writing only to a subarea given by (x0, y0) x (x1, y1) + const int max_ti = min(nx_+gc_x, x1+gc_x); + const int max_tj = min(ny_+gc_y, y1+gc_y); + + //Only write internal cells + if ((x0+gc_x <= ti) && (ti < max_ti) && (y0+gc_y <= tj) && (tj < max_tj)) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + float* const row = (float*) ((char*) ptr_ + pitch_*tj); + + //Handle runge-kutta timestepping here + row[ti] = shmem[ty][tx]; + + + + /** + * SSPRK1 (forward Euler) + * u^1 = u^n + dt*f(u^n) + */ + if (rk_order_ == 1) { + row[ti] = shmem[ty][tx]; + } + /** + * SSPRK2 + * u^1 = u^n + dt*f(u^n) + * u^n+1 = 1/2*u^n + 1/2*(u^1 + dt*f(u^1)) + */ + else if (rk_order_ == 2) { + if (rk_step_ == 0) { + row[ti] = shmem[ty][tx]; + } + else if (rk_step_ == 1) { + row[ti] = 0.5f*row[ti] + 0.5f*shmem[ty][tx]; + } + } + /** + * SSPRK3 + * u^1 = u^n + dt*f(u^n) + * u^2 = 3/4 * u^n + 1/4 * (u^1 + dt*f(u^1)) + * u^n+1 = 1/3 * u^n + 2/3 * (u^2 + dt*f(u^2)) + * FIXME: This is not correct now, need a temporary to hold intermediate step u^2 + */ + else if (rk_order_ == 3) { + if (rk_step_ == 0) { + row[ti] = shmem[ty][tx]; + } + else if (rk_step_ == 1) { + row[ti] = 0.75f*row[ti] + 0.25f*shmem[ty][tx]; + } + else if (rk_step_ == 2) { + const float t = 1.0f / 3.0f; //Not representable in base 2 + row[ti] = t*row[ti] + (1.0f-t)*shmem[ty][tx]; + } + } + + // DEBUG + //row[ti] = 99.0; + } +} + + + + + + + + + + + +template +__device__ void evolveF(float Q[vars][h+2*gc_y][w+2*gc_x], + float F[vars][h+2*gc_y][w+2*gc_x], + const float dx_, const float dt_) { + for (int var=0; var < vars; ++var) { + for (int j=threadIdx.y; j +__device__ void evolveG(float Q[vars][h+2*gc_y][w+2*gc_x], + float G[vars][h+2*gc_y][w+2*gc_x], + const float dy_, const float dt_) { + for (int var=0; var < vars; ++var) { + for (int j=threadIdx.y+gc_y; j +__device__ void memset(float Q[vars][shmem_height][shmem_width], float value) { + for (int k=0; k +__device__ void reduce_max(float* data, unsigned int n) { + __shared__ float sdata[threads]; + unsigned int tid = threadIdx.x; + + //Reduce to "threads" elements + sdata[tid] = FLT_MIN; + for (unsigned int i=tid; i= 512) { + if (tid < 256) { + sdata[tid] = max(sdata[tid], sdata[tid + 256]); + } + __syncthreads(); + } + if (threads >= 256) { + if (tid < 128) { + sdata[tid] = max(sdata[tid], sdata[tid + 128]); + } + __syncthreads(); + } + if (threads >= 128) { + if (tid < 64) { + sdata[tid] = max(sdata[tid], sdata[tid + 64]); + } + __syncthreads(); + } + if (tid < 32) { + volatile float* sdata_volatile = sdata; + if (threads >= 64) { + sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 32]); + } + if (tid < 16) { + if (threads >= 32) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 16]); + if (threads >= 16) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 8]); + if (threads >= 8) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 4]); + if (threads >= 4) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 2]); + if (threads >= 2) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 1]); + } + + if (tid == 0) { + return sdata_volatile[0]; + } + } +} + + + + + + + + + + diff --git a/GPUSimulators/cuda/limiters.h b/GPUSimulators/cuda/limiters.h new file mode 100644 index 0000000..c2effa7 --- /dev/null +++ b/GPUSimulators/cuda/limiters.h @@ -0,0 +1,118 @@ +/* +This file implements different flux and slope limiters + +Copyright (C) 2016, 2017, 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + + + + + + +/** + * Reconstructs a slope using the generalized minmod limiter based on three + * consecutive values + */ +__device__ __inline__ float minmodSlope(float left, float center, float right, float theta) { + const float backward = (center - left) * theta; + const float central = (right - left) * 0.5f; + const float forward = (right - center) * theta; + + return 0.25f + *copysign(1.0f, backward) + *(copysign(1.0f, backward) + copysign(1.0f, central)) + *(copysign(1.0f, central) + copysign(1.0f, forward)) + *min( min(fabs(backward), fabs(central)), fabs(forward) ); +} + + + + +/** + * Reconstructs a minmod slope for a whole block along the abscissa + */ +template +__device__ void minmodSlopeX(float Q[vars][h+2*gc_y][w+2*gc_x], + float Qx[vars][h+2*gc_y][w+2*gc_x], + const float theta_) { + //Reconstruct slopes along x axis + for (int p=0; p +__device__ void minmodSlopeY(float Q[vars][h+2*gc_y][w+2*gc_x], + float Qy[vars][h+2*gc_y][w+2*gc_x], + const float theta_) { + //Reconstruct slopes along y axis + for (int p=0; p. +""" + + +from GPUSimulators.Simulator import BoundaryCondition +import numpy as np +import gc + + +def getExtent(width, height, nx, ny, grid, index=None): + if grid is not None: + gx = grid.grid[0] + gy = grid.grid[1] + if index is not None: + i, j = grid.getCoordinate(index) + else: + i, j = grid.getCoordinate() + + dx = (width / gx) / nx + dy = (height / gy) / ny + + x0 = width*i/gx + 0.5*dx + y0 = height*j/gy + 0.5*dy + x1 = width*(i+1)/gx - 0.5*dx + y1 = height*(j+1)/gy - 0.5*dx + + else: + dx = width / nx + dy = height / ny + + x0 = 0.5*dx + y0 = 0.5*dy + x1 = width-0.5*dx + y1 = height-0.5*dy + + return [x0, x1, y0, y1, dx, dy] + + +def downsample(highres_solution, x_factor, y_factor=None): + if (y_factor == None): + y_factor = x_factor + + assert(highres_solution.shape[1] % x_factor == 0) + assert(highres_solution.shape[0] % y_factor == 0) + + if (x_factor*y_factor == 1): + return highres_solution + + if (len(highres_solution.shape) == 1): + highres_solution = highres_solution.reshape((1, highres_solution.size)) + + nx = highres_solution.shape[1] / x_factor + ny = highres_solution.shape[0] / y_factor + + return highres_solution.reshape([int(ny), int(y_factor), int(nx), int(x_factor)]).mean(3).mean(1) + + + + + +def bump(nx, ny, width, height, + bump_size=None, + ref_nx=None, ref_ny=None, + x_center=0.5, y_center=0.5, + h_ref=0.5, h_amp=0.1, u_ref=0.0, u_amp=0.1, v_ref=0.0, v_amp=0.1): + + if (ref_nx == None): + ref_nx = nx + assert(ref_nx >= nx) + + if (ref_ny == None): + ref_ny = ny + assert(ref_ny >= ny) + + if (bump_size == None): + bump_size = width/5.0 + + ref_dx = width / float(ref_nx) + ref_dy = height / float(ref_ny) + + x_center = ref_dx*ref_nx*x_center + y_center = ref_dy*ref_ny*y_center + + x = ref_dx*(np.arange(0, ref_nx, dtype=np.float32)+0.5) - x_center + y = ref_dy*(np.arange(0, ref_ny, dtype=np.float32)+0.5) - y_center + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + r = np.sqrt(xv**2 + yv**2) + xv = None + yv = None + gc.collect() + + #Generate highres then downsample + #h_highres = 0.5 + 0.1*np.exp(-(xv**2/size + yv**2/size)) + h_highres = h_ref + h_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + h = downsample(h_highres, ref_nx/nx, ref_ny/ny) + h_highres = None + gc.collect() + + #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) + u_highres = u_ref + u_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + hu = downsample(u_highres, ref_nx/nx, ref_ny/ny)*h + u_highres = None + gc.collect() + + #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) + v_highres = v_ref + v_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + hv = downsample(v_highres, ref_nx/nx, ref_ny/ny)*h + v_highres = None + gc.collect() + + dx = width/nx + dy = height/ny + + return h, hu, hv, dx, dy + + +def genShockBubble(nx, ny, gamma, grid=None): + """ + Generate Shock-bubble interaction case for the Euler equations + """ + + width = 4.0 + height = 1.0 + g = 0.0 + + + rho = np.ones((ny, nx), dtype=np.float32) + u = np.zeros((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + E = np.zeros((ny, nx), dtype=np.float32) + p = np.ones((ny, nx), dtype=np.float32) + + + x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) + x = np.linspace(x0, x1, nx, dtype=np.float32) + y = np.linspace(y0, y1, ny, dtype=np.float32) + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + + #Bubble + radius = 0.25 + x_center = 0.5 + y_center = 0.5 + bubble = np.sqrt((xv-x_center)**2+(yv-y_center)**2) <= radius + rho = np.where(bubble, 0.1, rho) + + #Left boundary + left = (xv < 0.1) + rho = np.where(left, 3.81250, rho) + u = np.where(left, 2.57669, u) + + #Energy + p = np.where(left, 10.0, p) + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Reflective, + 'south': BoundaryCondition.Type.Reflective, + 'east': BoundaryCondition.Type.Periodic, + 'west': BoundaryCondition.Type.Periodic + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + return arguments + + + + + + + +def genKelvinHelmholtz(nx, ny, gamma, roughness=0.125, grid=None, index=None): + """ + Roughness parameter in (0, 1.0] determines how "squiggly" + the interface betweeen the zones is + """ + + def genZones(nx, ny, n): + """ + Generates the zones of the two fluids of K-H + """ + zone = np.zeros((ny, nx), dtype=np.int32) + + + def genSmoothRandom(nx, n): + n = max(1, min(n, nx)) + + if n == nx: + return np.random.random(nx)-0.5 + else: + from scipy.interpolate import interp1d + + #Control points and interpolator + xp = np.linspace(0.0, 1.0, n) + yp = np.random.random(n) - 0.5 + + if (n == 1): + kind = 'nearest' + elif (n == 2): + kind = 'linear' + elif (n == 3): + kind = 'quadratic' + else: + kind = 'cubic' + + f = interp1d(xp, yp, kind=kind) + + #Interpolation points + x = np.linspace(0.0, 1.0, nx) + return f(x) + + + + x0, x1, y0, y1, _, dy = getExtent(1.0, 1.0, nx, ny, grid, index) + x = np.linspace(x0, x1, nx) + y = np.linspace(y0, y1, ny) + _, y = np.meshgrid(x, y) + + #print(y+a[0]) + + a = genSmoothRandom(nx, n)*dy + zone = np.where(y > 0.25+a, zone, 1) + + a = genSmoothRandom(nx, n)*dy + zone = np.where(y < 0.75+a, zone, 1) + + return zone + + width = 2.0 + height = 1.0 + g = 0.0 + gamma = 1.4 + + rho = np.empty((ny, nx), dtype=np.float32) + u = np.empty((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + p = 2.5*np.ones((ny, nx), dtype=np.float32) + + #Generate the different zones + zones = genZones(nx, ny, max(1, min(nx, int(nx*roughness)))) + + #Zone 0 + zone0 = zones == 0 + rho = np.where(zone0, 1.0, rho) + u = np.where(zone0, 0.5, u) + + #Zone 1 + zone1 = zones == 1 + rho = np.where(zone1, 2.0, rho) + u = np.where(zone1, -0.5, u) + + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + _, _, _, _, dx, dy = getExtent(width, height, nx, ny, grid, index) + + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Periodic, + 'south': BoundaryCondition.Type.Periodic, + 'east': BoundaryCondition.Type.Periodic, + 'west': BoundaryCondition.Type.Periodic + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + + return arguments + + + +def genRayleighTaylor(nx, ny, gamma, version=0, grid=None): + """ + Generates Rayleigh-Taylor instability case + """ + width = 0.5 + height = 1.5 + g = 0.1 + + rho = np.zeros((ny, nx), dtype=np.float32) + u = np.zeros((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + p = np.zeros((ny, nx), dtype=np.float32) + + + x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) + x = np.linspace(x0, x1, nx, dtype=np.float32)-width*0.5 + y = np.linspace(y0, y1, ny, dtype=np.float32)-height*0.5 + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + + #This gives a squigly interfact + if (version == 0): + y_threshold = 0.01*np.cos(2*np.pi*np.abs(x)/0.5) + rho = np.where(yv <= y_threshold, 1.0, rho) + rho = np.where(yv > y_threshold, 2.0, rho) + elif (version == 1): + rho = np.where(yv <= 0.0, 1.0, rho) + rho = np.where(yv > 0.0, 2.0, rho) + v = 0.01*(1.0 + np.cos(2*np.pi*xv/0.5))/4 + else: + assert False, "Invalid version" + + p = 2.5 - rho*g*yv + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Reflective, + 'south': BoundaryCondition.Type.Reflective, + 'east': BoundaryCondition.Type.Reflective, + 'west': BoundaryCondition.Type.Reflective + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + + return arguments \ No newline at end of file diff --git a/GPUSimulators/helpers/Visualization.py b/GPUSimulators/helpers/Visualization.py new file mode 100644 index 0000000..a2ff8f1 --- /dev/null +++ b/GPUSimulators/helpers/Visualization.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements visualization techniques/modes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import numpy as np + +from matplotlib.colors import Normalize + + + +def genSchlieren(rho): + #Compute length of z-component of normalized gradient vector + normal = np.gradient(rho) #[x, y, 1] + length = 1.0 / np.sqrt(normal[0]**2 + normal[1]**2 + 1.0) + schlieren = np.power(length, 128) + return schlieren + + +def genVorticity(rho, rho_u, rho_v): + u = rho_u / rho + v = rho_v / rho + u = np.sqrt(u**2 + v**2) + u_max = u.max() + + du_dy, _ = np.gradient(u) + _, dv_dx = np.gradient(v) + + #Length of curl + curl = dv_dx - du_dy + return curl + + +def genColors(rho, rho_u, rho_v, cmap, vmax, vmin): + schlieren = genSchlieren(rho) + curl = genVorticity(rho, rho_u, rho_v) + + colors = Normalize(vmin, vmax, clip=True)(curl) + colors = cmap(colors) + for k in range(3): + colors[:,:,k] = colors[:,:,k]*schlieren + + return colors \ No newline at end of file From 250af3bfc3c08a0f50d5f5ca8ed6e1c477a82d21 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 14:57:23 +0100 Subject: [PATCH 02/55] re-write with hip-python --- mpiTesting.py | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 mpiTesting.py diff --git a/mpiTesting.py b/mpiTesting.py new file mode 100644 index 0000000..5869fdd --- /dev/null +++ b/mpiTesting.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements MPI simulations for benchmarking + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import numpy as np +import gc +import time +import json +import logging +import os + +#GPU-aware MPI +from os import environ +if environ.get("MPICH_GPU_SUPPORT_ENABLED", False): + from ctypes import CDLL, RTLD_GLOBAL + CDLL(f"{environ.get('CRAY_MPICH_ROOTDIR')}/gtl/lib/libmpi_gtl_hsa.so", mode=RTLD_GLOBAL) + +# MPI +from mpi4py import MPI + +# CUDA +#import pycuda.driver as cuda +from hip import hip + +# Simulator engine etc +from GPUSimulators import MPISimulator, Common, CudaContext +from GPUSimulators import EE2D_KP07_dimsplit +from GPUSimulators.helpers import InitialConditions as IC +from GPUSimulators.Simulator import BoundaryCondition as BC + +import argparse +parser = argparse.ArgumentParser(description='Strong and weak scaling experiments.') +parser.add_argument('-nx', type=int, default=128) +parser.add_argument('-ny', type=int, default=128) +parser.add_argument('--profile', action='store_true') # default: False + +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + +args = parser.parse_args() + +if(args.profile): + profiling_data = {} + # profiling: total run time + t_total_start = time.time() + t_init_start = time.time() + + +# Get MPI COMM to use +comm = MPI.COMM_WORLD + + +#### +# Initialize logging +#### +log_level_console = 20 +log_level_file = 10 +log_filename = 'mpi_' + str(comm.rank) + '.log' +logger = logging.getLogger('GPUSimulators') +logger.setLevel(min(log_level_console, log_level_file)) + +ch = logging.StreamHandler() +ch.setLevel(log_level_console) +logger.addHandler(ch) +logger.info("Console logger using level %s", + logging.getLevelName(log_level_console)) + +fh = logging.FileHandler(log_filename) +formatter = logging.Formatter( + '%(asctime)s:%(name)s:%(levelname)s: %(message)s') +fh.setFormatter(formatter) +fh.setLevel(log_level_file) +logger.addHandler(fh) +logger.info("File logger using level %s to %s", + logging.getLevelName(log_level_file), log_filename) + + +#### +# Initialize MPI grid etc +#### +logger.info("Creating MPI grid") +grid = MPISimulator.MPIGrid(MPI.COMM_WORLD) + + +#### +# Initialize CUDA +#### +#cuda.init(flags=0) +#logger.info("Initializing CUDA") +local_rank = grid.getLocalRank() +#num_cuda_devices = cuda.Device.count() +num_cuda_devices = hip_check(hip.hipGetDeviceCount()) +cuda_device = local_rank % num_cuda_devices +logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device)) +cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False) + + +#### +# Set initial conditions +#### + +# DEBUGGING - setting random seed +np.random.seed(42) + +logger.info("Generating initial conditions") +nx = args.nx +ny = args.ny + +dt = 0.000001 + +gamma = 1.4 +#save_times = np.linspace(0, 0.000009, 2) +#save_times = np.linspace(0, 0.000099, 11) +#save_times = np.linspace(0, 0.000099, 2) +save_times = np.linspace(0, 0.0000999, 2) +outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc" +save_var_names = ['rho', 'rho_u', 'rho_v', 'E'] + +arguments = IC.genKelvinHelmholtz(nx, ny, gamma, grid=grid) +arguments['context'] = cuda_context +arguments['theta'] = 1.2 +arguments['grid'] = grid + +if(args.profile): + t_init_end = time.time() + t_init = t_init_end - t_init_start + profiling_data["t_init"] = t_init + +#### +# Run simulation +#### +logger.info("Running simulation") +# Helper function to create MPI simulator + + +def genSim(grid, **kwargs): + local_sim = EE2D_KP07_dimsplit.EE2D_KP07_dimsplit(**kwargs) + sim = MPISimulator.MPISimulator(local_sim, grid) + return sim + + +outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation( + genSim, arguments, outfile, save_times, save_var_names, dt) + +if(args.profile): + t_total_end = time.time() + t_total = t_total_end - t_total_start + profiling_data["t_total"] = t_total + print("Total run time on rank " + str(MPI.COMM_WORLD.rank) + " is " + str(t_total) + " s") + +# write profiling to json file +if(args.profile and MPI.COMM_WORLD.rank == 0): + job_id = "" + if "SLURM_JOB_ID" in os.environ: + job_id = int(os.environ["SLURM_JOB_ID"]) + allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"]) + allocated_gpus = int(os.environ["HIP_VISIBLE_DEVICES"].count(",") + 1) + # allocated_gpus = int(os.environ["CUDA_VISIBLE_DEVICES"].count(",") + 1) + profiling_file = "MPI_jobid_" + \ + str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json" + profiling_data["outfile"] = outfile + else: + profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json" + + for stage in sim_runner_profiling_data["start"].keys(): + profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage] + + for stage in sim_profiling_data["start"].keys(): + profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage] + + profiling_data["nx"] = nx + profiling_data["ny"] = ny + profiling_data["dt"] = dt + profiling_data["n_time_steps"] = sim_profiling_data["n_time_steps"] + + profiling_data["slurm_job_id"] = job_id + profiling_data["n_cuda_devices"] = str(num_cuda_devices) + profiling_data["n_processes"] = str(MPI.COMM_WORLD.size) + profiling_data["git_hash"] = Common.getGitHash() + profiling_data["git_status"] = Common.getGitStatus() + + with open(profiling_file, "w") as write_file: + json.dump(profiling_data, write_file) + +#### +# Clean shutdown +#### +sim = None +local_sim = None +cuda_context = None +arguments = None +logging.shutdown() +gc.collect() + + + +#### +# Print completion and exit +#### +print("Completed!") +exit(0) From d457b48d218aa4412fe2561ae0786d5cc73af629 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 14:59:22 +0100 Subject: [PATCH 03/55] include setup in readme --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9bb082 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +# FiniteVolumeGPU + +This is a HIP version of the [FiniteVolume code](#https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. + +## Setup +A good place to start exploring this codebase is the notebooks. Complete the following steps to run the notebooks: + +1. Install conda (see e.g. Miniconda or Anaconda) +2. Change directory to the repository root and run the following commands +3. conda env create -f conda_environment.yml +4. conda activate ShallowWaterGPU +5. jupyter notebook + +Make sure you are running the correct kernel ("conda:ShallowWaterGPU"). If not, change kernel using the "Kernel"-menu in the notebook. + +If you do not need to run notebooks you may use the conda environment found in conda_environment_hpc.yml + +## Troubleshooting +Have a look at the conda documentation and https://towardsdatascience.com/how-to-set-up-anaconda-and-jupyter-notebook-the-right-way-de3b7623ea4a + +## Setup on LUMI-G +Here is a step-by-step guide on installing packages on LUMI-G + +### Step 0: load modules +ml LUMI/23.03 +ml lumi-container-wrapper +ml cray-python/3.9.13.1 + +### Step 1: run conda-container +Installation via conda can be done as: + +conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml + +where the file `conda_environment_lumi.yml` contains packages to be installed. + +### Step 2: Set the env. variable to search for binaries +export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" + +### An alternative: Convert to a singularity container with cotainr +cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml From b739c4051c434646994663eb5416b30da0524590 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 14:59:58 +0100 Subject: [PATCH 04/55] add .yml file --- conda_environment_lumi.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 conda_environment_lumi.yml diff --git a/conda_environment_lumi.yml b/conda_environment_lumi.yml new file mode 100644 index 0000000..661ebc5 --- /dev/null +++ b/conda_environment_lumi.yml @@ -0,0 +1,36 @@ +# Assumes that conda, pip, build-essentials and cuda are installed +--- +name: ShallowWaterGPU_HPC +channels: +- conda-forge + +dependencies: +- python=3.9 +- numpy +- mpi4py +- six +- pytools +- netcdf4 +- scipy +- pip: + - hip-python + - hip-python-as-cuda + - -i https://test.pypi.org/simple/ + + +#On LUMI-G +#module load LUMI/23.03 +#module load lumi-container-wrapper +#ml cray-python/3.9.13.1 +#conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml +# export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" +# +# +# +# Install conda environment (one-time operation): +# $ conda env create -f conda_environment_hpc.yml +# Activate environment and install the following packages using pip: +# $ conda activate ShallowWaterGPU_HPC +# - pycuda: $ pip3 install --no-deps -U pycuda +# on Windows: make sure your visual studio c++ compiler is available in PATH +# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\ From d1788e52f99c38d420df073646553840608abb25 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 15:01:13 +0100 Subject: [PATCH 05/55] add SYSTEMS file --- SYSTEMS.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 SYSTEMS.md diff --git a/SYSTEMS.md b/SYSTEMS.md new file mode 100644 index 0000000..5b0e770 --- /dev/null +++ b/SYSTEMS.md @@ -0,0 +1,62 @@ +$ANSIBLE_VAULT;1.1;AES256 +61316265663939333638336466323036663861343233316466646432313138653633623662353937 +3232313165656633346432376237383566363537366534310a303231343936663438653835373161 +35616161323432653062323164623861353065333761663136313137333732313230626665386336 +6166656538396463370a356166316363326133313864386536323236346634323537393639653038 +66336337336132613061353964613638326233356336323962366531333932366539366339623563 +36333365326463616634323939333062363263636663373635653064626138363464666233316561 +63393735393233616437386537393739393433663631313864646535636262616336333631396166 +38643636323530386565396338623366393232313838356536303537393338393634666632656234 +65353930303762333639376638336364303439306132626531326132376264623063376464636430 +32333536386134333136313139313861306364333037323363393463333664633764653937623866 +34313064346261313330373132353563343761323435393930303136353865303163373937623831 +64343038373162333039653161643233353764633337366434396638376530636261323362373434 +38393630613065356632663533333331633039663935663732353234643131306665343339373265 +64356563653838613337663132663234356462343333623139626662363764656239326637653832 +35396636643937336431623531306133643137623831333936313839333738333730373136666336 +35623965643664343164373630313362656663386638376237616134343631386366313336626138 +62376436383837376539663438346431633138383363633862356366376537393932626262383637 +31323365333139653736623233636233323162343039663035346135326638633430303134396337 +36353264313835346130643736663665386364343835643166383361316631373338663731373335 +30313530326662663937666330643565363565616566633333363535656539656531666266613638 +30306264613438363265646332386535383238373433396337633636616532626161343236336533 +36366362653137333739353737386563613136653164383437316237643533633133313735633363 +64326433356266363133343339626333633063326533383632353639613163663966376465396231 +36663034363534396430316463386564663465323036613636343136643262666566303533346439 +63396466656639623836613130363835346435633437666463363333356231343038356434343861 +66643636633739336666316566653136363862346336353862653130346335363334616430366435 +30376365383262326438306266366265363030353764633630333034663037643037343132303631 +39316364366234363339363130333765616432306331373566393530653963356539636437383062 +34633938643563656363633864656361643539663833356638356365373061663964363530393535 +37646533386235613763396638393539303062326239633238373763326561313634313265613135 +64646138313562313732393732303133343234323438616165326530333234626363393735636530 +62353735313231353662353533636134306530623339383730306332613636663366653566313935 +32343935353566656130393533323639353863666436333839386463396337336635356663373136 +61323734613239396236393266363631313465363630306565636663396235626132336339623938 +62383435643661623938393662363262376566613365613465323432343534356433323330666133 +30303963656635303734316539333038663962626331313366666337663165323230646564623935 +61316630353739386365323339626166323562616630383538393733353864396565353039656333 +30343038636231363531383061613836653038373937616163643963393231356235626531366239 +62343333326434636665363931376235313535343135626261336439636663323233383565633964 +65333830613131396630336337646230393038386536336365313738316335386261393838383961 +64656331363738616539346663613261386639353437316231636533353031336464383432623939 +65386164396231393735643563663337643563633233373338643630313739373861356166616463 +35306263333963663434376263396464323135346663376334356134393066653439376263376231 +30333730383163366636323533393334336331633234306536376634313735613263366537346536 +62366564383861656662353738366665396639313833323038356661306135393338333466333563 +32653861346166663163383036386432343833333137663462343030363762663139366534326466 +66313864623438336164333430613766373430656536323964633863333931643036656563353639 +30313835666366383035343031643265386263316165323537613636656533376239633964393866 +61646163343032313036303738643763383364663134356634373262633361383035306231636364 +39333232636538643033313438396332383962656131363365666566633239366532326336363133 +38393064643030333538333562643435663434343863383834663266373337336433313663646164 +36343334343965623830613736393231666361643239663062393239613233376335383362666161 +66383035653330373736613234303631386163656561383138613363613539396332376162316131 +61313532653531653836343731636535623066383231613635316432323331623761383833623333 +39343632623961613561373261653939636363366531303839336237383166363733303538363237 +36373362636263666334316163633766303334373033636539353464393536356466636664333665 +32643135626366666137626464393961366165383334343063356334373534633764326162363837 +38643662326266313464343464646166643235663663303761313639376537306337353863336264 +66376335333738366265343636376363366365306137336665623466626261653937656461303332 +32613561616662383032393562613831626666373134303032626134313262363830326530643632 +61366133663564313933366430396430353762386133396436633839303766653765 From 77916aca2016ef22243f5fbd20f02c9234a9c650 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 15:01:27 +0100 Subject: [PATCH 06/55] add reference folder --- reference/swashes_1_nx=1024.csv | 1042 ++++++++ reference/swashes_1_nx=128.csv | 146 ++ reference/swashes_1_nx=2048.csv | 2066 ++++++++++++++++ reference/swashes_1_nx=256.csv | 274 ++ reference/swashes_1_nx=4096.csv | 4114 +++++++++++++++++++++++++++++++ reference/swashes_1_nx=512.csv | 530 ++++ 6 files changed, 8172 insertions(+) create mode 100644 reference/swashes_1_nx=1024.csv create mode 100644 reference/swashes_1_nx=128.csv create mode 100644 reference/swashes_1_nx=2048.csv create mode 100644 reference/swashes_1_nx=256.csv create mode 100644 reference/swashes_1_nx=4096.csv create mode 100644 reference/swashes_1_nx=512.csv diff --git a/reference/swashes_1_nx=1024.csv b/reference/swashes_1_nx=1024.csv new file mode 100644 index 0000000..bf7bda7 --- /dev/null +++ b/reference/swashes_1_nx=1024.csv @@ -0,0 +1,1042 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00976562 meters +# Number of cells: 1024 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00488281 0.005 0 0 0 0.005 0 0 +0.0146484 0.005 0 0 0 0.005 0 0 +0.0244141 0.005 0 0 0 0.005 0 0 +0.0341797 0.005 0 0 0 0.005 0 0 +0.0439453 0.005 0 0 0 0.005 0 0 +0.0537109 0.005 0 0 0 0.005 0 0 +0.0634766 0.005 0 0 0 0.005 0 0 +0.0732422 0.005 0 0 0 0.005 0 0 +0.0830078 0.005 0 0 0 0.005 0 0 +0.0927734 0.005 0 0 0 0.005 0 0 +0.102539 0.005 0 0 0 0.005 0 0 +0.112305 0.005 0 0 0 0.005 0 0 +0.12207 0.005 0 0 0 0.005 0 0 +0.131836 0.005 0 0 0 0.005 0 0 +0.141602 0.005 0 0 0 0.005 0 0 +0.151367 0.005 0 0 0 0.005 0 0 +0.161133 0.005 0 0 0 0.005 0 0 +0.170898 0.005 0 0 0 0.005 0 0 +0.180664 0.005 0 0 0 0.005 0 0 +0.19043 0.005 0 0 0 0.005 0 0 +0.200195 0.005 0 0 0 0.005 0 0 +0.209961 0.005 0 0 0 0.005 0 0 +0.219727 0.005 0 0 0 0.005 0 0 +0.229492 0.005 0 0 0 0.005 0 0 +0.239258 0.005 0 0 0 0.005 0 0 +0.249023 0.005 0 0 0 0.005 0 0 +0.258789 0.005 0 0 0 0.005 0 0 +0.268555 0.005 0 0 0 0.005 0 0 +0.27832 0.005 0 0 0 0.005 0 0 +0.288086 0.005 0 0 0 0.005 0 0 +0.297852 0.005 0 0 0 0.005 0 0 +0.307617 0.005 0 0 0 0.005 0 0 +0.317383 0.005 0 0 0 0.005 0 0 +0.327148 0.005 0 0 0 0.005 0 0 +0.336914 0.005 0 0 0 0.005 0 0 +0.34668 0.005 0 0 0 0.005 0 0 +0.356445 0.005 0 0 0 0.005 0 0 +0.366211 0.005 0 0 0 0.005 0 0 +0.375977 0.005 0 0 0 0.005 0 0 +0.385742 0.005 0 0 0 0.005 0 0 +0.395508 0.005 0 0 0 0.005 0 0 +0.405273 0.005 0 0 0 0.005 0 0 +0.415039 0.005 0 0 0 0.005 0 0 +0.424805 0.005 0 0 0 0.005 0 0 +0.43457 0.005 0 0 0 0.005 0 0 +0.444336 0.005 0 0 0 0.005 0 0 +0.454102 0.005 0 0 0 0.005 0 0 +0.463867 0.005 0 0 0 0.005 0 0 +0.473633 0.005 0 0 0 0.005 0 0 +0.483398 0.005 0 0 0 0.005 0 0 +0.493164 0.005 0 0 0 0.005 0 0 +0.50293 0.005 0 0 0 0.005 0 0 +0.512695 0.005 0 0 0 0.005 0 0 +0.522461 0.005 0 0 0 0.005 0 0 +0.532227 0.005 0 0 0 0.005 0 0 +0.541992 0.005 0 0 0 0.005 0 0 +0.551758 0.005 0 0 0 0.005 0 0 +0.561523 0.005 0 0 0 0.005 0 0 +0.571289 0.005 0 0 0 0.005 0 0 +0.581055 0.005 0 0 0 0.005 0 0 +0.59082 0.005 0 0 0 0.005 0 0 +0.600586 0.005 0 0 0 0.005 0 0 +0.610352 0.005 0 0 0 0.005 0 0 +0.620117 0.005 0 0 0 0.005 0 0 +0.629883 0.005 0 0 0 0.005 0 0 +0.639648 0.005 0 0 0 0.005 0 0 +0.649414 0.005 0 0 0 0.005 0 0 +0.65918 0.005 0 0 0 0.005 0 0 +0.668945 0.005 0 0 0 0.005 0 0 +0.678711 0.005 0 0 0 0.005 0 0 +0.688477 0.005 0 0 0 0.005 0 0 +0.698242 0.005 0 0 0 0.005 0 0 +0.708008 0.005 0 0 0 0.005 0 0 +0.717773 0.005 0 0 0 0.005 0 0 +0.727539 0.005 0 0 0 0.005 0 0 +0.737305 0.005 0 0 0 0.005 0 0 +0.74707 0.005 0 0 0 0.005 0 0 +0.756836 0.005 0 0 0 0.005 0 0 +0.766602 0.005 0 0 0 0.005 0 0 +0.776367 0.005 0 0 0 0.005 0 0 +0.786133 0.005 0 0 0 0.005 0 0 +0.795898 0.005 0 0 0 0.005 0 0 +0.805664 0.005 0 0 0 0.005 0 0 +0.81543 0.005 0 0 0 0.005 0 0 +0.825195 0.005 0 0 0 0.005 0 0 +0.834961 0.005 0 0 0 0.005 0 0 +0.844727 0.005 0 0 0 0.005 0 0 +0.854492 0.005 0 0 0 0.005 0 0 +0.864258 0.005 0 0 0 0.005 0 0 +0.874023 0.005 0 0 0 0.005 0 0 +0.883789 0.005 0 0 0 0.005 0 0 +0.893555 0.005 0 0 0 0.005 0 0 +0.90332 0.005 0 0 0 0.005 0 0 +0.913086 0.005 0 0 0 0.005 0 0 +0.922852 0.005 0 0 0 0.005 0 0 +0.932617 0.005 0 0 0 0.005 0 0 +0.942383 0.005 0 0 0 0.005 0 0 +0.952148 0.005 0 0 0 0.005 0 0 +0.961914 0.005 0 0 0 0.005 0 0 +0.97168 0.005 0 0 0 0.005 0 0 +0.981445 0.005 0 0 0 0.005 0 0 +0.991211 0.005 0 0 0 0.005 0 0 +1.00098 0.005 0 0 0 0.005 0 0 +1.01074 0.005 0 0 0 0.005 0 0 +1.02051 0.005 0 0 0 0.005 0 0 +1.03027 0.005 0 0 0 0.005 0 0 +1.04004 0.005 0 0 0 0.005 0 0 +1.0498 0.005 0 0 0 0.005 0 0 +1.05957 0.005 0 0 0 0.005 0 0 +1.06934 0.005 0 0 0 0.005 0 0 +1.0791 0.005 0 0 0 0.005 0 0 +1.08887 0.005 0 0 0 0.005 0 0 +1.09863 0.005 0 0 0 0.005 0 0 +1.1084 0.005 0 0 0 0.005 0 0 +1.11816 0.005 0 0 0 0.005 0 0 +1.12793 0.005 0 0 0 0.005 0 0 +1.1377 0.005 0 0 0 0.005 0 0 +1.14746 0.005 0 0 0 0.005 0 0 +1.15723 0.005 0 0 0 0.005 0 0 +1.16699 0.005 0 0 0 0.005 0 0 +1.17676 0.005 0 0 0 0.005 0 0 +1.18652 0.005 0 0 0 0.005 0 0 +1.19629 0.005 0 0 0 0.005 0 0 +1.20605 0.005 0 0 0 0.005 0 0 +1.21582 0.005 0 0 0 0.005 0 0 +1.22559 0.005 0 0 0 0.005 0 0 +1.23535 0.005 0 0 0 0.005 0 0 +1.24512 0.005 0 0 0 0.005 0 0 +1.25488 0.005 0 0 0 0.005 0 0 +1.26465 0.005 0 0 0 0.005 0 0 +1.27441 0.005 0 0 0 0.005 0 0 +1.28418 0.005 0 0 0 0.005 0 0 +1.29395 0.005 0 0 0 0.005 0 0 +1.30371 0.005 0 0 0 0.005 0 0 +1.31348 0.005 0 0 0 0.005 0 0 +1.32324 0.005 0 0 0 0.005 0 0 +1.33301 0.005 0 0 0 0.005 0 0 +1.34277 0.005 0 0 0 0.005 0 0 +1.35254 0.005 0 0 0 0.005 0 0 +1.3623 0.005 0 0 0 0.005 0 0 +1.37207 0.005 0 0 0 0.005 0 0 +1.38184 0.005 0 0 0 0.005 0 0 +1.3916 0.005 0 0 0 0.005 0 0 +1.40137 0.005 0 0 0 0.005 0 0 +1.41113 0.005 0 0 0 0.005 0 0 +1.4209 0.005 0 0 0 0.005 0 0 +1.43066 0.005 0 0 0 0.005 0 0 +1.44043 0.005 0 0 0 0.005 0 0 +1.4502 0.005 0 0 0 0.005 0 0 +1.45996 0.005 0 0 0 0.005 0 0 +1.46973 0.005 0 0 0 0.005 0 0 +1.47949 0.005 0 0 0 0.005 0 0 +1.48926 0.005 0 0 0 0.005 0 0 +1.49902 0.005 0 0 0 0.005 0 0 +1.50879 0.005 0 0 0 0.005 0 0 +1.51855 0.005 0 0 0 0.005 0 0 +1.52832 0.005 0 0 0 0.005 0 0 +1.53809 0.005 0 0 0 0.005 0 0 +1.54785 0.005 0 0 0 0.005 0 0 +1.55762 0.005 0 0 0 0.005 0 0 +1.56738 0.005 0 0 0 0.005 0 0 +1.57715 0.005 0 0 0 0.005 0 0 +1.58691 0.005 0 0 0 0.005 0 0 +1.59668 0.005 0 0 0 0.005 0 0 +1.60645 0.005 0 0 0 0.005 0 0 +1.61621 0.005 0 0 0 0.005 0 0 +1.62598 0.005 0 0 0 0.005 0 0 +1.63574 0.005 0 0 0 0.005 0 0 +1.64551 0.005 0 0 0 0.005 0 0 +1.65527 0.005 0 0 0 0.005 0 0 +1.66504 0.005 0 0 0 0.005 0 0 +1.6748 0.005 0 0 0 0.005 0 0 +1.68457 0.005 0 0 0 0.005 0 0 +1.69434 0.005 0 0 0 0.005 0 0 +1.7041 0.005 0 0 0 0.005 0 0 +1.71387 0.005 0 0 0 0.005 0 0 +1.72363 0.005 0 0 0 0.005 0 0 +1.7334 0.005 0 0 0 0.005 0 0 +1.74316 0.005 0 0 0 0.005 0 0 +1.75293 0.005 0 0 0 0.005 0 0 +1.7627 0.005 0 0 0 0.005 0 0 +1.77246 0.005 0 0 0 0.005 0 0 +1.78223 0.005 0 0 0 0.005 0 0 +1.79199 0.005 0 0 0 0.005 0 0 +1.80176 0.005 0 0 0 0.005 0 0 +1.81152 0.005 0 0 0 0.005 0 0 +1.82129 0.005 0 0 0 0.005 0 0 +1.83105 0.005 0 0 0 0.005 0 0 +1.84082 0.005 0 0 0 0.005 0 0 +1.85059 0.005 0 0 0 0.005 0 0 +1.86035 0.005 0 0 0 0.005 0 0 +1.87012 0.005 0 0 0 0.005 0 0 +1.87988 0.005 0 0 0 0.005 0 0 +1.88965 0.005 0 0 0 0.005 0 0 +1.89941 0.005 0 0 0 0.005 0 0 +1.90918 0.005 0 0 0 0.005 0 0 +1.91895 0.005 0 0 0 0.005 0 0 +1.92871 0.005 0 0 0 0.005 0 0 +1.93848 0.005 0 0 0 0.005 0 0 +1.94824 0.005 0 0 0 0.005 0 0 +1.95801 0.005 0 0 0 0.005 0 0 +1.96777 0.005 0 0 0 0.005 0 0 +1.97754 0.005 0 0 0 0.005 0 0 +1.9873 0.005 0 0 0 0.005 0 0 +1.99707 0.005 0 0 0 0.005 0 0 +2.00684 0.005 0 0 0 0.005 0 0 +2.0166 0.005 0 0 0 0.005 0 0 +2.02637 0.005 0 0 0 0.005 0 0 +2.03613 0.005 0 0 0 0.005 0 0 +2.0459 0.005 0 0 0 0.005 0 0 +2.05566 0.005 0 0 0 0.005 0 0 +2.06543 0.005 0 0 0 0.005 0 0 +2.0752 0.005 0 0 0 0.005 0 0 +2.08496 0.005 0 0 0 0.005 0 0 +2.09473 0.005 0 0 0 0.005 0 0 +2.10449 0.005 0 0 0 0.005 0 0 +2.11426 0.005 0 0 0 0.005 0 0 +2.12402 0.005 0 0 0 0.005 0 0 +2.13379 0.005 0 0 0 0.005 0 0 +2.14355 0.005 0 0 0 0.005 0 0 +2.15332 0.005 0 0 0 0.005 0 0 +2.16309 0.005 0 0 0 0.005 0 0 +2.17285 0.005 0 0 0 0.005 0 0 +2.18262 0.005 0 0 0 0.005 0 0 +2.19238 0.005 0 0 0 0.005 0 0 +2.20215 0.005 0 0 0 0.005 0 0 +2.21191 0.005 0 0 0 0.005 0 0 +2.22168 0.005 0 0 0 0.005 0 0 +2.23145 0.005 0 0 0 0.005 0 0 +2.24121 0.005 0 0 0 0.005 0 0 +2.25098 0.005 0 0 0 0.005 0 0 +2.26074 0.005 0 0 0 0.005 0 0 +2.27051 0.005 0 0 0 0.005 0 0 +2.28027 0.005 0 0 0 0.005 0 0 +2.29004 0.005 0 0 0 0.005 0 0 +2.2998 0.005 0 0 0 0.005 0 0 +2.30957 0.005 0 0 0 0.005 0 0 +2.31934 0.005 0 0 0 0.005 0 0 +2.3291 0.005 0 0 0 0.005 0 0 +2.33887 0.005 0 0 0 0.005 0 0 +2.34863 0.005 0 0 0 0.005 0 0 +2.3584 0.005 0 0 0 0.005 0 0 +2.36816 0.005 0 0 0 0.005 0 0 +2.37793 0.005 0 0 0 0.005 0 0 +2.3877 0.005 0 0 0 0.005 0 0 +2.39746 0.005 0 0 0 0.005 0 0 +2.40723 0.005 0 0 0 0.005 0 0 +2.41699 0.005 0 0 0 0.005 0 0 +2.42676 0.005 0 0 0 0.005 0 0 +2.43652 0.005 0 0 0 0.005 0 0 +2.44629 0.005 0 0 0 0.005 0 0 +2.45605 0.005 0 0 0 0.005 0 0 +2.46582 0.005 0 0 0 0.005 0 0 +2.47559 0.005 0 0 0 0.005 0 0 +2.48535 0.005 0 0 0 0.005 0 0 +2.49512 0.005 0 0 0 0.005 0 0 +2.50488 0.005 0 0 0 0.005 0 0 +2.51465 0.005 0 0 0 0.005 0 0 +2.52441 0.005 0 0 0 0.005 0 0 +2.53418 0.005 0 0 0 0.005 0 0 +2.54395 0.005 0 0 0 0.005 0 0 +2.55371 0.005 0 0 0 0.005 0 0 +2.56348 0.005 0 0 0 0.005 0 0 +2.57324 0.005 0 0 0 0.005 0 0 +2.58301 0.005 0 0 0 0.005 0 0 +2.59277 0.005 0 0 0 0.005 0 0 +2.60254 0.005 0 0 0 0.005 0 0 +2.6123 0.005 0 0 0 0.005 0 0 +2.62207 0.005 0 0 0 0.005 0 0 +2.63184 0.005 0 0 0 0.005 0 0 +2.6416 0.005 0 0 0 0.005 0 0 +2.65137 0.005 0 0 0 0.005 0 0 +2.66113 0.005 0 0 0 0.005 0 0 +2.6709 0.005 0 0 0 0.005 0 0 +2.68066 0.005 0 0 0 0.005 0 0 +2.69043 0.005 0 0 0 0.005 0 0 +2.7002 0.005 0 0 0 0.005 0 0 +2.70996 0.005 0 0 0 0.005 0 0 +2.71973 0.005 0 0 0 0.005 0 0 +2.72949 0.005 0 0 0 0.005 0 0 +2.73926 0.005 0 0 0 0.005 0 0 +2.74902 0.005 0 0 0 0.005 0 0 +2.75879 0.005 0 0 0 0.005 0 0 +2.76855 0.005 0 0 0 0.005 0 0 +2.77832 0.005 0 0 0 0.005 0 0 +2.78809 0.005 0 0 0 0.005 0 0 +2.79785 0.005 0 0 0 0.005 0 0 +2.80762 0.005 0 0 0 0.005 0 0 +2.81738 0.005 0 0 0 0.005 0 0 +2.82715 0.005 0 0 0 0.005 0 0 +2.83691 0.005 0 0 0 0.005 0 0 +2.84668 0.005 0 0 0 0.005 0 0 +2.85645 0.005 0 0 0 0.005 0 0 +2.86621 0.005 0 0 0 0.005 0 0 +2.87598 0.005 0 0 0 0.005 0 0 +2.88574 0.005 0 0 0 0.005 0 0 +2.89551 0.005 0 0 0 0.005 0 0 +2.90527 0.005 0 0 0 0.005 0 0 +2.91504 0.005 0 0 0 0.005 0 0 +2.9248 0.005 0 0 0 0.005 0 0 +2.93457 0.005 0 0 0 0.005 0 0 +2.94434 0.005 0 0 0 0.005 0 0 +2.9541 0.005 0 0 0 0.005 0 0 +2.96387 0.005 0 0 0 0.005 0 0 +2.97363 0.005 0 0 0 0.005 0 0 +2.9834 0.005 0 0 0 0.005 0 0 +2.99316 0.005 0 0 0 0.005 0 0 +3.00293 0.005 0 0 0 0.005 0 0 +3.0127 0.005 0 0 0 0.005 0 0 +3.02246 0.005 0 0 0 0.005 0 0 +3.03223 0.005 0 0 0 0.005 0 0 +3.04199 0.005 0 0 0 0.005 0 0 +3.05176 0.005 0 0 0 0.005 0 0 +3.06152 0.005 0 0 0 0.005 0 0 +3.07129 0.005 0 0 0 0.005 0 0 +3.08105 0.005 0 0 0 0.005 0 0 +3.09082 0.005 0 0 0 0.005 0 0 +3.10059 0.005 0 0 0 0.005 0 0 +3.11035 0.005 0 0 0 0.005 0 0 +3.12012 0.005 0 0 0 0.005 0 0 +3.12988 0.005 0 0 0 0.005 0 0 +3.13965 0.005 0 0 0 0.005 0 0 +3.14941 0.005 0 0 0 0.005 0 0 +3.15918 0.005 0 0 0 0.005 0 0 +3.16895 0.005 0 0 0 0.005 0 0 +3.17871 0.005 0 0 0 0.005 0 0 +3.18848 0.005 0 0 0 0.005 0 0 +3.19824 0.005 0 0 0 0.005 0 0 +3.20801 0.005 0 0 0 0.005 0 0 +3.21777 0.005 0 0 0 0.005 0 0 +3.22754 0.005 0 0 0 0.005 0 0 +3.2373 0.005 0 0 0 0.005 0 0 +3.24707 0.005 0 0 0 0.005 0 0 +3.25684 0.005 0 0 0 0.005 0 0 +3.2666 0.005 0 0 0 0.005 0 0 +3.27637 0.005 0 0 0 0.005 0 0 +3.28613 0.005 0 0 0 0.005 0 0 +3.2959 0.005 0 0 0 0.005 0 0 +3.30566 0.005 0 0 0 0.005 0 0 +3.31543 0.005 0 0 0 0.005 0 0 +3.3252 0.005 0 0 0 0.005 0 0 +3.33496 0.005 0 0 0 0.005 0 0 +3.34473 0.005 0 0 0 0.005 0 0 +3.35449 0.005 0 0 0 0.005 0 0 +3.36426 0.005 0 0 0 0.005 0 0 +3.37402 0.005 0 0 0 0.005 0 0 +3.38379 0.005 0 0 0 0.005 0 0 +3.39355 0.005 0 0 0 0.005 0 0 +3.40332 0.005 0 0 0 0.005 0 0 +3.41309 0.005 0 0 0 0.005 0 0 +3.42285 0.005 0 0 0 0.005 0 0 +3.43262 0.005 0 0 0 0.005 0 0 +3.44238 0.005 0 0 0 0.005 0 0 +3.45215 0.005 0 0 0 0.005 0 0 +3.46191 0.005 0 0 0 0.005 0 0 +3.47168 0.005 0 0 0 0.005 0 0 +3.48145 0.005 0 0 0 0.005 0 0 +3.49121 0.005 0 0 0 0.005 0 0 +3.50098 0.005 0 0 0 0.005 0 0 +3.51074 0.005 0 0 0 0.005 0 0 +3.52051 0.005 0 0 0 0.005 0 0 +3.53027 0.005 0 0 0 0.005 0 0 +3.54004 0.005 0 0 0 0.005 0 0 +3.5498 0.005 0 0 0 0.005 0 0 +3.55957 0.005 0 0 0 0.005 0 0 +3.56934 0.005 0 0 0 0.005 0 0 +3.5791 0.005 0 0 0 0.005 0 0 +3.58887 0.005 0 0 0 0.005 0 0 +3.59863 0.005 0 0 0 0.005 0 0 +3.6084 0.005 0 0 0 0.005 0 0 +3.61816 0.005 0 0 0 0.005 0 0 +3.62793 0.005 0 0 0 0.005 0 0 +3.6377 0.005 0 0 0 0.005 0 0 +3.64746 0.005 0 0 0 0.005 0 0 +3.65723 0.005 0 0 0 0.005 0 0 +3.66699 0.005 0 0 0 0.005 0 0 +3.67676 0.00498598 0.000621321 0 3.0979e-006 0.00498598 0.00280935 9.92708e-005 +3.68652 0.00496155 0.00170639 0 8.46634e-006 0.00496155 0.00773455 0.000194047 +3.69629 0.00493718 0.00279146 0 1.37819e-005 0.00493718 0.012684 0.000268524 +3.70605 0.00491287 0.00387653 0 1.90449e-005 0.00491287 0.017658 0.000333142 +3.71582 0.00488861 0.0049616 0 2.42553e-005 0.00488861 0.0226566 0.000391425 +3.72559 0.00486442 0.00604667 0 2.94135e-005 0.00486442 0.02768 0.000445118 +3.73535 0.00484029 0.00713174 0 3.45197e-005 0.00484029 0.0327284 0.000495246 +3.74512 0.00481622 0.00821681 0 3.95739e-005 0.00481622 0.0378021 0.000542479 +3.75488 0.0047922 0.00930188 0 4.45765e-005 0.0047922 0.0429011 0.000587283 +3.76465 0.00476825 0.0103869 0 4.95276e-005 0.00476825 0.0480257 0.000630002 +3.77441 0.00474436 0.011472 0 5.44274e-005 0.00474436 0.0531761 0.000670896 +3.78418 0.00472053 0.0125571 0 5.92761e-005 0.00472053 0.0583524 0.000710171 +3.79395 0.00469676 0.0136422 0 6.40739e-005 0.00469676 0.063555 0.000747993 +3.80371 0.00467304 0.0147272 0 6.88209e-005 0.00467304 0.0687839 0.000784496 +3.81348 0.00464939 0.0158123 0 7.35175e-005 0.00464939 0.0740393 0.000819793 +3.82324 0.0046258 0.0168974 0 7.81638e-005 0.0046258 0.0793215 0.000853979 +3.83301 0.00460227 0.0179824 0 8.27599e-005 0.00460227 0.0846307 0.000887136 +3.84277 0.00457879 0.0190675 0 8.73062e-005 0.00457879 0.0899671 0.000919335 +3.85254 0.00455538 0.0201526 0 9.18027e-005 0.00455538 0.0953309 0.000950635 +3.8623 0.00453203 0.0212376 0 9.62496e-005 0.00453203 0.100722 0.000981092 +3.87207 0.00450874 0.0223227 0 0.000100647 0.00450874 0.106141 0.00101075 +3.88184 0.00448551 0.0234078 0 0.000104996 0.00448551 0.111589 0.00103966 +3.8916 0.00446233 0.0244928 0 0.000109295 0.00446233 0.117064 0.00106785 +3.90137 0.00443922 0.0255779 0 0.000113546 0.00443922 0.122568 0.00109536 +3.91113 0.00441617 0.026663 0 0.000117748 0.00441617 0.128101 0.00112223 +3.9209 0.00439318 0.0277481 0 0.000121902 0.00439318 0.133662 0.00114847 +3.93066 0.00437024 0.0288331 0 0.000126008 0.00437024 0.139253 0.00117411 +3.94043 0.00434737 0.0299182 0 0.000130066 0.00434737 0.144873 0.00119918 +3.9502 0.00432456 0.0310033 0 0.000134075 0.00432456 0.150523 0.0012237 +3.95996 0.00430181 0.0320883 0 0.000138038 0.00430181 0.156202 0.0012477 +3.96973 0.00427912 0.0331734 0 0.000141953 0.00427912 0.161912 0.00127118 +3.97949 0.00425648 0.0342585 0 0.000145821 0.00425648 0.167652 0.00129417 +3.98926 0.00423391 0.0353435 0 0.000149641 0.00423391 0.173422 0.00131667 +3.99902 0.0042114 0.0364286 0 0.000153415 0.0042114 0.179223 0.00133872 +4.00879 0.00418895 0.0375137 0 0.000157143 0.00418895 0.185056 0.00136032 +4.01855 0.00416656 0.0385988 0 0.000160824 0.00416656 0.190919 0.00138148 +4.02832 0.00414422 0.0396838 0 0.000164459 0.00414422 0.196815 0.00140222 +4.03809 0.00412195 0.0407689 0 0.000168047 0.00412195 0.202742 0.00142254 +4.04785 0.00409974 0.041854 0 0.00017159 0.00409974 0.208701 0.00144247 +4.05762 0.00407759 0.042939 0 0.000175088 0.00407759 0.214692 0.001462 +4.06738 0.0040555 0.0440241 0 0.00017854 0.0040555 0.220716 0.00148115 +4.07715 0.00403346 0.0451092 0 0.000181946 0.00403346 0.226773 0.00149993 +4.08691 0.00401149 0.0461942 0 0.000185308 0.00401149 0.232863 0.00151835 +4.09668 0.00398958 0.0472793 0 0.000188625 0.00398958 0.238986 0.00153642 +4.10645 0.00396773 0.0483644 0 0.000191897 0.00396773 0.245143 0.00155413 +4.11621 0.00394594 0.0494494 0 0.000195124 0.00394594 0.251334 0.00157151 +4.12598 0.0039242 0.0505345 0 0.000198308 0.0039242 0.25756 0.00158856 +4.13574 0.00390253 0.0516196 0 0.000201447 0.00390253 0.263819 0.00160528 +4.14551 0.00388092 0.0527047 0 0.000204543 0.00388092 0.270114 0.00162168 +4.15527 0.00385937 0.0537897 0 0.000207594 0.00385937 0.276444 0.00163777 +4.16504 0.00383788 0.0548748 0 0.000210603 0.00383788 0.282809 0.00165356 +4.1748 0.00381644 0.0559599 0 0.000213568 0.00381644 0.28921 0.00166904 +4.18457 0.00379507 0.0570449 0 0.00021649 0.00379507 0.295646 0.00168423 +4.19434 0.00377376 0.05813 0 0.000219369 0.00377376 0.302119 0.00169913 +4.2041 0.00375251 0.0592151 0 0.000222205 0.00375251 0.308629 0.00171375 +4.21387 0.00373132 0.0603001 0 0.000224999 0.00373132 0.315176 0.00172808 +4.22363 0.00371018 0.0613852 0 0.00022775 0.00371018 0.32176 0.00174214 +4.2334 0.00368911 0.0624703 0 0.00023046 0.00368911 0.328381 0.00175593 +4.24316 0.0036681 0.0635553 0 0.000233127 0.0036681 0.33504 0.00176945 +4.25293 0.00364715 0.0646404 0 0.000235753 0.00364715 0.341738 0.00178272 +4.2627 0.00362626 0.0657255 0 0.000238338 0.00362626 0.348474 0.00179572 +4.27246 0.00360543 0.0668106 0 0.00024088 0.00360543 0.355249 0.00180847 +4.28223 0.00358465 0.0678956 0 0.000243382 0.00358465 0.362063 0.00182097 +4.29199 0.00356394 0.0689807 0 0.000245843 0.00356394 0.368916 0.00183322 +4.30176 0.00354329 0.0700658 0 0.000248263 0.00354329 0.37581 0.00184524 +4.31152 0.0035227 0.0711508 0 0.000250643 0.0035227 0.382743 0.00185701 +4.32129 0.00350217 0.0722359 0 0.000252982 0.00350217 0.389718 0.00186855 +4.33105 0.00348169 0.073321 0 0.000255281 0.00348169 0.396733 0.00187985 +4.34082 0.00346128 0.074406 0 0.00025754 0.00346128 0.40379 0.00189092 +4.35059 0.00344093 0.0754911 0 0.00025976 0.00344093 0.410888 0.00190177 +4.36035 0.00342064 0.0765762 0 0.000261939 0.00342064 0.418028 0.0019124 +4.37012 0.00340041 0.0776613 0 0.00026408 0.00340041 0.425211 0.0019228 +4.37988 0.00338024 0.0787463 0 0.000266181 0.00338024 0.432436 0.00193299 +4.38965 0.00336012 0.0798314 0 0.000268243 0.00336012 0.439705 0.00194296 +4.39941 0.00334007 0.0809165 0 0.000270267 0.00334007 0.447017 0.00195271 +4.40918 0.00332008 0.0820015 0 0.000272252 0.00332008 0.454374 0.00196226 +4.41895 0.00330015 0.0830866 0 0.000274198 0.00330015 0.461774 0.00197161 +4.42871 0.00328028 0.0841717 0 0.000276106 0.00328028 0.46922 0.00198074 +4.43848 0.00326047 0.0852567 0 0.000277977 0.00326047 0.47671 0.00198968 +4.44824 0.00324071 0.0863418 0 0.000279809 0.00324071 0.484246 0.00199841 +4.45801 0.00322102 0.0874269 0 0.000281604 0.00322102 0.491828 0.00200695 +4.46777 0.00320139 0.0885119 0 0.000283361 0.00320139 0.499457 0.00201529 +4.47754 0.00318182 0.089597 0 0.000285081 0.00318182 0.507132 0.00202344 +4.4873 0.00316231 0.0906821 0 0.000286765 0.00316231 0.514855 0.00203139 +4.49707 0.00314286 0.0917672 0 0.000288411 0.00314286 0.522625 0.00203916 +4.50684 0.00312346 0.0928522 0 0.000290021 0.00312346 0.530444 0.00204674 +4.5166 0.00310413 0.0939373 0 0.000291594 0.00310413 0.538311 0.00205414 +4.52637 0.00308486 0.0950224 0 0.000293131 0.00308486 0.546227 0.00206135 +4.53613 0.00306565 0.0961074 0 0.000294632 0.00306565 0.554193 0.00206838 +4.5459 0.0030465 0.0971925 0 0.000296097 0.0030465 0.562209 0.00207523 +4.55566 0.00302741 0.0982776 0 0.000297526 0.00302741 0.570275 0.0020819 +4.56543 0.00300837 0.0993626 0 0.00029892 0.00300837 0.578392 0.0020884 +4.5752 0.0029894 0.100448 0 0.000300279 0.0029894 0.586561 0.00209472 +4.58496 0.00297049 0.101533 0 0.000301602 0.00297049 0.594782 0.00210087 +4.59473 0.00295164 0.102618 0 0.000302891 0.00295164 0.603055 0.00210685 +4.60449 0.00293285 0.103703 0 0.000304145 0.00293285 0.611381 0.00211267 +4.61426 0.00291412 0.104788 0 0.000305364 0.00291412 0.61976 0.00211831 +4.62402 0.00289545 0.105873 0 0.00030655 0.00289545 0.628193 0.00212379 +4.63379 0.00287683 0.106958 0 0.000307701 0.00287683 0.636681 0.0021291 +4.64355 0.00285828 0.108043 0 0.000308818 0.00285828 0.645224 0.00213425 +4.65332 0.00283979 0.109128 0 0.000309901 0.00283979 0.653822 0.00213924 +4.66309 0.00282136 0.110213 0 0.000310951 0.00282136 0.662476 0.00214407 +4.67285 0.00280299 0.111298 0 0.000311968 0.00280299 0.671187 0.00214874 +4.68262 0.00278468 0.112383 0 0.000312952 0.00278468 0.679956 0.00215325 +4.69238 0.00276643 0.113469 0 0.000313902 0.00276643 0.688782 0.00215761 +4.70215 0.00274823 0.114554 0 0.00031482 0.00274823 0.697666 0.00216182 +4.71191 0.0027301 0.115639 0 0.000315705 0.0027301 0.706609 0.00216587 +4.72168 0.00271203 0.116724 0 0.000316558 0.00271203 0.715612 0.00216977 +4.73145 0.00269402 0.117809 0 0.000317379 0.00269402 0.724674 0.00217352 +4.74121 0.00267607 0.118894 0 0.000318168 0.00267607 0.733798 0.00217712 +4.75098 0.00265818 0.119979 0 0.000318925 0.00265818 0.742983 0.00218057 +4.76074 0.00264035 0.121064 0 0.000319651 0.00264035 0.752229 0.00218387 +4.77051 0.00262257 0.122149 0 0.000320345 0.00262257 0.761539 0.00218703 +4.78027 0.00260486 0.123234 0 0.000321008 0.00260486 0.770911 0.00219005 +4.79004 0.00258721 0.124319 0 0.00032164 0.00258721 0.780347 0.00219293 +4.7998 0.00256962 0.125404 0 0.000322241 0.00256962 0.789848 0.00219566 +4.80957 0.00255209 0.126489 0 0.000322812 0.00255209 0.799414 0.00219825 +4.81934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01465 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02441 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04395 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05371 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06348 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07324 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08301 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09277 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10254 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13184 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15137 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16113 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18066 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19043 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20996 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21973 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22949 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23926 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24902 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25879 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26855 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28809 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29785 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30762 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31738 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32715 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33691 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36621 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37598 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38574 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39551 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40527 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41504 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43457 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44434 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46387 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47363 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49316 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52246 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53223 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54199 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55176 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56152 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57129 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58105 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60059 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61035 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62012 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62988 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63965 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64941 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65918 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67871 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68848 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69824 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70801 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71777 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72754 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75684 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77637 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78613 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80566 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83496 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84473 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85449 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86426 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87402 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88379 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90332 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91309 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92285 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93262 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94238 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95215 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96191 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97168 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99121 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00098 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01074 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02051 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03027 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04004 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26465 0.001 0 0 0 0.001 0 0 +6.27441 0.001 0 0 0 0.001 0 0 +6.28418 0.001 0 0 0 0.001 0 0 +6.29395 0.001 0 0 0 0.001 0 0 +6.30371 0.001 0 0 0 0.001 0 0 +6.31348 0.001 0 0 0 0.001 0 0 +6.32324 0.001 0 0 0 0.001 0 0 +6.33301 0.001 0 0 0 0.001 0 0 +6.34277 0.001 0 0 0 0.001 0 0 +6.35254 0.001 0 0 0 0.001 0 0 +6.3623 0.001 0 0 0 0.001 0 0 +6.37207 0.001 0 0 0 0.001 0 0 +6.38184 0.001 0 0 0 0.001 0 0 +6.3916 0.001 0 0 0 0.001 0 0 +6.40137 0.001 0 0 0 0.001 0 0 +6.41113 0.001 0 0 0 0.001 0 0 +6.4209 0.001 0 0 0 0.001 0 0 +6.43066 0.001 0 0 0 0.001 0 0 +6.44043 0.001 0 0 0 0.001 0 0 +6.4502 0.001 0 0 0 0.001 0 0 +6.45996 0.001 0 0 0 0.001 0 0 +6.46973 0.001 0 0 0 0.001 0 0 +6.47949 0.001 0 0 0 0.001 0 0 +6.48926 0.001 0 0 0 0.001 0 0 +6.49902 0.001 0 0 0 0.001 0 0 +6.50879 0.001 0 0 0 0.001 0 0 +6.51855 0.001 0 0 0 0.001 0 0 +6.52832 0.001 0 0 0 0.001 0 0 +6.53809 0.001 0 0 0 0.001 0 0 +6.54785 0.001 0 0 0 0.001 0 0 +6.55762 0.001 0 0 0 0.001 0 0 +6.56738 0.001 0 0 0 0.001 0 0 +6.57715 0.001 0 0 0 0.001 0 0 +6.58691 0.001 0 0 0 0.001 0 0 +6.59668 0.001 0 0 0 0.001 0 0 +6.60645 0.001 0 0 0 0.001 0 0 +6.61621 0.001 0 0 0 0.001 0 0 +6.62598 0.001 0 0 0 0.001 0 0 +6.63574 0.001 0 0 0 0.001 0 0 +6.64551 0.001 0 0 0 0.001 0 0 +6.65527 0.001 0 0 0 0.001 0 0 +6.66504 0.001 0 0 0 0.001 0 0 +6.6748 0.001 0 0 0 0.001 0 0 +6.68457 0.001 0 0 0 0.001 0 0 +6.69434 0.001 0 0 0 0.001 0 0 +6.7041 0.001 0 0 0 0.001 0 0 +6.71387 0.001 0 0 0 0.001 0 0 +6.72363 0.001 0 0 0 0.001 0 0 +6.7334 0.001 0 0 0 0.001 0 0 +6.74316 0.001 0 0 0 0.001 0 0 +6.75293 0.001 0 0 0 0.001 0 0 +6.7627 0.001 0 0 0 0.001 0 0 +6.77246 0.001 0 0 0 0.001 0 0 +6.78223 0.001 0 0 0 0.001 0 0 +6.79199 0.001 0 0 0 0.001 0 0 +6.80176 0.001 0 0 0 0.001 0 0 +6.81152 0.001 0 0 0 0.001 0 0 +6.82129 0.001 0 0 0 0.001 0 0 +6.83105 0.001 0 0 0 0.001 0 0 +6.84082 0.001 0 0 0 0.001 0 0 +6.85059 0.001 0 0 0 0.001 0 0 +6.86035 0.001 0 0 0 0.001 0 0 +6.87012 0.001 0 0 0 0.001 0 0 +6.87988 0.001 0 0 0 0.001 0 0 +6.88965 0.001 0 0 0 0.001 0 0 +6.89941 0.001 0 0 0 0.001 0 0 +6.90918 0.001 0 0 0 0.001 0 0 +6.91895 0.001 0 0 0 0.001 0 0 +6.92871 0.001 0 0 0 0.001 0 0 +6.93848 0.001 0 0 0 0.001 0 0 +6.94824 0.001 0 0 0 0.001 0 0 +6.95801 0.001 0 0 0 0.001 0 0 +6.96777 0.001 0 0 0 0.001 0 0 +6.97754 0.001 0 0 0 0.001 0 0 +6.9873 0.001 0 0 0 0.001 0 0 +6.99707 0.001 0 0 0 0.001 0 0 +7.00684 0.001 0 0 0 0.001 0 0 +7.0166 0.001 0 0 0 0.001 0 0 +7.02637 0.001 0 0 0 0.001 0 0 +7.03613 0.001 0 0 0 0.001 0 0 +7.0459 0.001 0 0 0 0.001 0 0 +7.05566 0.001 0 0 0 0.001 0 0 +7.06543 0.001 0 0 0 0.001 0 0 +7.0752 0.001 0 0 0 0.001 0 0 +7.08496 0.001 0 0 0 0.001 0 0 +7.09473 0.001 0 0 0 0.001 0 0 +7.10449 0.001 0 0 0 0.001 0 0 +7.11426 0.001 0 0 0 0.001 0 0 +7.12402 0.001 0 0 0 0.001 0 0 +7.13379 0.001 0 0 0 0.001 0 0 +7.14355 0.001 0 0 0 0.001 0 0 +7.15332 0.001 0 0 0 0.001 0 0 +7.16309 0.001 0 0 0 0.001 0 0 +7.17285 0.001 0 0 0 0.001 0 0 +7.18262 0.001 0 0 0 0.001 0 0 +7.19238 0.001 0 0 0 0.001 0 0 +7.20215 0.001 0 0 0 0.001 0 0 +7.21191 0.001 0 0 0 0.001 0 0 +7.22168 0.001 0 0 0 0.001 0 0 +7.23145 0.001 0 0 0 0.001 0 0 +7.24121 0.001 0 0 0 0.001 0 0 +7.25098 0.001 0 0 0 0.001 0 0 +7.26074 0.001 0 0 0 0.001 0 0 +7.27051 0.001 0 0 0 0.001 0 0 +7.28027 0.001 0 0 0 0.001 0 0 +7.29004 0.001 0 0 0 0.001 0 0 +7.2998 0.001 0 0 0 0.001 0 0 +7.30957 0.001 0 0 0 0.001 0 0 +7.31934 0.001 0 0 0 0.001 0 0 +7.3291 0.001 0 0 0 0.001 0 0 +7.33887 0.001 0 0 0 0.001 0 0 +7.34863 0.001 0 0 0 0.001 0 0 +7.3584 0.001 0 0 0 0.001 0 0 +7.36816 0.001 0 0 0 0.001 0 0 +7.37793 0.001 0 0 0 0.001 0 0 +7.3877 0.001 0 0 0 0.001 0 0 +7.39746 0.001 0 0 0 0.001 0 0 +7.40723 0.001 0 0 0 0.001 0 0 +7.41699 0.001 0 0 0 0.001 0 0 +7.42676 0.001 0 0 0 0.001 0 0 +7.43652 0.001 0 0 0 0.001 0 0 +7.44629 0.001 0 0 0 0.001 0 0 +7.45605 0.001 0 0 0 0.001 0 0 +7.46582 0.001 0 0 0 0.001 0 0 +7.47559 0.001 0 0 0 0.001 0 0 +7.48535 0.001 0 0 0 0.001 0 0 +7.49512 0.001 0 0 0 0.001 0 0 +7.50488 0.001 0 0 0 0.001 0 0 +7.51465 0.001 0 0 0 0.001 0 0 +7.52441 0.001 0 0 0 0.001 0 0 +7.53418 0.001 0 0 0 0.001 0 0 +7.54395 0.001 0 0 0 0.001 0 0 +7.55371 0.001 0 0 0 0.001 0 0 +7.56348 0.001 0 0 0 0.001 0 0 +7.57324 0.001 0 0 0 0.001 0 0 +7.58301 0.001 0 0 0 0.001 0 0 +7.59277 0.001 0 0 0 0.001 0 0 +7.60254 0.001 0 0 0 0.001 0 0 +7.6123 0.001 0 0 0 0.001 0 0 +7.62207 0.001 0 0 0 0.001 0 0 +7.63184 0.001 0 0 0 0.001 0 0 +7.6416 0.001 0 0 0 0.001 0 0 +7.65137 0.001 0 0 0 0.001 0 0 +7.66113 0.001 0 0 0 0.001 0 0 +7.6709 0.001 0 0 0 0.001 0 0 +7.68066 0.001 0 0 0 0.001 0 0 +7.69043 0.001 0 0 0 0.001 0 0 +7.7002 0.001 0 0 0 0.001 0 0 +7.70996 0.001 0 0 0 0.001 0 0 +7.71973 0.001 0 0 0 0.001 0 0 +7.72949 0.001 0 0 0 0.001 0 0 +7.73926 0.001 0 0 0 0.001 0 0 +7.74902 0.001 0 0 0 0.001 0 0 +7.75879 0.001 0 0 0 0.001 0 0 +7.76855 0.001 0 0 0 0.001 0 0 +7.77832 0.001 0 0 0 0.001 0 0 +7.78809 0.001 0 0 0 0.001 0 0 +7.79785 0.001 0 0 0 0.001 0 0 +7.80762 0.001 0 0 0 0.001 0 0 +7.81738 0.001 0 0 0 0.001 0 0 +7.82715 0.001 0 0 0 0.001 0 0 +7.83691 0.001 0 0 0 0.001 0 0 +7.84668 0.001 0 0 0 0.001 0 0 +7.85645 0.001 0 0 0 0.001 0 0 +7.86621 0.001 0 0 0 0.001 0 0 +7.87598 0.001 0 0 0 0.001 0 0 +7.88574 0.001 0 0 0 0.001 0 0 +7.89551 0.001 0 0 0 0.001 0 0 +7.90527 0.001 0 0 0 0.001 0 0 +7.91504 0.001 0 0 0 0.001 0 0 +7.9248 0.001 0 0 0 0.001 0 0 +7.93457 0.001 0 0 0 0.001 0 0 +7.94434 0.001 0 0 0 0.001 0 0 +7.9541 0.001 0 0 0 0.001 0 0 +7.96387 0.001 0 0 0 0.001 0 0 +7.97363 0.001 0 0 0 0.001 0 0 +7.9834 0.001 0 0 0 0.001 0 0 +7.99316 0.001 0 0 0 0.001 0 0 +8.00293 0.001 0 0 0 0.001 0 0 +8.0127 0.001 0 0 0 0.001 0 0 +8.02246 0.001 0 0 0 0.001 0 0 +8.03223 0.001 0 0 0 0.001 0 0 +8.04199 0.001 0 0 0 0.001 0 0 +8.05176 0.001 0 0 0 0.001 0 0 +8.06152 0.001 0 0 0 0.001 0 0 +8.07129 0.001 0 0 0 0.001 0 0 +8.08105 0.001 0 0 0 0.001 0 0 +8.09082 0.001 0 0 0 0.001 0 0 +8.10059 0.001 0 0 0 0.001 0 0 +8.11035 0.001 0 0 0 0.001 0 0 +8.12012 0.001 0 0 0 0.001 0 0 +8.12988 0.001 0 0 0 0.001 0 0 +8.13965 0.001 0 0 0 0.001 0 0 +8.14941 0.001 0 0 0 0.001 0 0 +8.15918 0.001 0 0 0 0.001 0 0 +8.16895 0.001 0 0 0 0.001 0 0 +8.17871 0.001 0 0 0 0.001 0 0 +8.18848 0.001 0 0 0 0.001 0 0 +8.19824 0.001 0 0 0 0.001 0 0 +8.20801 0.001 0 0 0 0.001 0 0 +8.21777 0.001 0 0 0 0.001 0 0 +8.22754 0.001 0 0 0 0.001 0 0 +8.2373 0.001 0 0 0 0.001 0 0 +8.24707 0.001 0 0 0 0.001 0 0 +8.25684 0.001 0 0 0 0.001 0 0 +8.2666 0.001 0 0 0 0.001 0 0 +8.27637 0.001 0 0 0 0.001 0 0 +8.28613 0.001 0 0 0 0.001 0 0 +8.2959 0.001 0 0 0 0.001 0 0 +8.30566 0.001 0 0 0 0.001 0 0 +8.31543 0.001 0 0 0 0.001 0 0 +8.3252 0.001 0 0 0 0.001 0 0 +8.33496 0.001 0 0 0 0.001 0 0 +8.34473 0.001 0 0 0 0.001 0 0 +8.35449 0.001 0 0 0 0.001 0 0 +8.36426 0.001 0 0 0 0.001 0 0 +8.37402 0.001 0 0 0 0.001 0 0 +8.38379 0.001 0 0 0 0.001 0 0 +8.39355 0.001 0 0 0 0.001 0 0 +8.40332 0.001 0 0 0 0.001 0 0 +8.41309 0.001 0 0 0 0.001 0 0 +8.42285 0.001 0 0 0 0.001 0 0 +8.43262 0.001 0 0 0 0.001 0 0 +8.44238 0.001 0 0 0 0.001 0 0 +8.45215 0.001 0 0 0 0.001 0 0 +8.46191 0.001 0 0 0 0.001 0 0 +8.47168 0.001 0 0 0 0.001 0 0 +8.48145 0.001 0 0 0 0.001 0 0 +8.49121 0.001 0 0 0 0.001 0 0 +8.50098 0.001 0 0 0 0.001 0 0 +8.51074 0.001 0 0 0 0.001 0 0 +8.52051 0.001 0 0 0 0.001 0 0 +8.53027 0.001 0 0 0 0.001 0 0 +8.54004 0.001 0 0 0 0.001 0 0 +8.5498 0.001 0 0 0 0.001 0 0 +8.55957 0.001 0 0 0 0.001 0 0 +8.56934 0.001 0 0 0 0.001 0 0 +8.5791 0.001 0 0 0 0.001 0 0 +8.58887 0.001 0 0 0 0.001 0 0 +8.59863 0.001 0 0 0 0.001 0 0 +8.6084 0.001 0 0 0 0.001 0 0 +8.61816 0.001 0 0 0 0.001 0 0 +8.62793 0.001 0 0 0 0.001 0 0 +8.6377 0.001 0 0 0 0.001 0 0 +8.64746 0.001 0 0 0 0.001 0 0 +8.65723 0.001 0 0 0 0.001 0 0 +8.66699 0.001 0 0 0 0.001 0 0 +8.67676 0.001 0 0 0 0.001 0 0 +8.68652 0.001 0 0 0 0.001 0 0 +8.69629 0.001 0 0 0 0.001 0 0 +8.70605 0.001 0 0 0 0.001 0 0 +8.71582 0.001 0 0 0 0.001 0 0 +8.72559 0.001 0 0 0 0.001 0 0 +8.73535 0.001 0 0 0 0.001 0 0 +8.74512 0.001 0 0 0 0.001 0 0 +8.75488 0.001 0 0 0 0.001 0 0 +8.76465 0.001 0 0 0 0.001 0 0 +8.77441 0.001 0 0 0 0.001 0 0 +8.78418 0.001 0 0 0 0.001 0 0 +8.79395 0.001 0 0 0 0.001 0 0 +8.80371 0.001 0 0 0 0.001 0 0 +8.81348 0.001 0 0 0 0.001 0 0 +8.82324 0.001 0 0 0 0.001 0 0 +8.83301 0.001 0 0 0 0.001 0 0 +8.84277 0.001 0 0 0 0.001 0 0 +8.85254 0.001 0 0 0 0.001 0 0 +8.8623 0.001 0 0 0 0.001 0 0 +8.87207 0.001 0 0 0 0.001 0 0 +8.88184 0.001 0 0 0 0.001 0 0 +8.8916 0.001 0 0 0 0.001 0 0 +8.90137 0.001 0 0 0 0.001 0 0 +8.91113 0.001 0 0 0 0.001 0 0 +8.9209 0.001 0 0 0 0.001 0 0 +8.93066 0.001 0 0 0 0.001 0 0 +8.94043 0.001 0 0 0 0.001 0 0 +8.9502 0.001 0 0 0 0.001 0 0 +8.95996 0.001 0 0 0 0.001 0 0 +8.96973 0.001 0 0 0 0.001 0 0 +8.97949 0.001 0 0 0 0.001 0 0 +8.98926 0.001 0 0 0 0.001 0 0 +8.99902 0.001 0 0 0 0.001 0 0 +9.00879 0.001 0 0 0 0.001 0 0 +9.01855 0.001 0 0 0 0.001 0 0 +9.02832 0.001 0 0 0 0.001 0 0 +9.03809 0.001 0 0 0 0.001 0 0 +9.04785 0.001 0 0 0 0.001 0 0 +9.05762 0.001 0 0 0 0.001 0 0 +9.06738 0.001 0 0 0 0.001 0 0 +9.07715 0.001 0 0 0 0.001 0 0 +9.08691 0.001 0 0 0 0.001 0 0 +9.09668 0.001 0 0 0 0.001 0 0 +9.10645 0.001 0 0 0 0.001 0 0 +9.11621 0.001 0 0 0 0.001 0 0 +9.12598 0.001 0 0 0 0.001 0 0 +9.13574 0.001 0 0 0 0.001 0 0 +9.14551 0.001 0 0 0 0.001 0 0 +9.15527 0.001 0 0 0 0.001 0 0 +9.16504 0.001 0 0 0 0.001 0 0 +9.1748 0.001 0 0 0 0.001 0 0 +9.18457 0.001 0 0 0 0.001 0 0 +9.19434 0.001 0 0 0 0.001 0 0 +9.2041 0.001 0 0 0 0.001 0 0 +9.21387 0.001 0 0 0 0.001 0 0 +9.22363 0.001 0 0 0 0.001 0 0 +9.2334 0.001 0 0 0 0.001 0 0 +9.24316 0.001 0 0 0 0.001 0 0 +9.25293 0.001 0 0 0 0.001 0 0 +9.2627 0.001 0 0 0 0.001 0 0 +9.27246 0.001 0 0 0 0.001 0 0 +9.28223 0.001 0 0 0 0.001 0 0 +9.29199 0.001 0 0 0 0.001 0 0 +9.30176 0.001 0 0 0 0.001 0 0 +9.31152 0.001 0 0 0 0.001 0 0 +9.32129 0.001 0 0 0 0.001 0 0 +9.33105 0.001 0 0 0 0.001 0 0 +9.34082 0.001 0 0 0 0.001 0 0 +9.35059 0.001 0 0 0 0.001 0 0 +9.36035 0.001 0 0 0 0.001 0 0 +9.37012 0.001 0 0 0 0.001 0 0 +9.37988 0.001 0 0 0 0.001 0 0 +9.38965 0.001 0 0 0 0.001 0 0 +9.39941 0.001 0 0 0 0.001 0 0 +9.40918 0.001 0 0 0 0.001 0 0 +9.41895 0.001 0 0 0 0.001 0 0 +9.42871 0.001 0 0 0 0.001 0 0 +9.43848 0.001 0 0 0 0.001 0 0 +9.44824 0.001 0 0 0 0.001 0 0 +9.45801 0.001 0 0 0 0.001 0 0 +9.46777 0.001 0 0 0 0.001 0 0 +9.47754 0.001 0 0 0 0.001 0 0 +9.4873 0.001 0 0 0 0.001 0 0 +9.49707 0.001 0 0 0 0.001 0 0 +9.50684 0.001 0 0 0 0.001 0 0 +9.5166 0.001 0 0 0 0.001 0 0 +9.52637 0.001 0 0 0 0.001 0 0 +9.53613 0.001 0 0 0 0.001 0 0 +9.5459 0.001 0 0 0 0.001 0 0 +9.55566 0.001 0 0 0 0.001 0 0 +9.56543 0.001 0 0 0 0.001 0 0 +9.5752 0.001 0 0 0 0.001 0 0 +9.58496 0.001 0 0 0 0.001 0 0 +9.59473 0.001 0 0 0 0.001 0 0 +9.60449 0.001 0 0 0 0.001 0 0 +9.61426 0.001 0 0 0 0.001 0 0 +9.62402 0.001 0 0 0 0.001 0 0 +9.63379 0.001 0 0 0 0.001 0 0 +9.64355 0.001 0 0 0 0.001 0 0 +9.65332 0.001 0 0 0 0.001 0 0 +9.66309 0.001 0 0 0 0.001 0 0 +9.67285 0.001 0 0 0 0.001 0 0 +9.68262 0.001 0 0 0 0.001 0 0 +9.69238 0.001 0 0 0 0.001 0 0 +9.70215 0.001 0 0 0 0.001 0 0 +9.71191 0.001 0 0 0 0.001 0 0 +9.72168 0.001 0 0 0 0.001 0 0 +9.73145 0.001 0 0 0 0.001 0 0 +9.74121 0.001 0 0 0 0.001 0 0 +9.75098 0.001 0 0 0 0.001 0 0 +9.76074 0.001 0 0 0 0.001 0 0 +9.77051 0.001 0 0 0 0.001 0 0 +9.78027 0.001 0 0 0 0.001 0 0 +9.79004 0.001 0 0 0 0.001 0 0 +9.7998 0.001 0 0 0 0.001 0 0 +9.80957 0.001 0 0 0 0.001 0 0 +9.81934 0.001 0 0 0 0.001 0 0 +9.8291 0.001 0 0 0 0.001 0 0 +9.83887 0.001 0 0 0 0.001 0 0 +9.84863 0.001 0 0 0 0.001 0 0 +9.8584 0.001 0 0 0 0.001 0 0 +9.86816 0.001 0 0 0 0.001 0 0 +9.87793 0.001 0 0 0 0.001 0 0 +9.8877 0.001 0 0 0 0.001 0 0 +9.89746 0.001 0 0 0 0.001 0 0 +9.90723 0.001 0 0 0 0.001 0 0 +9.91699 0.001 0 0 0 0.001 0 0 +9.92676 0.001 0 0 0 0.001 0 0 +9.93652 0.001 0 0 0 0.001 0 0 +9.94629 0.001 0 0 0 0.001 0 0 +9.95605 0.001 0 0 0 0.001 0 0 +9.96582 0.001 0 0 0 0.001 0 0 +9.97559 0.001 0 0 0 0.001 0 0 +9.98535 0.001 0 0 0 0.001 0 0 +9.99512 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=128.csv b/reference/swashes_1_nx=128.csv new file mode 100644 index 0000000..b5357d4 --- /dev/null +++ b/reference/swashes_1_nx=128.csv @@ -0,0 +1,146 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.078125 meters +# Number of cells: 128 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0390625 0.005 0 0 0 0.005 0 0 +0.117188 0.005 0 0 0 0.005 0 0 +0.195312 0.005 0 0 0 0.005 0 0 +0.273438 0.005 0 0 0 0.005 0 0 +0.351562 0.005 0 0 0 0.005 0 0 +0.429688 0.005 0 0 0 0.005 0 0 +0.507812 0.005 0 0 0 0.005 0 0 +0.585938 0.005 0 0 0 0.005 0 0 +0.664062 0.005 0 0 0 0.005 0 0 +0.742188 0.005 0 0 0 0.005 0 0 +0.820312 0.005 0 0 0 0.005 0 0 +0.898438 0.005 0 0 0 0.005 0 0 +0.976562 0.005 0 0 0 0.005 0 0 +1.05469 0.005 0 0 0 0.005 0 0 +1.13281 0.005 0 0 0 0.005 0 0 +1.21094 0.005 0 0 0 0.005 0 0 +1.28906 0.005 0 0 0 0.005 0 0 +1.36719 0.005 0 0 0 0.005 0 0 +1.44531 0.005 0 0 0 0.005 0 0 +1.52344 0.005 0 0 0 0.005 0 0 +1.60156 0.005 0 0 0 0.005 0 0 +1.67969 0.005 0 0 0 0.005 0 0 +1.75781 0.005 0 0 0 0.005 0 0 +1.83594 0.005 0 0 0 0.005 0 0 +1.91406 0.005 0 0 0 0.005 0 0 +1.99219 0.005 0 0 0 0.005 0 0 +2.07031 0.005 0 0 0 0.005 0 0 +2.14844 0.005 0 0 0 0.005 0 0 +2.22656 0.005 0 0 0 0.005 0 0 +2.30469 0.005 0 0 0 0.005 0 0 +2.38281 0.005 0 0 0 0.005 0 0 +2.46094 0.005 0 0 0 0.005 0 0 +2.53906 0.005 0 0 0 0.005 0 0 +2.61719 0.005 0 0 0 0.005 0 0 +2.69531 0.005 0 0 0 0.005 0 0 +2.77344 0.005 0 0 0 0.005 0 0 +2.85156 0.005 0 0 0 0.005 0 0 +2.92969 0.005 0 0 0 0.005 0 0 +3.00781 0.005 0 0 0 0.005 0 0 +3.08594 0.005 0 0 0 0.005 0 0 +3.16406 0.005 0 0 0 0.005 0 0 +3.24219 0.005 0 0 0 0.005 0 0 +3.32031 0.005 0 0 0 0.005 0 0 +3.39844 0.005 0 0 0 0.005 0 0 +3.47656 0.005 0 0 0 0.005 0 0 +3.55469 0.005 0 0 0 0.005 0 0 +3.63281 0.005 0 0 0 0.005 0 0 +3.71094 0.00490073 0.00441906 0 2.16566e-005 0.00490073 0.0201542 0.000362943 +3.78906 0.00470863 0.0130996 0 6.16813e-005 0.00470863 0.0609504 0.000729255 +3.86719 0.00452038 0.0217802 0 9.84546e-005 0.00452038 0.103428 0.000996019 +3.94531 0.00433596 0.0304607 0 0.000132076 0.00433596 0.147694 0.00121151 +4.02344 0.00415538 0.0391413 0 0.000162647 0.00415538 0.193863 0.0013919 +4.10156 0.00397865 0.0478218 0 0.000190266 0.00397865 0.242061 0.00154532 +4.17969 0.00380575 0.0565024 0 0.000215034 0.00380575 0.292423 0.00167667 +4.25781 0.0036367 0.065183 0 0.000237051 0.0036367 0.345101 0.00178925 +4.33594 0.00347148 0.0738635 0 0.000256416 0.00347148 0.400256 0.00188541 +4.41406 0.00331011 0.0825441 0 0.00027323 0.00331011 0.458068 0.00196696 +4.49219 0.00315257 0.0912246 0 0.000287592 0.00315257 0.518734 0.0020353 +4.57031 0.00299888 0.0999052 0 0.000299604 0.00299888 0.58247 0.00209158 +4.64844 0.00284903 0.108586 0 0.000309364 0.00284903 0.649516 0.00213677 +4.72656 0.00270302 0.117266 0 0.000316973 0.00270302 0.720135 0.00217166 +4.80469 0.00256085 0.125947 0 0.000322531 0.00256085 0.794623 0.00219697 +4.88281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03906 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11719 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19531 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27344 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35156 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42969 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50781 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58594 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66406 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74219 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82031 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89844 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97656 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05469 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.28906 0.001 0 0 0 0.001 0 0 +6.36719 0.001 0 0 0 0.001 0 0 +6.44531 0.001 0 0 0 0.001 0 0 +6.52344 0.001 0 0 0 0.001 0 0 +6.60156 0.001 0 0 0 0.001 0 0 +6.67969 0.001 0 0 0 0.001 0 0 +6.75781 0.001 0 0 0 0.001 0 0 +6.83594 0.001 0 0 0 0.001 0 0 +6.91406 0.001 0 0 0 0.001 0 0 +6.99219 0.001 0 0 0 0.001 0 0 +7.07031 0.001 0 0 0 0.001 0 0 +7.14844 0.001 0 0 0 0.001 0 0 +7.22656 0.001 0 0 0 0.001 0 0 +7.30469 0.001 0 0 0 0.001 0 0 +7.38281 0.001 0 0 0 0.001 0 0 +7.46094 0.001 0 0 0 0.001 0 0 +7.53906 0.001 0 0 0 0.001 0 0 +7.61719 0.001 0 0 0 0.001 0 0 +7.69531 0.001 0 0 0 0.001 0 0 +7.77344 0.001 0 0 0 0.001 0 0 +7.85156 0.001 0 0 0 0.001 0 0 +7.92969 0.001 0 0 0 0.001 0 0 +8.00781 0.001 0 0 0 0.001 0 0 +8.08594 0.001 0 0 0 0.001 0 0 +8.16406 0.001 0 0 0 0.001 0 0 +8.24219 0.001 0 0 0 0.001 0 0 +8.32031 0.001 0 0 0 0.001 0 0 +8.39844 0.001 0 0 0 0.001 0 0 +8.47656 0.001 0 0 0 0.001 0 0 +8.55469 0.001 0 0 0 0.001 0 0 +8.63281 0.001 0 0 0 0.001 0 0 +8.71094 0.001 0 0 0 0.001 0 0 +8.78906 0.001 0 0 0 0.001 0 0 +8.86719 0.001 0 0 0 0.001 0 0 +8.94531 0.001 0 0 0 0.001 0 0 +9.02344 0.001 0 0 0 0.001 0 0 +9.10156 0.001 0 0 0 0.001 0 0 +9.17969 0.001 0 0 0 0.001 0 0 +9.25781 0.001 0 0 0 0.001 0 0 +9.33594 0.001 0 0 0 0.001 0 0 +9.41406 0.001 0 0 0 0.001 0 0 +9.49219 0.001 0 0 0 0.001 0 0 +9.57031 0.001 0 0 0 0.001 0 0 +9.64844 0.001 0 0 0 0.001 0 0 +9.72656 0.001 0 0 0 0.001 0 0 +9.80469 0.001 0 0 0 0.001 0 0 +9.88281 0.001 0 0 0 0.001 0 0 +9.96094 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=2048.csv b/reference/swashes_1_nx=2048.csv new file mode 100644 index 0000000..184b240 --- /dev/null +++ b/reference/swashes_1_nx=2048.csv @@ -0,0 +1,2066 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00488281 meters +# Number of cells: 2048 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00244141 0.005 0 0 0 0.005 0 0 +0.00732422 0.005 0 0 0 0.005 0 0 +0.012207 0.005 0 0 0 0.005 0 0 +0.0170898 0.005 0 0 0 0.005 0 0 +0.0219727 0.005 0 0 0 0.005 0 0 +0.0268555 0.005 0 0 0 0.005 0 0 +0.0317383 0.005 0 0 0 0.005 0 0 +0.0366211 0.005 0 0 0 0.005 0 0 +0.0415039 0.005 0 0 0 0.005 0 0 +0.0463867 0.005 0 0 0 0.005 0 0 +0.0512695 0.005 0 0 0 0.005 0 0 +0.0561523 0.005 0 0 0 0.005 0 0 +0.0610352 0.005 0 0 0 0.005 0 0 +0.065918 0.005 0 0 0 0.005 0 0 +0.0708008 0.005 0 0 0 0.005 0 0 +0.0756836 0.005 0 0 0 0.005 0 0 +0.0805664 0.005 0 0 0 0.005 0 0 +0.0854492 0.005 0 0 0 0.005 0 0 +0.090332 0.005 0 0 0 0.005 0 0 +0.0952148 0.005 0 0 0 0.005 0 0 +0.100098 0.005 0 0 0 0.005 0 0 +0.10498 0.005 0 0 0 0.005 0 0 +0.109863 0.005 0 0 0 0.005 0 0 +0.114746 0.005 0 0 0 0.005 0 0 +0.119629 0.005 0 0 0 0.005 0 0 +0.124512 0.005 0 0 0 0.005 0 0 +0.129395 0.005 0 0 0 0.005 0 0 +0.134277 0.005 0 0 0 0.005 0 0 +0.13916 0.005 0 0 0 0.005 0 0 +0.144043 0.005 0 0 0 0.005 0 0 +0.148926 0.005 0 0 0 0.005 0 0 +0.153809 0.005 0 0 0 0.005 0 0 +0.158691 0.005 0 0 0 0.005 0 0 +0.163574 0.005 0 0 0 0.005 0 0 +0.168457 0.005 0 0 0 0.005 0 0 +0.17334 0.005 0 0 0 0.005 0 0 +0.178223 0.005 0 0 0 0.005 0 0 +0.183105 0.005 0 0 0 0.005 0 0 +0.187988 0.005 0 0 0 0.005 0 0 +0.192871 0.005 0 0 0 0.005 0 0 +0.197754 0.005 0 0 0 0.005 0 0 +0.202637 0.005 0 0 0 0.005 0 0 +0.20752 0.005 0 0 0 0.005 0 0 +0.212402 0.005 0 0 0 0.005 0 0 +0.217285 0.005 0 0 0 0.005 0 0 +0.222168 0.005 0 0 0 0.005 0 0 +0.227051 0.005 0 0 0 0.005 0 0 +0.231934 0.005 0 0 0 0.005 0 0 +0.236816 0.005 0 0 0 0.005 0 0 +0.241699 0.005 0 0 0 0.005 0 0 +0.246582 0.005 0 0 0 0.005 0 0 +0.251465 0.005 0 0 0 0.005 0 0 +0.256348 0.005 0 0 0 0.005 0 0 +0.26123 0.005 0 0 0 0.005 0 0 +0.266113 0.005 0 0 0 0.005 0 0 +0.270996 0.005 0 0 0 0.005 0 0 +0.275879 0.005 0 0 0 0.005 0 0 +0.280762 0.005 0 0 0 0.005 0 0 +0.285645 0.005 0 0 0 0.005 0 0 +0.290527 0.005 0 0 0 0.005 0 0 +0.29541 0.005 0 0 0 0.005 0 0 +0.300293 0.005 0 0 0 0.005 0 0 +0.305176 0.005 0 0 0 0.005 0 0 +0.310059 0.005 0 0 0 0.005 0 0 +0.314941 0.005 0 0 0 0.005 0 0 +0.319824 0.005 0 0 0 0.005 0 0 +0.324707 0.005 0 0 0 0.005 0 0 +0.32959 0.005 0 0 0 0.005 0 0 +0.334473 0.005 0 0 0 0.005 0 0 +0.339355 0.005 0 0 0 0.005 0 0 +0.344238 0.005 0 0 0 0.005 0 0 +0.349121 0.005 0 0 0 0.005 0 0 +0.354004 0.005 0 0 0 0.005 0 0 +0.358887 0.005 0 0 0 0.005 0 0 +0.36377 0.005 0 0 0 0.005 0 0 +0.368652 0.005 0 0 0 0.005 0 0 +0.373535 0.005 0 0 0 0.005 0 0 +0.378418 0.005 0 0 0 0.005 0 0 +0.383301 0.005 0 0 0 0.005 0 0 +0.388184 0.005 0 0 0 0.005 0 0 +0.393066 0.005 0 0 0 0.005 0 0 +0.397949 0.005 0 0 0 0.005 0 0 +0.402832 0.005 0 0 0 0.005 0 0 +0.407715 0.005 0 0 0 0.005 0 0 +0.412598 0.005 0 0 0 0.005 0 0 +0.41748 0.005 0 0 0 0.005 0 0 +0.422363 0.005 0 0 0 0.005 0 0 +0.427246 0.005 0 0 0 0.005 0 0 +0.432129 0.005 0 0 0 0.005 0 0 +0.437012 0.005 0 0 0 0.005 0 0 +0.441895 0.005 0 0 0 0.005 0 0 +0.446777 0.005 0 0 0 0.005 0 0 +0.45166 0.005 0 0 0 0.005 0 0 +0.456543 0.005 0 0 0 0.005 0 0 +0.461426 0.005 0 0 0 0.005 0 0 +0.466309 0.005 0 0 0 0.005 0 0 +0.471191 0.005 0 0 0 0.005 0 0 +0.476074 0.005 0 0 0 0.005 0 0 +0.480957 0.005 0 0 0 0.005 0 0 +0.48584 0.005 0 0 0 0.005 0 0 +0.490723 0.005 0 0 0 0.005 0 0 +0.495605 0.005 0 0 0 0.005 0 0 +0.500488 0.005 0 0 0 0.005 0 0 +0.505371 0.005 0 0 0 0.005 0 0 +0.510254 0.005 0 0 0 0.005 0 0 +0.515137 0.005 0 0 0 0.005 0 0 +0.52002 0.005 0 0 0 0.005 0 0 +0.524902 0.005 0 0 0 0.005 0 0 +0.529785 0.005 0 0 0 0.005 0 0 +0.534668 0.005 0 0 0 0.005 0 0 +0.539551 0.005 0 0 0 0.005 0 0 +0.544434 0.005 0 0 0 0.005 0 0 +0.549316 0.005 0 0 0 0.005 0 0 +0.554199 0.005 0 0 0 0.005 0 0 +0.559082 0.005 0 0 0 0.005 0 0 +0.563965 0.005 0 0 0 0.005 0 0 +0.568848 0.005 0 0 0 0.005 0 0 +0.57373 0.005 0 0 0 0.005 0 0 +0.578613 0.005 0 0 0 0.005 0 0 +0.583496 0.005 0 0 0 0.005 0 0 +0.588379 0.005 0 0 0 0.005 0 0 +0.593262 0.005 0 0 0 0.005 0 0 +0.598145 0.005 0 0 0 0.005 0 0 +0.603027 0.005 0 0 0 0.005 0 0 +0.60791 0.005 0 0 0 0.005 0 0 +0.612793 0.005 0 0 0 0.005 0 0 +0.617676 0.005 0 0 0 0.005 0 0 +0.622559 0.005 0 0 0 0.005 0 0 +0.627441 0.005 0 0 0 0.005 0 0 +0.632324 0.005 0 0 0 0.005 0 0 +0.637207 0.005 0 0 0 0.005 0 0 +0.64209 0.005 0 0 0 0.005 0 0 +0.646973 0.005 0 0 0 0.005 0 0 +0.651855 0.005 0 0 0 0.005 0 0 +0.656738 0.005 0 0 0 0.005 0 0 +0.661621 0.005 0 0 0 0.005 0 0 +0.666504 0.005 0 0 0 0.005 0 0 +0.671387 0.005 0 0 0 0.005 0 0 +0.67627 0.005 0 0 0 0.005 0 0 +0.681152 0.005 0 0 0 0.005 0 0 +0.686035 0.005 0 0 0 0.005 0 0 +0.690918 0.005 0 0 0 0.005 0 0 +0.695801 0.005 0 0 0 0.005 0 0 +0.700684 0.005 0 0 0 0.005 0 0 +0.705566 0.005 0 0 0 0.005 0 0 +0.710449 0.005 0 0 0 0.005 0 0 +0.715332 0.005 0 0 0 0.005 0 0 +0.720215 0.005 0 0 0 0.005 0 0 +0.725098 0.005 0 0 0 0.005 0 0 +0.72998 0.005 0 0 0 0.005 0 0 +0.734863 0.005 0 0 0 0.005 0 0 +0.739746 0.005 0 0 0 0.005 0 0 +0.744629 0.005 0 0 0 0.005 0 0 +0.749512 0.005 0 0 0 0.005 0 0 +0.754395 0.005 0 0 0 0.005 0 0 +0.759277 0.005 0 0 0 0.005 0 0 +0.76416 0.005 0 0 0 0.005 0 0 +0.769043 0.005 0 0 0 0.005 0 0 +0.773926 0.005 0 0 0 0.005 0 0 +0.778809 0.005 0 0 0 0.005 0 0 +0.783691 0.005 0 0 0 0.005 0 0 +0.788574 0.005 0 0 0 0.005 0 0 +0.793457 0.005 0 0 0 0.005 0 0 +0.79834 0.005 0 0 0 0.005 0 0 +0.803223 0.005 0 0 0 0.005 0 0 +0.808105 0.005 0 0 0 0.005 0 0 +0.812988 0.005 0 0 0 0.005 0 0 +0.817871 0.005 0 0 0 0.005 0 0 +0.822754 0.005 0 0 0 0.005 0 0 +0.827637 0.005 0 0 0 0.005 0 0 +0.83252 0.005 0 0 0 0.005 0 0 +0.837402 0.005 0 0 0 0.005 0 0 +0.842285 0.005 0 0 0 0.005 0 0 +0.847168 0.005 0 0 0 0.005 0 0 +0.852051 0.005 0 0 0 0.005 0 0 +0.856934 0.005 0 0 0 0.005 0 0 +0.861816 0.005 0 0 0 0.005 0 0 +0.866699 0.005 0 0 0 0.005 0 0 +0.871582 0.005 0 0 0 0.005 0 0 +0.876465 0.005 0 0 0 0.005 0 0 +0.881348 0.005 0 0 0 0.005 0 0 +0.88623 0.005 0 0 0 0.005 0 0 +0.891113 0.005 0 0 0 0.005 0 0 +0.895996 0.005 0 0 0 0.005 0 0 +0.900879 0.005 0 0 0 0.005 0 0 +0.905762 0.005 0 0 0 0.005 0 0 +0.910645 0.005 0 0 0 0.005 0 0 +0.915527 0.005 0 0 0 0.005 0 0 +0.92041 0.005 0 0 0 0.005 0 0 +0.925293 0.005 0 0 0 0.005 0 0 +0.930176 0.005 0 0 0 0.005 0 0 +0.935059 0.005 0 0 0 0.005 0 0 +0.939941 0.005 0 0 0 0.005 0 0 +0.944824 0.005 0 0 0 0.005 0 0 +0.949707 0.005 0 0 0 0.005 0 0 +0.95459 0.005 0 0 0 0.005 0 0 +0.959473 0.005 0 0 0 0.005 0 0 +0.964355 0.005 0 0 0 0.005 0 0 +0.969238 0.005 0 0 0 0.005 0 0 +0.974121 0.005 0 0 0 0.005 0 0 +0.979004 0.005 0 0 0 0.005 0 0 +0.983887 0.005 0 0 0 0.005 0 0 +0.98877 0.005 0 0 0 0.005 0 0 +0.993652 0.005 0 0 0 0.005 0 0 +0.998535 0.005 0 0 0 0.005 0 0 +1.00342 0.005 0 0 0 0.005 0 0 +1.0083 0.005 0 0 0 0.005 0 0 +1.01318 0.005 0 0 0 0.005 0 0 +1.01807 0.005 0 0 0 0.005 0 0 +1.02295 0.005 0 0 0 0.005 0 0 +1.02783 0.005 0 0 0 0.005 0 0 +1.03271 0.005 0 0 0 0.005 0 0 +1.0376 0.005 0 0 0 0.005 0 0 +1.04248 0.005 0 0 0 0.005 0 0 +1.04736 0.005 0 0 0 0.005 0 0 +1.05225 0.005 0 0 0 0.005 0 0 +1.05713 0.005 0 0 0 0.005 0 0 +1.06201 0.005 0 0 0 0.005 0 0 +1.06689 0.005 0 0 0 0.005 0 0 +1.07178 0.005 0 0 0 0.005 0 0 +1.07666 0.005 0 0 0 0.005 0 0 +1.08154 0.005 0 0 0 0.005 0 0 +1.08643 0.005 0 0 0 0.005 0 0 +1.09131 0.005 0 0 0 0.005 0 0 +1.09619 0.005 0 0 0 0.005 0 0 +1.10107 0.005 0 0 0 0.005 0 0 +1.10596 0.005 0 0 0 0.005 0 0 +1.11084 0.005 0 0 0 0.005 0 0 +1.11572 0.005 0 0 0 0.005 0 0 +1.12061 0.005 0 0 0 0.005 0 0 +1.12549 0.005 0 0 0 0.005 0 0 +1.13037 0.005 0 0 0 0.005 0 0 +1.13525 0.005 0 0 0 0.005 0 0 +1.14014 0.005 0 0 0 0.005 0 0 +1.14502 0.005 0 0 0 0.005 0 0 +1.1499 0.005 0 0 0 0.005 0 0 +1.15479 0.005 0 0 0 0.005 0 0 +1.15967 0.005 0 0 0 0.005 0 0 +1.16455 0.005 0 0 0 0.005 0 0 +1.16943 0.005 0 0 0 0.005 0 0 +1.17432 0.005 0 0 0 0.005 0 0 +1.1792 0.005 0 0 0 0.005 0 0 +1.18408 0.005 0 0 0 0.005 0 0 +1.18896 0.005 0 0 0 0.005 0 0 +1.19385 0.005 0 0 0 0.005 0 0 +1.19873 0.005 0 0 0 0.005 0 0 +1.20361 0.005 0 0 0 0.005 0 0 +1.2085 0.005 0 0 0 0.005 0 0 +1.21338 0.005 0 0 0 0.005 0 0 +1.21826 0.005 0 0 0 0.005 0 0 +1.22314 0.005 0 0 0 0.005 0 0 +1.22803 0.005 0 0 0 0.005 0 0 +1.23291 0.005 0 0 0 0.005 0 0 +1.23779 0.005 0 0 0 0.005 0 0 +1.24268 0.005 0 0 0 0.005 0 0 +1.24756 0.005 0 0 0 0.005 0 0 +1.25244 0.005 0 0 0 0.005 0 0 +1.25732 0.005 0 0 0 0.005 0 0 +1.26221 0.005 0 0 0 0.005 0 0 +1.26709 0.005 0 0 0 0.005 0 0 +1.27197 0.005 0 0 0 0.005 0 0 +1.27686 0.005 0 0 0 0.005 0 0 +1.28174 0.005 0 0 0 0.005 0 0 +1.28662 0.005 0 0 0 0.005 0 0 +1.2915 0.005 0 0 0 0.005 0 0 +1.29639 0.005 0 0 0 0.005 0 0 +1.30127 0.005 0 0 0 0.005 0 0 +1.30615 0.005 0 0 0 0.005 0 0 +1.31104 0.005 0 0 0 0.005 0 0 +1.31592 0.005 0 0 0 0.005 0 0 +1.3208 0.005 0 0 0 0.005 0 0 +1.32568 0.005 0 0 0 0.005 0 0 +1.33057 0.005 0 0 0 0.005 0 0 +1.33545 0.005 0 0 0 0.005 0 0 +1.34033 0.005 0 0 0 0.005 0 0 +1.34521 0.005 0 0 0 0.005 0 0 +1.3501 0.005 0 0 0 0.005 0 0 +1.35498 0.005 0 0 0 0.005 0 0 +1.35986 0.005 0 0 0 0.005 0 0 +1.36475 0.005 0 0 0 0.005 0 0 +1.36963 0.005 0 0 0 0.005 0 0 +1.37451 0.005 0 0 0 0.005 0 0 +1.37939 0.005 0 0 0 0.005 0 0 +1.38428 0.005 0 0 0 0.005 0 0 +1.38916 0.005 0 0 0 0.005 0 0 +1.39404 0.005 0 0 0 0.005 0 0 +1.39893 0.005 0 0 0 0.005 0 0 +1.40381 0.005 0 0 0 0.005 0 0 +1.40869 0.005 0 0 0 0.005 0 0 +1.41357 0.005 0 0 0 0.005 0 0 +1.41846 0.005 0 0 0 0.005 0 0 +1.42334 0.005 0 0 0 0.005 0 0 +1.42822 0.005 0 0 0 0.005 0 0 +1.43311 0.005 0 0 0 0.005 0 0 +1.43799 0.005 0 0 0 0.005 0 0 +1.44287 0.005 0 0 0 0.005 0 0 +1.44775 0.005 0 0 0 0.005 0 0 +1.45264 0.005 0 0 0 0.005 0 0 +1.45752 0.005 0 0 0 0.005 0 0 +1.4624 0.005 0 0 0 0.005 0 0 +1.46729 0.005 0 0 0 0.005 0 0 +1.47217 0.005 0 0 0 0.005 0 0 +1.47705 0.005 0 0 0 0.005 0 0 +1.48193 0.005 0 0 0 0.005 0 0 +1.48682 0.005 0 0 0 0.005 0 0 +1.4917 0.005 0 0 0 0.005 0 0 +1.49658 0.005 0 0 0 0.005 0 0 +1.50146 0.005 0 0 0 0.005 0 0 +1.50635 0.005 0 0 0 0.005 0 0 +1.51123 0.005 0 0 0 0.005 0 0 +1.51611 0.005 0 0 0 0.005 0 0 +1.521 0.005 0 0 0 0.005 0 0 +1.52588 0.005 0 0 0 0.005 0 0 +1.53076 0.005 0 0 0 0.005 0 0 +1.53564 0.005 0 0 0 0.005 0 0 +1.54053 0.005 0 0 0 0.005 0 0 +1.54541 0.005 0 0 0 0.005 0 0 +1.55029 0.005 0 0 0 0.005 0 0 +1.55518 0.005 0 0 0 0.005 0 0 +1.56006 0.005 0 0 0 0.005 0 0 +1.56494 0.005 0 0 0 0.005 0 0 +1.56982 0.005 0 0 0 0.005 0 0 +1.57471 0.005 0 0 0 0.005 0 0 +1.57959 0.005 0 0 0 0.005 0 0 +1.58447 0.005 0 0 0 0.005 0 0 +1.58936 0.005 0 0 0 0.005 0 0 +1.59424 0.005 0 0 0 0.005 0 0 +1.59912 0.005 0 0 0 0.005 0 0 +1.604 0.005 0 0 0 0.005 0 0 +1.60889 0.005 0 0 0 0.005 0 0 +1.61377 0.005 0 0 0 0.005 0 0 +1.61865 0.005 0 0 0 0.005 0 0 +1.62354 0.005 0 0 0 0.005 0 0 +1.62842 0.005 0 0 0 0.005 0 0 +1.6333 0.005 0 0 0 0.005 0 0 +1.63818 0.005 0 0 0 0.005 0 0 +1.64307 0.005 0 0 0 0.005 0 0 +1.64795 0.005 0 0 0 0.005 0 0 +1.65283 0.005 0 0 0 0.005 0 0 +1.65771 0.005 0 0 0 0.005 0 0 +1.6626 0.005 0 0 0 0.005 0 0 +1.66748 0.005 0 0 0 0.005 0 0 +1.67236 0.005 0 0 0 0.005 0 0 +1.67725 0.005 0 0 0 0.005 0 0 +1.68213 0.005 0 0 0 0.005 0 0 +1.68701 0.005 0 0 0 0.005 0 0 +1.69189 0.005 0 0 0 0.005 0 0 +1.69678 0.005 0 0 0 0.005 0 0 +1.70166 0.005 0 0 0 0.005 0 0 +1.70654 0.005 0 0 0 0.005 0 0 +1.71143 0.005 0 0 0 0.005 0 0 +1.71631 0.005 0 0 0 0.005 0 0 +1.72119 0.005 0 0 0 0.005 0 0 +1.72607 0.005 0 0 0 0.005 0 0 +1.73096 0.005 0 0 0 0.005 0 0 +1.73584 0.005 0 0 0 0.005 0 0 +1.74072 0.005 0 0 0 0.005 0 0 +1.74561 0.005 0 0 0 0.005 0 0 +1.75049 0.005 0 0 0 0.005 0 0 +1.75537 0.005 0 0 0 0.005 0 0 +1.76025 0.005 0 0 0 0.005 0 0 +1.76514 0.005 0 0 0 0.005 0 0 +1.77002 0.005 0 0 0 0.005 0 0 +1.7749 0.005 0 0 0 0.005 0 0 +1.77979 0.005 0 0 0 0.005 0 0 +1.78467 0.005 0 0 0 0.005 0 0 +1.78955 0.005 0 0 0 0.005 0 0 +1.79443 0.005 0 0 0 0.005 0 0 +1.79932 0.005 0 0 0 0.005 0 0 +1.8042 0.005 0 0 0 0.005 0 0 +1.80908 0.005 0 0 0 0.005 0 0 +1.81396 0.005 0 0 0 0.005 0 0 +1.81885 0.005 0 0 0 0.005 0 0 +1.82373 0.005 0 0 0 0.005 0 0 +1.82861 0.005 0 0 0 0.005 0 0 +1.8335 0.005 0 0 0 0.005 0 0 +1.83838 0.005 0 0 0 0.005 0 0 +1.84326 0.005 0 0 0 0.005 0 0 +1.84814 0.005 0 0 0 0.005 0 0 +1.85303 0.005 0 0 0 0.005 0 0 +1.85791 0.005 0 0 0 0.005 0 0 +1.86279 0.005 0 0 0 0.005 0 0 +1.86768 0.005 0 0 0 0.005 0 0 +1.87256 0.005 0 0 0 0.005 0 0 +1.87744 0.005 0 0 0 0.005 0 0 +1.88232 0.005 0 0 0 0.005 0 0 +1.88721 0.005 0 0 0 0.005 0 0 +1.89209 0.005 0 0 0 0.005 0 0 +1.89697 0.005 0 0 0 0.005 0 0 +1.90186 0.005 0 0 0 0.005 0 0 +1.90674 0.005 0 0 0 0.005 0 0 +1.91162 0.005 0 0 0 0.005 0 0 +1.9165 0.005 0 0 0 0.005 0 0 +1.92139 0.005 0 0 0 0.005 0 0 +1.92627 0.005 0 0 0 0.005 0 0 +1.93115 0.005 0 0 0 0.005 0 0 +1.93604 0.005 0 0 0 0.005 0 0 +1.94092 0.005 0 0 0 0.005 0 0 +1.9458 0.005 0 0 0 0.005 0 0 +1.95068 0.005 0 0 0 0.005 0 0 +1.95557 0.005 0 0 0 0.005 0 0 +1.96045 0.005 0 0 0 0.005 0 0 +1.96533 0.005 0 0 0 0.005 0 0 +1.97021 0.005 0 0 0 0.005 0 0 +1.9751 0.005 0 0 0 0.005 0 0 +1.97998 0.005 0 0 0 0.005 0 0 +1.98486 0.005 0 0 0 0.005 0 0 +1.98975 0.005 0 0 0 0.005 0 0 +1.99463 0.005 0 0 0 0.005 0 0 +1.99951 0.005 0 0 0 0.005 0 0 +2.00439 0.005 0 0 0 0.005 0 0 +2.00928 0.005 0 0 0 0.005 0 0 +2.01416 0.005 0 0 0 0.005 0 0 +2.01904 0.005 0 0 0 0.005 0 0 +2.02393 0.005 0 0 0 0.005 0 0 +2.02881 0.005 0 0 0 0.005 0 0 +2.03369 0.005 0 0 0 0.005 0 0 +2.03857 0.005 0 0 0 0.005 0 0 +2.04346 0.005 0 0 0 0.005 0 0 +2.04834 0.005 0 0 0 0.005 0 0 +2.05322 0.005 0 0 0 0.005 0 0 +2.05811 0.005 0 0 0 0.005 0 0 +2.06299 0.005 0 0 0 0.005 0 0 +2.06787 0.005 0 0 0 0.005 0 0 +2.07275 0.005 0 0 0 0.005 0 0 +2.07764 0.005 0 0 0 0.005 0 0 +2.08252 0.005 0 0 0 0.005 0 0 +2.0874 0.005 0 0 0 0.005 0 0 +2.09229 0.005 0 0 0 0.005 0 0 +2.09717 0.005 0 0 0 0.005 0 0 +2.10205 0.005 0 0 0 0.005 0 0 +2.10693 0.005 0 0 0 0.005 0 0 +2.11182 0.005 0 0 0 0.005 0 0 +2.1167 0.005 0 0 0 0.005 0 0 +2.12158 0.005 0 0 0 0.005 0 0 +2.12646 0.005 0 0 0 0.005 0 0 +2.13135 0.005 0 0 0 0.005 0 0 +2.13623 0.005 0 0 0 0.005 0 0 +2.14111 0.005 0 0 0 0.005 0 0 +2.146 0.005 0 0 0 0.005 0 0 +2.15088 0.005 0 0 0 0.005 0 0 +2.15576 0.005 0 0 0 0.005 0 0 +2.16064 0.005 0 0 0 0.005 0 0 +2.16553 0.005 0 0 0 0.005 0 0 +2.17041 0.005 0 0 0 0.005 0 0 +2.17529 0.005 0 0 0 0.005 0 0 +2.18018 0.005 0 0 0 0.005 0 0 +2.18506 0.005 0 0 0 0.005 0 0 +2.18994 0.005 0 0 0 0.005 0 0 +2.19482 0.005 0 0 0 0.005 0 0 +2.19971 0.005 0 0 0 0.005 0 0 +2.20459 0.005 0 0 0 0.005 0 0 +2.20947 0.005 0 0 0 0.005 0 0 +2.21436 0.005 0 0 0 0.005 0 0 +2.21924 0.005 0 0 0 0.005 0 0 +2.22412 0.005 0 0 0 0.005 0 0 +2.229 0.005 0 0 0 0.005 0 0 +2.23389 0.005 0 0 0 0.005 0 0 +2.23877 0.005 0 0 0 0.005 0 0 +2.24365 0.005 0 0 0 0.005 0 0 +2.24854 0.005 0 0 0 0.005 0 0 +2.25342 0.005 0 0 0 0.005 0 0 +2.2583 0.005 0 0 0 0.005 0 0 +2.26318 0.005 0 0 0 0.005 0 0 +2.26807 0.005 0 0 0 0.005 0 0 +2.27295 0.005 0 0 0 0.005 0 0 +2.27783 0.005 0 0 0 0.005 0 0 +2.28271 0.005 0 0 0 0.005 0 0 +2.2876 0.005 0 0 0 0.005 0 0 +2.29248 0.005 0 0 0 0.005 0 0 +2.29736 0.005 0 0 0 0.005 0 0 +2.30225 0.005 0 0 0 0.005 0 0 +2.30713 0.005 0 0 0 0.005 0 0 +2.31201 0.005 0 0 0 0.005 0 0 +2.31689 0.005 0 0 0 0.005 0 0 +2.32178 0.005 0 0 0 0.005 0 0 +2.32666 0.005 0 0 0 0.005 0 0 +2.33154 0.005 0 0 0 0.005 0 0 +2.33643 0.005 0 0 0 0.005 0 0 +2.34131 0.005 0 0 0 0.005 0 0 +2.34619 0.005 0 0 0 0.005 0 0 +2.35107 0.005 0 0 0 0.005 0 0 +2.35596 0.005 0 0 0 0.005 0 0 +2.36084 0.005 0 0 0 0.005 0 0 +2.36572 0.005 0 0 0 0.005 0 0 +2.37061 0.005 0 0 0 0.005 0 0 +2.37549 0.005 0 0 0 0.005 0 0 +2.38037 0.005 0 0 0 0.005 0 0 +2.38525 0.005 0 0 0 0.005 0 0 +2.39014 0.005 0 0 0 0.005 0 0 +2.39502 0.005 0 0 0 0.005 0 0 +2.3999 0.005 0 0 0 0.005 0 0 +2.40479 0.005 0 0 0 0.005 0 0 +2.40967 0.005 0 0 0 0.005 0 0 +2.41455 0.005 0 0 0 0.005 0 0 +2.41943 0.005 0 0 0 0.005 0 0 +2.42432 0.005 0 0 0 0.005 0 0 +2.4292 0.005 0 0 0 0.005 0 0 +2.43408 0.005 0 0 0 0.005 0 0 +2.43896 0.005 0 0 0 0.005 0 0 +2.44385 0.005 0 0 0 0.005 0 0 +2.44873 0.005 0 0 0 0.005 0 0 +2.45361 0.005 0 0 0 0.005 0 0 +2.4585 0.005 0 0 0 0.005 0 0 +2.46338 0.005 0 0 0 0.005 0 0 +2.46826 0.005 0 0 0 0.005 0 0 +2.47314 0.005 0 0 0 0.005 0 0 +2.47803 0.005 0 0 0 0.005 0 0 +2.48291 0.005 0 0 0 0.005 0 0 +2.48779 0.005 0 0 0 0.005 0 0 +2.49268 0.005 0 0 0 0.005 0 0 +2.49756 0.005 0 0 0 0.005 0 0 +2.50244 0.005 0 0 0 0.005 0 0 +2.50732 0.005 0 0 0 0.005 0 0 +2.51221 0.005 0 0 0 0.005 0 0 +2.51709 0.005 0 0 0 0.005 0 0 +2.52197 0.005 0 0 0 0.005 0 0 +2.52686 0.005 0 0 0 0.005 0 0 +2.53174 0.005 0 0 0 0.005 0 0 +2.53662 0.005 0 0 0 0.005 0 0 +2.5415 0.005 0 0 0 0.005 0 0 +2.54639 0.005 0 0 0 0.005 0 0 +2.55127 0.005 0 0 0 0.005 0 0 +2.55615 0.005 0 0 0 0.005 0 0 +2.56104 0.005 0 0 0 0.005 0 0 +2.56592 0.005 0 0 0 0.005 0 0 +2.5708 0.005 0 0 0 0.005 0 0 +2.57568 0.005 0 0 0 0.005 0 0 +2.58057 0.005 0 0 0 0.005 0 0 +2.58545 0.005 0 0 0 0.005 0 0 +2.59033 0.005 0 0 0 0.005 0 0 +2.59521 0.005 0 0 0 0.005 0 0 +2.6001 0.005 0 0 0 0.005 0 0 +2.60498 0.005 0 0 0 0.005 0 0 +2.60986 0.005 0 0 0 0.005 0 0 +2.61475 0.005 0 0 0 0.005 0 0 +2.61963 0.005 0 0 0 0.005 0 0 +2.62451 0.005 0 0 0 0.005 0 0 +2.62939 0.005 0 0 0 0.005 0 0 +2.63428 0.005 0 0 0 0.005 0 0 +2.63916 0.005 0 0 0 0.005 0 0 +2.64404 0.005 0 0 0 0.005 0 0 +2.64893 0.005 0 0 0 0.005 0 0 +2.65381 0.005 0 0 0 0.005 0 0 +2.65869 0.005 0 0 0 0.005 0 0 +2.66357 0.005 0 0 0 0.005 0 0 +2.66846 0.005 0 0 0 0.005 0 0 +2.67334 0.005 0 0 0 0.005 0 0 +2.67822 0.005 0 0 0 0.005 0 0 +2.68311 0.005 0 0 0 0.005 0 0 +2.68799 0.005 0 0 0 0.005 0 0 +2.69287 0.005 0 0 0 0.005 0 0 +2.69775 0.005 0 0 0 0.005 0 0 +2.70264 0.005 0 0 0 0.005 0 0 +2.70752 0.005 0 0 0 0.005 0 0 +2.7124 0.005 0 0 0 0.005 0 0 +2.71729 0.005 0 0 0 0.005 0 0 +2.72217 0.005 0 0 0 0.005 0 0 +2.72705 0.005 0 0 0 0.005 0 0 +2.73193 0.005 0 0 0 0.005 0 0 +2.73682 0.005 0 0 0 0.005 0 0 +2.7417 0.005 0 0 0 0.005 0 0 +2.74658 0.005 0 0 0 0.005 0 0 +2.75146 0.005 0 0 0 0.005 0 0 +2.75635 0.005 0 0 0 0.005 0 0 +2.76123 0.005 0 0 0 0.005 0 0 +2.76611 0.005 0 0 0 0.005 0 0 +2.771 0.005 0 0 0 0.005 0 0 +2.77588 0.005 0 0 0 0.005 0 0 +2.78076 0.005 0 0 0 0.005 0 0 +2.78564 0.005 0 0 0 0.005 0 0 +2.79053 0.005 0 0 0 0.005 0 0 +2.79541 0.005 0 0 0 0.005 0 0 +2.80029 0.005 0 0 0 0.005 0 0 +2.80518 0.005 0 0 0 0.005 0 0 +2.81006 0.005 0 0 0 0.005 0 0 +2.81494 0.005 0 0 0 0.005 0 0 +2.81982 0.005 0 0 0 0.005 0 0 +2.82471 0.005 0 0 0 0.005 0 0 +2.82959 0.005 0 0 0 0.005 0 0 +2.83447 0.005 0 0 0 0.005 0 0 +2.83936 0.005 0 0 0 0.005 0 0 +2.84424 0.005 0 0 0 0.005 0 0 +2.84912 0.005 0 0 0 0.005 0 0 +2.854 0.005 0 0 0 0.005 0 0 +2.85889 0.005 0 0 0 0.005 0 0 +2.86377 0.005 0 0 0 0.005 0 0 +2.86865 0.005 0 0 0 0.005 0 0 +2.87354 0.005 0 0 0 0.005 0 0 +2.87842 0.005 0 0 0 0.005 0 0 +2.8833 0.005 0 0 0 0.005 0 0 +2.88818 0.005 0 0 0 0.005 0 0 +2.89307 0.005 0 0 0 0.005 0 0 +2.89795 0.005 0 0 0 0.005 0 0 +2.90283 0.005 0 0 0 0.005 0 0 +2.90771 0.005 0 0 0 0.005 0 0 +2.9126 0.005 0 0 0 0.005 0 0 +2.91748 0.005 0 0 0 0.005 0 0 +2.92236 0.005 0 0 0 0.005 0 0 +2.92725 0.005 0 0 0 0.005 0 0 +2.93213 0.005 0 0 0 0.005 0 0 +2.93701 0.005 0 0 0 0.005 0 0 +2.94189 0.005 0 0 0 0.005 0 0 +2.94678 0.005 0 0 0 0.005 0 0 +2.95166 0.005 0 0 0 0.005 0 0 +2.95654 0.005 0 0 0 0.005 0 0 +2.96143 0.005 0 0 0 0.005 0 0 +2.96631 0.005 0 0 0 0.005 0 0 +2.97119 0.005 0 0 0 0.005 0 0 +2.97607 0.005 0 0 0 0.005 0 0 +2.98096 0.005 0 0 0 0.005 0 0 +2.98584 0.005 0 0 0 0.005 0 0 +2.99072 0.005 0 0 0 0.005 0 0 +2.99561 0.005 0 0 0 0.005 0 0 +3.00049 0.005 0 0 0 0.005 0 0 +3.00537 0.005 0 0 0 0.005 0 0 +3.01025 0.005 0 0 0 0.005 0 0 +3.01514 0.005 0 0 0 0.005 0 0 +3.02002 0.005 0 0 0 0.005 0 0 +3.0249 0.005 0 0 0 0.005 0 0 +3.02979 0.005 0 0 0 0.005 0 0 +3.03467 0.005 0 0 0 0.005 0 0 +3.03955 0.005 0 0 0 0.005 0 0 +3.04443 0.005 0 0 0 0.005 0 0 +3.04932 0.005 0 0 0 0.005 0 0 +3.0542 0.005 0 0 0 0.005 0 0 +3.05908 0.005 0 0 0 0.005 0 0 +3.06396 0.005 0 0 0 0.005 0 0 +3.06885 0.005 0 0 0 0.005 0 0 +3.07373 0.005 0 0 0 0.005 0 0 +3.07861 0.005 0 0 0 0.005 0 0 +3.0835 0.005 0 0 0 0.005 0 0 +3.08838 0.005 0 0 0 0.005 0 0 +3.09326 0.005 0 0 0 0.005 0 0 +3.09814 0.005 0 0 0 0.005 0 0 +3.10303 0.005 0 0 0 0.005 0 0 +3.10791 0.005 0 0 0 0.005 0 0 +3.11279 0.005 0 0 0 0.005 0 0 +3.11768 0.005 0 0 0 0.005 0 0 +3.12256 0.005 0 0 0 0.005 0 0 +3.12744 0.005 0 0 0 0.005 0 0 +3.13232 0.005 0 0 0 0.005 0 0 +3.13721 0.005 0 0 0 0.005 0 0 +3.14209 0.005 0 0 0 0.005 0 0 +3.14697 0.005 0 0 0 0.005 0 0 +3.15186 0.005 0 0 0 0.005 0 0 +3.15674 0.005 0 0 0 0.005 0 0 +3.16162 0.005 0 0 0 0.005 0 0 +3.1665 0.005 0 0 0 0.005 0 0 +3.17139 0.005 0 0 0 0.005 0 0 +3.17627 0.005 0 0 0 0.005 0 0 +3.18115 0.005 0 0 0 0.005 0 0 +3.18604 0.005 0 0 0 0.005 0 0 +3.19092 0.005 0 0 0 0.005 0 0 +3.1958 0.005 0 0 0 0.005 0 0 +3.20068 0.005 0 0 0 0.005 0 0 +3.20557 0.005 0 0 0 0.005 0 0 +3.21045 0.005 0 0 0 0.005 0 0 +3.21533 0.005 0 0 0 0.005 0 0 +3.22021 0.005 0 0 0 0.005 0 0 +3.2251 0.005 0 0 0 0.005 0 0 +3.22998 0.005 0 0 0 0.005 0 0 +3.23486 0.005 0 0 0 0.005 0 0 +3.23975 0.005 0 0 0 0.005 0 0 +3.24463 0.005 0 0 0 0.005 0 0 +3.24951 0.005 0 0 0 0.005 0 0 +3.25439 0.005 0 0 0 0.005 0 0 +3.25928 0.005 0 0 0 0.005 0 0 +3.26416 0.005 0 0 0 0.005 0 0 +3.26904 0.005 0 0 0 0.005 0 0 +3.27393 0.005 0 0 0 0.005 0 0 +3.27881 0.005 0 0 0 0.005 0 0 +3.28369 0.005 0 0 0 0.005 0 0 +3.28857 0.005 0 0 0 0.005 0 0 +3.29346 0.005 0 0 0 0.005 0 0 +3.29834 0.005 0 0 0 0.005 0 0 +3.30322 0.005 0 0 0 0.005 0 0 +3.30811 0.005 0 0 0 0.005 0 0 +3.31299 0.005 0 0 0 0.005 0 0 +3.31787 0.005 0 0 0 0.005 0 0 +3.32275 0.005 0 0 0 0.005 0 0 +3.32764 0.005 0 0 0 0.005 0 0 +3.33252 0.005 0 0 0 0.005 0 0 +3.3374 0.005 0 0 0 0.005 0 0 +3.34229 0.005 0 0 0 0.005 0 0 +3.34717 0.005 0 0 0 0.005 0 0 +3.35205 0.005 0 0 0 0.005 0 0 +3.35693 0.005 0 0 0 0.005 0 0 +3.36182 0.005 0 0 0 0.005 0 0 +3.3667 0.005 0 0 0 0.005 0 0 +3.37158 0.005 0 0 0 0.005 0 0 +3.37646 0.005 0 0 0 0.005 0 0 +3.38135 0.005 0 0 0 0.005 0 0 +3.38623 0.005 0 0 0 0.005 0 0 +3.39111 0.005 0 0 0 0.005 0 0 +3.396 0.005 0 0 0 0.005 0 0 +3.40088 0.005 0 0 0 0.005 0 0 +3.40576 0.005 0 0 0 0.005 0 0 +3.41064 0.005 0 0 0 0.005 0 0 +3.41553 0.005 0 0 0 0.005 0 0 +3.42041 0.005 0 0 0 0.005 0 0 +3.42529 0.005 0 0 0 0.005 0 0 +3.43018 0.005 0 0 0 0.005 0 0 +3.43506 0.005 0 0 0 0.005 0 0 +3.43994 0.005 0 0 0 0.005 0 0 +3.44482 0.005 0 0 0 0.005 0 0 +3.44971 0.005 0 0 0 0.005 0 0 +3.45459 0.005 0 0 0 0.005 0 0 +3.45947 0.005 0 0 0 0.005 0 0 +3.46436 0.005 0 0 0 0.005 0 0 +3.46924 0.005 0 0 0 0.005 0 0 +3.47412 0.005 0 0 0 0.005 0 0 +3.479 0.005 0 0 0 0.005 0 0 +3.48389 0.005 0 0 0 0.005 0 0 +3.48877 0.005 0 0 0 0.005 0 0 +3.49365 0.005 0 0 0 0.005 0 0 +3.49854 0.005 0 0 0 0.005 0 0 +3.50342 0.005 0 0 0 0.005 0 0 +3.5083 0.005 0 0 0 0.005 0 0 +3.51318 0.005 0 0 0 0.005 0 0 +3.51807 0.005 0 0 0 0.005 0 0 +3.52295 0.005 0 0 0 0.005 0 0 +3.52783 0.005 0 0 0 0.005 0 0 +3.53271 0.005 0 0 0 0.005 0 0 +3.5376 0.005 0 0 0 0.005 0 0 +3.54248 0.005 0 0 0 0.005 0 0 +3.54736 0.005 0 0 0 0.005 0 0 +3.55225 0.005 0 0 0 0.005 0 0 +3.55713 0.005 0 0 0 0.005 0 0 +3.56201 0.005 0 0 0 0.005 0 0 +3.56689 0.005 0 0 0 0.005 0 0 +3.57178 0.005 0 0 0 0.005 0 0 +3.57666 0.005 0 0 0 0.005 0 0 +3.58154 0.005 0 0 0 0.005 0 0 +3.58643 0.005 0 0 0 0.005 0 0 +3.59131 0.005 0 0 0 0.005 0 0 +3.59619 0.005 0 0 0 0.005 0 0 +3.60107 0.005 0 0 0 0.005 0 0 +3.60596 0.005 0 0 0 0.005 0 0 +3.61084 0.005 0 0 0 0.005 0 0 +3.61572 0.005 0 0 0 0.005 0 0 +3.62061 0.005 0 0 0 0.005 0 0 +3.62549 0.005 0 0 0 0.005 0 0 +3.63037 0.005 0 0 0 0.005 0 0 +3.63525 0.005 0 0 0 0.005 0 0 +3.64014 0.005 0 0 0 0.005 0 0 +3.64502 0.005 0 0 0 0.005 0 0 +3.6499 0.005 0 0 0 0.005 0 0 +3.65479 0.005 0 0 0 0.005 0 0 +3.65967 0.005 0 0 0 0.005 0 0 +3.66455 0.005 0 0 0 0.005 0 0 +3.66943 0.005 0 0 0 0.005 0 0 +3.67432 0.0049921 0.000350054 0 1.7475e-006 0.0049921 0.00158182 6.77728e-005 +3.6792 0.00497987 0.000892588 0 4.44497e-006 0.00497987 0.00403838 0.000126286 +3.68408 0.00496765 0.00143512 0 7.12919e-006 0.00496765 0.00650098 0.000173036 +3.68896 0.00495545 0.00197766 0 9.80019e-006 0.00495545 0.00896964 0.000213927 +3.69385 0.00494327 0.00252019 0 1.2458e-005 0.00494327 0.0114444 0.00025104 +3.69873 0.00493109 0.00306273 0 1.51026e-005 0.00493109 0.0139252 0.000285416 +3.70361 0.00491894 0.00360526 0 1.77341e-005 0.00491894 0.0164122 0.000317675 +3.7085 0.0049068 0.0041478 0 2.03524e-005 0.0049068 0.0189053 0.00034822 +3.71338 0.00489467 0.00469033 0 2.29576e-005 0.00489467 0.0214046 0.000377336 +3.71826 0.00488256 0.00523287 0 2.55498e-005 0.00488256 0.0239101 0.00040523 +3.72314 0.00487046 0.0057754 0 2.81289e-005 0.00487046 0.0264218 0.000432061 +3.72803 0.00485838 0.00631794 0 3.06949e-005 0.00485838 0.0289398 0.000457954 +3.73291 0.00484632 0.00686047 0 3.3248e-005 0.00484632 0.031464 0.000483008 +3.73779 0.00483427 0.007403 0 3.57881e-005 0.00483427 0.0339945 0.000507305 +3.74268 0.00482223 0.00794554 0 3.83152e-005 0.00482223 0.0365313 0.000530914 +3.74756 0.00481021 0.00848807 0 4.08294e-005 0.00481021 0.0390744 0.000553893 +3.75244 0.0047982 0.00903061 0 4.33307e-005 0.0047982 0.041624 0.000576289 +3.75732 0.00478621 0.00957314 0 4.58191e-005 0.00478621 0.0441798 0.000598147 +3.76221 0.00477423 0.0101157 0 4.82946e-005 0.00477423 0.0467421 0.000619502 +3.76709 0.00476227 0.0106582 0 5.07573e-005 0.00476227 0.0493109 0.000640387 +3.77197 0.00475033 0.0112007 0 5.32072e-005 0.00475033 0.0518861 0.000660831 +3.77686 0.0047384 0.0117433 0 5.56443e-005 0.0047384 0.0544677 0.00068086 +3.78174 0.00472648 0.0122858 0 5.80687e-005 0.00472648 0.0570559 0.000700495 +3.78662 0.00471458 0.0128284 0 6.04803e-005 0.00471458 0.0596506 0.000719757 +3.7915 0.00470269 0.0133709 0 6.28792e-005 0.00470269 0.0622519 0.000738666 +3.79639 0.00469082 0.0139134 0 6.52654e-005 0.00469082 0.0648597 0.000757238 +3.80127 0.00467897 0.014456 0 6.76389e-005 0.00467897 0.0674741 0.000775487 +3.80615 0.00466712 0.0149985 0 6.99998e-005 0.00466712 0.0700952 0.000793429 +3.81104 0.0046553 0.015541 0 7.23481e-005 0.0046553 0.0727229 0.000811076 +3.81592 0.00464349 0.0160836 0 7.46838e-005 0.00464349 0.0753573 0.00082844 +3.8208 0.00463169 0.0166261 0 7.70069e-005 0.00463169 0.0779985 0.000845532 +3.82568 0.00461991 0.0171686 0 7.93175e-005 0.00461991 0.0806463 0.000862362 +3.83057 0.00460814 0.0177112 0 8.16156e-005 0.00460814 0.0833009 0.00087894 +3.83545 0.00459639 0.0182537 0 8.39012e-005 0.00459639 0.0859623 0.000895273 +3.84033 0.00458466 0.0187962 0 8.61743e-005 0.00458466 0.0886304 0.000911371 +3.84521 0.00457294 0.0193388 0 8.84349e-005 0.00457294 0.0913055 0.000927242 +3.8501 0.00456123 0.0198813 0 9.06832e-005 0.00456123 0.0939874 0.000942891 +3.85498 0.00454954 0.0204238 0 9.2919e-005 0.00454954 0.0966761 0.000958326 +3.85986 0.00453786 0.0209664 0 9.51425e-005 0.00453786 0.0993718 0.000973554 +3.86475 0.0045262 0.0215089 0 9.73536e-005 0.0045262 0.102074 0.00098858 +3.86963 0.00451455 0.0220514 0 9.95524e-005 0.00451455 0.104784 0.00100341 +3.87451 0.00450292 0.022594 0 0.000101739 0.00450292 0.107501 0.00101805 +3.87939 0.00449131 0.0231365 0 0.000103913 0.00449131 0.110224 0.0010325 +3.88428 0.00447971 0.023679 0 0.000106075 0.00447971 0.112955 0.00104677 +3.88916 0.00446812 0.0242216 0 0.000108225 0.00446812 0.115693 0.00106087 +3.89404 0.00445655 0.0247641 0 0.000110363 0.00445655 0.118437 0.00107479 +3.89893 0.00444499 0.0253067 0 0.000112488 0.00444499 0.121189 0.00108855 +3.90381 0.00443345 0.0258492 0 0.000114601 0.00443345 0.123949 0.00110214 +3.90869 0.00442193 0.0263917 0 0.000116702 0.00442193 0.126715 0.00111557 +3.91357 0.00441041 0.0269343 0 0.000118791 0.00441041 0.129488 0.00112884 +3.91846 0.00439892 0.0274768 0 0.000120868 0.00439892 0.132269 0.00114196 +3.92334 0.00438744 0.0280193 0 0.000122933 0.00438744 0.135057 0.00115493 +3.92822 0.00437597 0.0285619 0 0.000124986 0.00437597 0.137853 0.00116775 +3.93311 0.00436452 0.0291044 0 0.000127027 0.00436452 0.140655 0.00118043 +3.93799 0.00435308 0.0296469 0 0.000129056 0.00435308 0.143465 0.00119297 +3.94287 0.00434166 0.0301895 0 0.000131072 0.00434166 0.146283 0.00120536 +3.94775 0.00433026 0.030732 0 0.000133077 0.00433026 0.149107 0.00121762 +3.95264 0.00431887 0.0312745 0 0.000135071 0.00431887 0.15194 0.00122975 +3.95752 0.00430749 0.0318171 0 0.000137052 0.00430749 0.15478 0.00124175 +3.9624 0.00429613 0.0323596 0 0.000139021 0.00429613 0.157627 0.00125361 +3.96729 0.00428478 0.0329021 0 0.000140979 0.00428478 0.160482 0.00126535 +3.97217 0.00427345 0.0334447 0 0.000142924 0.00427345 0.163344 0.00127697 +3.97705 0.00426214 0.0339872 0 0.000144858 0.00426214 0.166214 0.00128846 +3.98193 0.00425084 0.0345297 0 0.00014678 0.00425084 0.169091 0.00129984 +3.98682 0.00423955 0.0350723 0 0.000148691 0.00423955 0.171977 0.00131109 +3.9917 0.00422828 0.0356148 0 0.000150589 0.00422828 0.17487 0.00132223 +3.99658 0.00421702 0.0361573 0 0.000152476 0.00421702 0.17777 0.00133325 +4.00146 0.00420578 0.0366999 0 0.000154352 0.00420578 0.180679 0.00134416 +4.00635 0.00419455 0.0372424 0 0.000156215 0.00419455 0.183595 0.00135496 +4.01123 0.00418334 0.0377849 0 0.000158067 0.00418334 0.186519 0.00136565 +4.01611 0.00417215 0.0383275 0 0.000159908 0.00417215 0.189451 0.00137623 +4.021 0.00416097 0.03887 0 0.000161737 0.00416097 0.19239 0.0013867 +4.02588 0.0041498 0.0394126 0 0.000163554 0.0041498 0.195338 0.00139707 +4.03076 0.00413865 0.0399551 0 0.00016536 0.00413865 0.198293 0.00140734 +4.03564 0.00412751 0.0404976 0 0.000167154 0.00412751 0.201257 0.0014175 +4.04053 0.00411639 0.0410402 0 0.000168937 0.00411639 0.204228 0.00142756 +4.04541 0.00410529 0.0415827 0 0.000170709 0.00410529 0.207208 0.00143752 +4.05029 0.0040942 0.0421252 0 0.000172469 0.0040942 0.210196 0.00144739 +4.05518 0.00408312 0.0426678 0 0.000174218 0.00408312 0.213191 0.00145715 +4.06006 0.00407206 0.0432103 0 0.000175955 0.00407206 0.216195 0.00146682 +4.06494 0.00406101 0.0437528 0 0.000177681 0.00406101 0.219207 0.0014764 +4.06982 0.00404998 0.0442954 0 0.000179395 0.00404998 0.222227 0.00148588 +4.07471 0.00403897 0.0448379 0 0.000181099 0.00403897 0.225256 0.00149527 +4.07959 0.00402796 0.0453804 0 0.000182791 0.00402796 0.228292 0.00150457 +4.08447 0.00401698 0.045923 0 0.000184472 0.00401698 0.231337 0.00151378 +4.08936 0.00400601 0.0464655 0 0.000186141 0.00400601 0.234391 0.0015229 +4.09424 0.00399505 0.047008 0 0.0001878 0.00399505 0.237452 0.00153193 +4.09912 0.00398411 0.0475506 0 0.000189447 0.00398411 0.240522 0.00154088 +4.104 0.00397318 0.0480931 0 0.000191083 0.00397318 0.243601 0.00154974 +4.10889 0.00396227 0.0486356 0 0.000192708 0.00396227 0.246688 0.00155851 +4.11377 0.00395138 0.0491782 0 0.000194322 0.00395138 0.249783 0.0015672 +4.11865 0.0039405 0.0497207 0 0.000195924 0.0039405 0.252887 0.00157581 +4.12354 0.00392963 0.0502632 0 0.000197516 0.00392963 0.256 0.00158433 +4.12842 0.00391878 0.0508058 0 0.000199097 0.00391878 0.259121 0.00159277 +4.1333 0.00390794 0.0513483 0 0.000200666 0.00390794 0.262251 0.00160113 +4.13818 0.00389712 0.0518909 0 0.000202225 0.00389712 0.26539 0.00160941 +4.14307 0.00388632 0.0524334 0 0.000203773 0.00388632 0.268537 0.00161761 +4.14795 0.00387553 0.0529759 0 0.00020531 0.00387553 0.271693 0.00162574 +4.15283 0.00386475 0.0535185 0 0.000206835 0.00386475 0.274858 0.00163378 +4.15771 0.00385399 0.054061 0 0.00020835 0.00385399 0.278032 0.00164175 +4.1626 0.00384324 0.0546035 0 0.000209855 0.00384324 0.281214 0.00164964 +4.16748 0.00383251 0.0551461 0 0.000211348 0.00383251 0.284406 0.00165746 +4.17236 0.0038218 0.0556886 0 0.00021283 0.0038218 0.287606 0.0016652 +4.17725 0.0038111 0.0562311 0 0.000214302 0.0038111 0.290815 0.00167287 +4.18213 0.00380041 0.0567737 0 0.000215763 0.00380041 0.294034 0.00168046 +4.18701 0.00378974 0.0573162 0 0.000217213 0.00378974 0.297261 0.00168798 +4.19189 0.00377908 0.0578587 0 0.000218653 0.00377908 0.300498 0.00169543 +4.19678 0.00376844 0.0584013 0 0.000220082 0.00376844 0.303743 0.00170281 +4.20166 0.00375782 0.0589438 0 0.0002215 0.00375782 0.306998 0.00171012 +4.20654 0.0037472 0.0594863 0 0.000222907 0.0037472 0.310262 0.00171736 +4.21143 0.00373661 0.0600289 0 0.000224304 0.00373661 0.313536 0.00172452 +4.21631 0.00372603 0.0605714 0 0.000225691 0.00372603 0.316818 0.00173162 +4.22119 0.00371546 0.0611139 0 0.000227067 0.00371546 0.32011 0.00173865 +4.22607 0.00370491 0.0616565 0 0.000228432 0.00370491 0.323411 0.00174561 +4.23096 0.00369437 0.062199 0 0.000229786 0.00369437 0.326722 0.00175251 +4.23584 0.00368385 0.0627415 0 0.000231131 0.00368385 0.330042 0.00175934 +4.24072 0.00367335 0.0632841 0 0.000232464 0.00367335 0.333372 0.0017661 +4.24561 0.00366286 0.0638266 0 0.000233788 0.00366286 0.336711 0.00177279 +4.25049 0.00365238 0.0643692 0 0.000235101 0.00365238 0.34006 0.00177942 +4.25537 0.00364192 0.0649117 0 0.000236403 0.00364192 0.343418 0.00178599 +4.26025 0.00363147 0.0654542 0 0.000237695 0.00363147 0.346786 0.00179249 +4.26514 0.00362104 0.0659968 0 0.000238977 0.00362104 0.350164 0.00179893 +4.27002 0.00361063 0.0665393 0 0.000240249 0.00361063 0.353551 0.00180531 +4.2749 0.00360023 0.0670818 0 0.00024151 0.00360023 0.356948 0.00181162 +4.27979 0.00358984 0.0676244 0 0.000242761 0.00358984 0.360355 0.00181787 +4.28467 0.00357947 0.0681669 0 0.000244001 0.00357947 0.363772 0.00182406 +4.28955 0.00356911 0.0687094 0 0.000245232 0.00356911 0.367199 0.00183018 +4.29443 0.00355877 0.069252 0 0.000246452 0.00355877 0.370636 0.00183625 +4.29932 0.00354845 0.0697945 0 0.000247662 0.00354845 0.374083 0.00184226 +4.3042 0.00353814 0.070337 0 0.000248862 0.00353814 0.377539 0.0018482 +4.30908 0.00352784 0.0708796 0 0.000250052 0.00352784 0.381006 0.00185409 +4.31396 0.00351756 0.0714221 0 0.000251231 0.00351756 0.384483 0.00185991 +4.31885 0.00350729 0.0719646 0 0.000252401 0.00350729 0.38797 0.00186568 +4.32373 0.00349704 0.0725072 0 0.000253561 0.00349704 0.391468 0.00187139 +4.32861 0.00348681 0.0730497 0 0.00025471 0.00348681 0.394975 0.00187704 +4.3335 0.00347659 0.0735922 0 0.00025585 0.00347659 0.398493 0.00188264 +4.33838 0.00346638 0.0741348 0 0.000256979 0.00346638 0.402022 0.00188818 +4.34326 0.00345619 0.0746773 0 0.000258099 0.00345619 0.40556 0.00189366 +4.34814 0.00344601 0.0752198 0 0.000259209 0.00344601 0.409109 0.00189908 +4.35303 0.00343585 0.0757624 0 0.000260308 0.00343585 0.412669 0.00190445 +4.35791 0.00342571 0.0763049 0 0.000261398 0.00342571 0.416239 0.00190976 +4.36279 0.00341558 0.0768474 0 0.000262478 0.00341558 0.41982 0.00191502 +4.36768 0.00340546 0.07739 0 0.000263548 0.00340546 0.423411 0.00192022 +4.37256 0.00339536 0.0779325 0 0.000264609 0.00339536 0.427013 0.00192537 +4.37744 0.00338527 0.0784751 0 0.000265659 0.00338527 0.430626 0.00193046 +4.38232 0.0033752 0.0790176 0 0.0002667 0.0033752 0.43425 0.0019355 +4.38721 0.00336515 0.0795601 0 0.000267731 0.00336515 0.437884 0.00194048 +4.39209 0.00335511 0.0801027 0 0.000268753 0.00335511 0.441529 0.00194542 +4.39697 0.00334508 0.0806452 0 0.000269765 0.00334508 0.445185 0.00195029 +4.40186 0.00333507 0.0811877 0 0.000270767 0.00333507 0.448852 0.00195512 +4.40674 0.00332507 0.0817303 0 0.000271759 0.00332507 0.45253 0.0019599 +4.41162 0.00331509 0.0822728 0 0.000272742 0.00331509 0.45622 0.00196462 +4.4165 0.00330513 0.0828153 0 0.000273715 0.00330513 0.45992 0.00196929 +4.42139 0.00329518 0.0833579 0 0.000274679 0.00329518 0.463631 0.00197391 +4.42627 0.00328524 0.0839004 0 0.000275633 0.00328524 0.467354 0.00197848 +4.43115 0.00327532 0.0844429 0 0.000276578 0.00327532 0.471088 0.00198299 +4.43604 0.00326541 0.0849855 0 0.000277513 0.00326541 0.474833 0.00198746 +4.44092 0.00325552 0.085528 0 0.000278438 0.00325552 0.47859 0.00199188 +4.4458 0.00324565 0.0860705 0 0.000279355 0.00324565 0.482358 0.00199625 +4.45068 0.00323579 0.0866131 0 0.000280261 0.00323579 0.486137 0.00200056 +4.45557 0.00322594 0.0871556 0 0.000281159 0.00322594 0.489929 0.00200483 +4.46045 0.00321611 0.0876981 0 0.000282047 0.00321611 0.493731 0.00200905 +4.46533 0.00320629 0.0882407 0 0.000282925 0.00320629 0.497545 0.00201322 +4.47021 0.00319649 0.0887832 0 0.000283795 0.00319649 0.501371 0.00201734 +4.4751 0.00318671 0.0893257 0 0.000284655 0.00318671 0.505209 0.00202142 +4.47998 0.00317694 0.0898683 0 0.000285506 0.00317694 0.509059 0.00202544 +4.48486 0.00316718 0.0904108 0 0.000286347 0.00316718 0.51292 0.00202942 +4.48975 0.00315744 0.0909534 0 0.00028718 0.00315744 0.516793 0.00203335 +4.49463 0.00314771 0.0914959 0 0.000288003 0.00314771 0.520678 0.00203724 +4.49951 0.003138 0.0920384 0 0.000288817 0.003138 0.524576 0.00204107 +4.50439 0.00312831 0.092581 0 0.000289622 0.00312831 0.528485 0.00204486 +4.50928 0.00311863 0.0931235 0 0.000290417 0.00311863 0.532406 0.00204861 +4.51416 0.00310896 0.093666 0 0.000291204 0.00310896 0.53634 0.0020523 +4.51904 0.00309931 0.0942086 0 0.000291981 0.00309931 0.540286 0.00205596 +4.52393 0.00308967 0.0947511 0 0.00029275 0.00308967 0.544244 0.00205956 +4.52881 0.00308005 0.0952936 0 0.000293509 0.00308005 0.548214 0.00206312 +4.53369 0.00307045 0.0958362 0 0.00029426 0.00307045 0.552197 0.00206664 +4.53857 0.00306086 0.0963787 0 0.000295001 0.00306086 0.556192 0.00207011 +4.54346 0.00305128 0.0969212 0 0.000295734 0.00305128 0.5602 0.00207353 +4.54834 0.00304172 0.0974638 0 0.000296457 0.00304172 0.564221 0.00207691 +4.55322 0.00303217 0.0980063 0 0.000297172 0.00303217 0.568254 0.00208025 +4.55811 0.00302264 0.0985488 0 0.000297878 0.00302264 0.5723 0.00208354 +4.56299 0.00301313 0.0990914 0 0.000298575 0.00301313 0.576358 0.00208679 +4.56787 0.00300363 0.0996339 0 0.000299263 0.00300363 0.58043 0.00209 +4.57275 0.00299414 0.100176 0 0.000299942 0.00299414 0.584514 0.00209316 +4.57764 0.00298467 0.100719 0 0.000300613 0.00298467 0.588611 0.00209628 +4.58252 0.00297521 0.101262 0 0.000301275 0.00297521 0.592722 0.00209935 +4.5874 0.00296577 0.101804 0 0.000301928 0.00296577 0.596845 0.00210239 +4.59229 0.00295635 0.102347 0 0.000302572 0.00295635 0.600981 0.00210538 +4.59717 0.00294694 0.102889 0 0.000303208 0.00294694 0.605131 0.00210832 +4.60205 0.00293754 0.103432 0 0.000303835 0.00293754 0.609294 0.00211123 +4.60693 0.00292816 0.103974 0 0.000304453 0.00292816 0.61347 0.00211409 +4.61182 0.00291879 0.104517 0 0.000305063 0.00291879 0.61766 0.00211691 +4.6167 0.00290944 0.105059 0 0.000305664 0.00290944 0.621863 0.00211969 +4.62158 0.00290011 0.105602 0 0.000306257 0.00290011 0.62608 0.00212243 +4.62646 0.00289079 0.106144 0 0.000306841 0.00289079 0.63031 0.00212513 +4.63135 0.00288148 0.106687 0 0.000307416 0.00288148 0.634554 0.00212779 +4.63623 0.00287219 0.107229 0 0.000307983 0.00287219 0.638811 0.0021304 +4.64111 0.00286291 0.107772 0 0.000308542 0.00286291 0.643083 0.00213298 +4.646 0.00285365 0.108314 0 0.000309092 0.00285365 0.647368 0.00213551 +4.65088 0.00284441 0.108857 0 0.000309634 0.00284441 0.651667 0.00213801 +4.65576 0.00283518 0.1094 0 0.000310167 0.00283518 0.65598 0.00214046 +4.66064 0.00282596 0.109942 0 0.000310692 0.00282596 0.660308 0.00214288 +4.66553 0.00281676 0.110485 0 0.000311209 0.00281676 0.664649 0.00214525 +4.67041 0.00280758 0.111027 0 0.000311717 0.00280758 0.669004 0.00214759 +4.67529 0.0027984 0.11157 0 0.000312217 0.0027984 0.673374 0.00214988 +4.68018 0.00278925 0.112112 0 0.000312709 0.00278925 0.677758 0.00215214 +4.68506 0.00278011 0.112655 0 0.000313192 0.00278011 0.682157 0.00215436 +4.68994 0.00277098 0.113197 0 0.000313668 0.00277098 0.68657 0.00215654 +4.69482 0.00276187 0.11374 0 0.000314135 0.00276187 0.690997 0.00215868 +4.69971 0.00275278 0.114282 0 0.000314594 0.00275278 0.695439 0.00216078 +4.70459 0.0027437 0.114825 0 0.000315045 0.0027437 0.699896 0.00216284 +4.70947 0.00273463 0.115367 0 0.000315487 0.00273463 0.704368 0.00216487 +4.71436 0.00272558 0.11591 0 0.000315922 0.00272558 0.708854 0.00216686 +4.71924 0.00271654 0.116452 0 0.000316348 0.00271654 0.713355 0.00216881 +4.72412 0.00270752 0.116995 0 0.000316767 0.00270752 0.717872 0.00217072 +4.729 0.00269852 0.117538 0 0.000317177 0.00269852 0.722403 0.00217259 +4.73389 0.00268953 0.11808 0 0.000317579 0.00268953 0.72695 0.00217443 +4.73877 0.00268055 0.118623 0 0.000317974 0.00268055 0.731511 0.00217623 +4.74365 0.00267159 0.119165 0 0.00031836 0.00267159 0.736088 0.00217799 +4.74854 0.00266264 0.119708 0 0.000318739 0.00266264 0.740681 0.00217972 +4.75342 0.00265371 0.12025 0 0.00031911 0.00265371 0.745288 0.00218141 +4.7583 0.0026448 0.120793 0 0.000319472 0.0026448 0.749912 0.00218306 +4.76318 0.0026359 0.121335 0 0.000319827 0.0026359 0.754551 0.00218468 +4.76807 0.00262701 0.121878 0 0.000320174 0.00262701 0.759205 0.00218626 +4.77295 0.00261814 0.12242 0 0.000320514 0.00261814 0.763876 0.0021878 +4.77783 0.00260929 0.122963 0 0.000320845 0.00260929 0.768562 0.00218931 +4.78271 0.00260044 0.123505 0 0.000321169 0.00260044 0.773264 0.00219078 +4.7876 0.00259162 0.124048 0 0.000321485 0.00259162 0.777982 0.00219222 +4.79248 0.00258281 0.124591 0 0.000321793 0.00258281 0.782716 0.00219362 +4.79736 0.00257401 0.125133 0 0.000322094 0.00257401 0.787467 0.00219499 +4.80225 0.00256523 0.125676 0 0.000322387 0.00256523 0.792233 0.00219632 +4.80713 0.00255647 0.126218 0 0.000322672 0.00255647 0.797016 0.00219761 +4.81201 0.00254772 0.126761 0 0.00032295 0.00254772 0.801816 0.00219888 +4.81689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01221 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02197 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02686 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03174 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03662 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04639 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05615 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06592 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07568 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08057 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08545 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09033 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10986 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11475 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11963 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12451 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13428 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13916 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14404 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14893 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15381 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15869 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16357 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16846 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17334 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17822 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18799 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19287 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19775 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20264 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20752 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22217 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22705 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23193 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23682 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24658 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25635 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26611 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27588 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28076 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28564 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29053 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30029 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30518 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31006 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31494 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31982 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32471 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33447 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33936 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34424 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34912 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35889 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36865 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37842 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38818 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39307 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39795 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40283 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41748 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42236 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42725 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43213 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43701 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44678 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45166 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45654 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46143 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46631 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47119 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47607 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48096 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49072 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50049 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50537 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51025 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51514 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53467 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53955 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54443 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54932 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55908 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56885 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57861 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58838 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59326 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59814 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60303 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61279 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61768 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62256 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62744 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63232 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63721 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64209 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64697 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65674 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66162 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67139 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67627 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68115 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69092 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70068 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70557 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71045 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71533 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72998 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73486 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73975 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74463 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74951 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75928 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76904 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77393 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77881 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78369 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78857 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79346 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80322 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80811 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81299 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81787 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82275 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82764 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84717 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85205 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85693 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86182 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87158 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88135 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88623 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89111 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90088 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90576 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91553 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92041 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92529 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93018 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93506 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93994 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94482 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94971 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95459 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95947 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96924 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97412 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98389 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99365 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00342 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01318 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01807 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02295 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02783 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04736 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05225 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05713 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06201 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26221 0.001 0 0 0 0.001 0 0 +6.26709 0.001 0 0 0 0.001 0 0 +6.27197 0.001 0 0 0 0.001 0 0 +6.27686 0.001 0 0 0 0.001 0 0 +6.28174 0.001 0 0 0 0.001 0 0 +6.28662 0.001 0 0 0 0.001 0 0 +6.2915 0.001 0 0 0 0.001 0 0 +6.29639 0.001 0 0 0 0.001 0 0 +6.30127 0.001 0 0 0 0.001 0 0 +6.30615 0.001 0 0 0 0.001 0 0 +6.31104 0.001 0 0 0 0.001 0 0 +6.31592 0.001 0 0 0 0.001 0 0 +6.3208 0.001 0 0 0 0.001 0 0 +6.32568 0.001 0 0 0 0.001 0 0 +6.33057 0.001 0 0 0 0.001 0 0 +6.33545 0.001 0 0 0 0.001 0 0 +6.34033 0.001 0 0 0 0.001 0 0 +6.34521 0.001 0 0 0 0.001 0 0 +6.3501 0.001 0 0 0 0.001 0 0 +6.35498 0.001 0 0 0 0.001 0 0 +6.35986 0.001 0 0 0 0.001 0 0 +6.36475 0.001 0 0 0 0.001 0 0 +6.36963 0.001 0 0 0 0.001 0 0 +6.37451 0.001 0 0 0 0.001 0 0 +6.37939 0.001 0 0 0 0.001 0 0 +6.38428 0.001 0 0 0 0.001 0 0 +6.38916 0.001 0 0 0 0.001 0 0 +6.39404 0.001 0 0 0 0.001 0 0 +6.39893 0.001 0 0 0 0.001 0 0 +6.40381 0.001 0 0 0 0.001 0 0 +6.40869 0.001 0 0 0 0.001 0 0 +6.41357 0.001 0 0 0 0.001 0 0 +6.41846 0.001 0 0 0 0.001 0 0 +6.42334 0.001 0 0 0 0.001 0 0 +6.42822 0.001 0 0 0 0.001 0 0 +6.43311 0.001 0 0 0 0.001 0 0 +6.43799 0.001 0 0 0 0.001 0 0 +6.44287 0.001 0 0 0 0.001 0 0 +6.44775 0.001 0 0 0 0.001 0 0 +6.45264 0.001 0 0 0 0.001 0 0 +6.45752 0.001 0 0 0 0.001 0 0 +6.4624 0.001 0 0 0 0.001 0 0 +6.46729 0.001 0 0 0 0.001 0 0 +6.47217 0.001 0 0 0 0.001 0 0 +6.47705 0.001 0 0 0 0.001 0 0 +6.48193 0.001 0 0 0 0.001 0 0 +6.48682 0.001 0 0 0 0.001 0 0 +6.4917 0.001 0 0 0 0.001 0 0 +6.49658 0.001 0 0 0 0.001 0 0 +6.50146 0.001 0 0 0 0.001 0 0 +6.50635 0.001 0 0 0 0.001 0 0 +6.51123 0.001 0 0 0 0.001 0 0 +6.51611 0.001 0 0 0 0.001 0 0 +6.521 0.001 0 0 0 0.001 0 0 +6.52588 0.001 0 0 0 0.001 0 0 +6.53076 0.001 0 0 0 0.001 0 0 +6.53564 0.001 0 0 0 0.001 0 0 +6.54053 0.001 0 0 0 0.001 0 0 +6.54541 0.001 0 0 0 0.001 0 0 +6.55029 0.001 0 0 0 0.001 0 0 +6.55518 0.001 0 0 0 0.001 0 0 +6.56006 0.001 0 0 0 0.001 0 0 +6.56494 0.001 0 0 0 0.001 0 0 +6.56982 0.001 0 0 0 0.001 0 0 +6.57471 0.001 0 0 0 0.001 0 0 +6.57959 0.001 0 0 0 0.001 0 0 +6.58447 0.001 0 0 0 0.001 0 0 +6.58936 0.001 0 0 0 0.001 0 0 +6.59424 0.001 0 0 0 0.001 0 0 +6.59912 0.001 0 0 0 0.001 0 0 +6.604 0.001 0 0 0 0.001 0 0 +6.60889 0.001 0 0 0 0.001 0 0 +6.61377 0.001 0 0 0 0.001 0 0 +6.61865 0.001 0 0 0 0.001 0 0 +6.62354 0.001 0 0 0 0.001 0 0 +6.62842 0.001 0 0 0 0.001 0 0 +6.6333 0.001 0 0 0 0.001 0 0 +6.63818 0.001 0 0 0 0.001 0 0 +6.64307 0.001 0 0 0 0.001 0 0 +6.64795 0.001 0 0 0 0.001 0 0 +6.65283 0.001 0 0 0 0.001 0 0 +6.65771 0.001 0 0 0 0.001 0 0 +6.6626 0.001 0 0 0 0.001 0 0 +6.66748 0.001 0 0 0 0.001 0 0 +6.67236 0.001 0 0 0 0.001 0 0 +6.67725 0.001 0 0 0 0.001 0 0 +6.68213 0.001 0 0 0 0.001 0 0 +6.68701 0.001 0 0 0 0.001 0 0 +6.69189 0.001 0 0 0 0.001 0 0 +6.69678 0.001 0 0 0 0.001 0 0 +6.70166 0.001 0 0 0 0.001 0 0 +6.70654 0.001 0 0 0 0.001 0 0 +6.71143 0.001 0 0 0 0.001 0 0 +6.71631 0.001 0 0 0 0.001 0 0 +6.72119 0.001 0 0 0 0.001 0 0 +6.72607 0.001 0 0 0 0.001 0 0 +6.73096 0.001 0 0 0 0.001 0 0 +6.73584 0.001 0 0 0 0.001 0 0 +6.74072 0.001 0 0 0 0.001 0 0 +6.74561 0.001 0 0 0 0.001 0 0 +6.75049 0.001 0 0 0 0.001 0 0 +6.75537 0.001 0 0 0 0.001 0 0 +6.76025 0.001 0 0 0 0.001 0 0 +6.76514 0.001 0 0 0 0.001 0 0 +6.77002 0.001 0 0 0 0.001 0 0 +6.7749 0.001 0 0 0 0.001 0 0 +6.77979 0.001 0 0 0 0.001 0 0 +6.78467 0.001 0 0 0 0.001 0 0 +6.78955 0.001 0 0 0 0.001 0 0 +6.79443 0.001 0 0 0 0.001 0 0 +6.79932 0.001 0 0 0 0.001 0 0 +6.8042 0.001 0 0 0 0.001 0 0 +6.80908 0.001 0 0 0 0.001 0 0 +6.81396 0.001 0 0 0 0.001 0 0 +6.81885 0.001 0 0 0 0.001 0 0 +6.82373 0.001 0 0 0 0.001 0 0 +6.82861 0.001 0 0 0 0.001 0 0 +6.8335 0.001 0 0 0 0.001 0 0 +6.83838 0.001 0 0 0 0.001 0 0 +6.84326 0.001 0 0 0 0.001 0 0 +6.84814 0.001 0 0 0 0.001 0 0 +6.85303 0.001 0 0 0 0.001 0 0 +6.85791 0.001 0 0 0 0.001 0 0 +6.86279 0.001 0 0 0 0.001 0 0 +6.86768 0.001 0 0 0 0.001 0 0 +6.87256 0.001 0 0 0 0.001 0 0 +6.87744 0.001 0 0 0 0.001 0 0 +6.88232 0.001 0 0 0 0.001 0 0 +6.88721 0.001 0 0 0 0.001 0 0 +6.89209 0.001 0 0 0 0.001 0 0 +6.89697 0.001 0 0 0 0.001 0 0 +6.90186 0.001 0 0 0 0.001 0 0 +6.90674 0.001 0 0 0 0.001 0 0 +6.91162 0.001 0 0 0 0.001 0 0 +6.9165 0.001 0 0 0 0.001 0 0 +6.92139 0.001 0 0 0 0.001 0 0 +6.92627 0.001 0 0 0 0.001 0 0 +6.93115 0.001 0 0 0 0.001 0 0 +6.93604 0.001 0 0 0 0.001 0 0 +6.94092 0.001 0 0 0 0.001 0 0 +6.9458 0.001 0 0 0 0.001 0 0 +6.95068 0.001 0 0 0 0.001 0 0 +6.95557 0.001 0 0 0 0.001 0 0 +6.96045 0.001 0 0 0 0.001 0 0 +6.96533 0.001 0 0 0 0.001 0 0 +6.97021 0.001 0 0 0 0.001 0 0 +6.9751 0.001 0 0 0 0.001 0 0 +6.97998 0.001 0 0 0 0.001 0 0 +6.98486 0.001 0 0 0 0.001 0 0 +6.98975 0.001 0 0 0 0.001 0 0 +6.99463 0.001 0 0 0 0.001 0 0 +6.99951 0.001 0 0 0 0.001 0 0 +7.00439 0.001 0 0 0 0.001 0 0 +7.00928 0.001 0 0 0 0.001 0 0 +7.01416 0.001 0 0 0 0.001 0 0 +7.01904 0.001 0 0 0 0.001 0 0 +7.02393 0.001 0 0 0 0.001 0 0 +7.02881 0.001 0 0 0 0.001 0 0 +7.03369 0.001 0 0 0 0.001 0 0 +7.03857 0.001 0 0 0 0.001 0 0 +7.04346 0.001 0 0 0 0.001 0 0 +7.04834 0.001 0 0 0 0.001 0 0 +7.05322 0.001 0 0 0 0.001 0 0 +7.05811 0.001 0 0 0 0.001 0 0 +7.06299 0.001 0 0 0 0.001 0 0 +7.06787 0.001 0 0 0 0.001 0 0 +7.07275 0.001 0 0 0 0.001 0 0 +7.07764 0.001 0 0 0 0.001 0 0 +7.08252 0.001 0 0 0 0.001 0 0 +7.0874 0.001 0 0 0 0.001 0 0 +7.09229 0.001 0 0 0 0.001 0 0 +7.09717 0.001 0 0 0 0.001 0 0 +7.10205 0.001 0 0 0 0.001 0 0 +7.10693 0.001 0 0 0 0.001 0 0 +7.11182 0.001 0 0 0 0.001 0 0 +7.1167 0.001 0 0 0 0.001 0 0 +7.12158 0.001 0 0 0 0.001 0 0 +7.12646 0.001 0 0 0 0.001 0 0 +7.13135 0.001 0 0 0 0.001 0 0 +7.13623 0.001 0 0 0 0.001 0 0 +7.14111 0.001 0 0 0 0.001 0 0 +7.146 0.001 0 0 0 0.001 0 0 +7.15088 0.001 0 0 0 0.001 0 0 +7.15576 0.001 0 0 0 0.001 0 0 +7.16064 0.001 0 0 0 0.001 0 0 +7.16553 0.001 0 0 0 0.001 0 0 +7.17041 0.001 0 0 0 0.001 0 0 +7.17529 0.001 0 0 0 0.001 0 0 +7.18018 0.001 0 0 0 0.001 0 0 +7.18506 0.001 0 0 0 0.001 0 0 +7.18994 0.001 0 0 0 0.001 0 0 +7.19482 0.001 0 0 0 0.001 0 0 +7.19971 0.001 0 0 0 0.001 0 0 +7.20459 0.001 0 0 0 0.001 0 0 +7.20947 0.001 0 0 0 0.001 0 0 +7.21436 0.001 0 0 0 0.001 0 0 +7.21924 0.001 0 0 0 0.001 0 0 +7.22412 0.001 0 0 0 0.001 0 0 +7.229 0.001 0 0 0 0.001 0 0 +7.23389 0.001 0 0 0 0.001 0 0 +7.23877 0.001 0 0 0 0.001 0 0 +7.24365 0.001 0 0 0 0.001 0 0 +7.24854 0.001 0 0 0 0.001 0 0 +7.25342 0.001 0 0 0 0.001 0 0 +7.2583 0.001 0 0 0 0.001 0 0 +7.26318 0.001 0 0 0 0.001 0 0 +7.26807 0.001 0 0 0 0.001 0 0 +7.27295 0.001 0 0 0 0.001 0 0 +7.27783 0.001 0 0 0 0.001 0 0 +7.28271 0.001 0 0 0 0.001 0 0 +7.2876 0.001 0 0 0 0.001 0 0 +7.29248 0.001 0 0 0 0.001 0 0 +7.29736 0.001 0 0 0 0.001 0 0 +7.30225 0.001 0 0 0 0.001 0 0 +7.30713 0.001 0 0 0 0.001 0 0 +7.31201 0.001 0 0 0 0.001 0 0 +7.31689 0.001 0 0 0 0.001 0 0 +7.32178 0.001 0 0 0 0.001 0 0 +7.32666 0.001 0 0 0 0.001 0 0 +7.33154 0.001 0 0 0 0.001 0 0 +7.33643 0.001 0 0 0 0.001 0 0 +7.34131 0.001 0 0 0 0.001 0 0 +7.34619 0.001 0 0 0 0.001 0 0 +7.35107 0.001 0 0 0 0.001 0 0 +7.35596 0.001 0 0 0 0.001 0 0 +7.36084 0.001 0 0 0 0.001 0 0 +7.36572 0.001 0 0 0 0.001 0 0 +7.37061 0.001 0 0 0 0.001 0 0 +7.37549 0.001 0 0 0 0.001 0 0 +7.38037 0.001 0 0 0 0.001 0 0 +7.38525 0.001 0 0 0 0.001 0 0 +7.39014 0.001 0 0 0 0.001 0 0 +7.39502 0.001 0 0 0 0.001 0 0 +7.3999 0.001 0 0 0 0.001 0 0 +7.40479 0.001 0 0 0 0.001 0 0 +7.40967 0.001 0 0 0 0.001 0 0 +7.41455 0.001 0 0 0 0.001 0 0 +7.41943 0.001 0 0 0 0.001 0 0 +7.42432 0.001 0 0 0 0.001 0 0 +7.4292 0.001 0 0 0 0.001 0 0 +7.43408 0.001 0 0 0 0.001 0 0 +7.43896 0.001 0 0 0 0.001 0 0 +7.44385 0.001 0 0 0 0.001 0 0 +7.44873 0.001 0 0 0 0.001 0 0 +7.45361 0.001 0 0 0 0.001 0 0 +7.4585 0.001 0 0 0 0.001 0 0 +7.46338 0.001 0 0 0 0.001 0 0 +7.46826 0.001 0 0 0 0.001 0 0 +7.47314 0.001 0 0 0 0.001 0 0 +7.47803 0.001 0 0 0 0.001 0 0 +7.48291 0.001 0 0 0 0.001 0 0 +7.48779 0.001 0 0 0 0.001 0 0 +7.49268 0.001 0 0 0 0.001 0 0 +7.49756 0.001 0 0 0 0.001 0 0 +7.50244 0.001 0 0 0 0.001 0 0 +7.50732 0.001 0 0 0 0.001 0 0 +7.51221 0.001 0 0 0 0.001 0 0 +7.51709 0.001 0 0 0 0.001 0 0 +7.52197 0.001 0 0 0 0.001 0 0 +7.52686 0.001 0 0 0 0.001 0 0 +7.53174 0.001 0 0 0 0.001 0 0 +7.53662 0.001 0 0 0 0.001 0 0 +7.5415 0.001 0 0 0 0.001 0 0 +7.54639 0.001 0 0 0 0.001 0 0 +7.55127 0.001 0 0 0 0.001 0 0 +7.55615 0.001 0 0 0 0.001 0 0 +7.56104 0.001 0 0 0 0.001 0 0 +7.56592 0.001 0 0 0 0.001 0 0 +7.5708 0.001 0 0 0 0.001 0 0 +7.57568 0.001 0 0 0 0.001 0 0 +7.58057 0.001 0 0 0 0.001 0 0 +7.58545 0.001 0 0 0 0.001 0 0 +7.59033 0.001 0 0 0 0.001 0 0 +7.59521 0.001 0 0 0 0.001 0 0 +7.6001 0.001 0 0 0 0.001 0 0 +7.60498 0.001 0 0 0 0.001 0 0 +7.60986 0.001 0 0 0 0.001 0 0 +7.61475 0.001 0 0 0 0.001 0 0 +7.61963 0.001 0 0 0 0.001 0 0 +7.62451 0.001 0 0 0 0.001 0 0 +7.62939 0.001 0 0 0 0.001 0 0 +7.63428 0.001 0 0 0 0.001 0 0 +7.63916 0.001 0 0 0 0.001 0 0 +7.64404 0.001 0 0 0 0.001 0 0 +7.64893 0.001 0 0 0 0.001 0 0 +7.65381 0.001 0 0 0 0.001 0 0 +7.65869 0.001 0 0 0 0.001 0 0 +7.66357 0.001 0 0 0 0.001 0 0 +7.66846 0.001 0 0 0 0.001 0 0 +7.67334 0.001 0 0 0 0.001 0 0 +7.67822 0.001 0 0 0 0.001 0 0 +7.68311 0.001 0 0 0 0.001 0 0 +7.68799 0.001 0 0 0 0.001 0 0 +7.69287 0.001 0 0 0 0.001 0 0 +7.69775 0.001 0 0 0 0.001 0 0 +7.70264 0.001 0 0 0 0.001 0 0 +7.70752 0.001 0 0 0 0.001 0 0 +7.7124 0.001 0 0 0 0.001 0 0 +7.71729 0.001 0 0 0 0.001 0 0 +7.72217 0.001 0 0 0 0.001 0 0 +7.72705 0.001 0 0 0 0.001 0 0 +7.73193 0.001 0 0 0 0.001 0 0 +7.73682 0.001 0 0 0 0.001 0 0 +7.7417 0.001 0 0 0 0.001 0 0 +7.74658 0.001 0 0 0 0.001 0 0 +7.75146 0.001 0 0 0 0.001 0 0 +7.75635 0.001 0 0 0 0.001 0 0 +7.76123 0.001 0 0 0 0.001 0 0 +7.76611 0.001 0 0 0 0.001 0 0 +7.771 0.001 0 0 0 0.001 0 0 +7.77588 0.001 0 0 0 0.001 0 0 +7.78076 0.001 0 0 0 0.001 0 0 +7.78564 0.001 0 0 0 0.001 0 0 +7.79053 0.001 0 0 0 0.001 0 0 +7.79541 0.001 0 0 0 0.001 0 0 +7.80029 0.001 0 0 0 0.001 0 0 +7.80518 0.001 0 0 0 0.001 0 0 +7.81006 0.001 0 0 0 0.001 0 0 +7.81494 0.001 0 0 0 0.001 0 0 +7.81982 0.001 0 0 0 0.001 0 0 +7.82471 0.001 0 0 0 0.001 0 0 +7.82959 0.001 0 0 0 0.001 0 0 +7.83447 0.001 0 0 0 0.001 0 0 +7.83936 0.001 0 0 0 0.001 0 0 +7.84424 0.001 0 0 0 0.001 0 0 +7.84912 0.001 0 0 0 0.001 0 0 +7.854 0.001 0 0 0 0.001 0 0 +7.85889 0.001 0 0 0 0.001 0 0 +7.86377 0.001 0 0 0 0.001 0 0 +7.86865 0.001 0 0 0 0.001 0 0 +7.87354 0.001 0 0 0 0.001 0 0 +7.87842 0.001 0 0 0 0.001 0 0 +7.8833 0.001 0 0 0 0.001 0 0 +7.88818 0.001 0 0 0 0.001 0 0 +7.89307 0.001 0 0 0 0.001 0 0 +7.89795 0.001 0 0 0 0.001 0 0 +7.90283 0.001 0 0 0 0.001 0 0 +7.90771 0.001 0 0 0 0.001 0 0 +7.9126 0.001 0 0 0 0.001 0 0 +7.91748 0.001 0 0 0 0.001 0 0 +7.92236 0.001 0 0 0 0.001 0 0 +7.92725 0.001 0 0 0 0.001 0 0 +7.93213 0.001 0 0 0 0.001 0 0 +7.93701 0.001 0 0 0 0.001 0 0 +7.94189 0.001 0 0 0 0.001 0 0 +7.94678 0.001 0 0 0 0.001 0 0 +7.95166 0.001 0 0 0 0.001 0 0 +7.95654 0.001 0 0 0 0.001 0 0 +7.96143 0.001 0 0 0 0.001 0 0 +7.96631 0.001 0 0 0 0.001 0 0 +7.97119 0.001 0 0 0 0.001 0 0 +7.97607 0.001 0 0 0 0.001 0 0 +7.98096 0.001 0 0 0 0.001 0 0 +7.98584 0.001 0 0 0 0.001 0 0 +7.99072 0.001 0 0 0 0.001 0 0 +7.99561 0.001 0 0 0 0.001 0 0 +8.00049 0.001 0 0 0 0.001 0 0 +8.00537 0.001 0 0 0 0.001 0 0 +8.01025 0.001 0 0 0 0.001 0 0 +8.01514 0.001 0 0 0 0.001 0 0 +8.02002 0.001 0 0 0 0.001 0 0 +8.0249 0.001 0 0 0 0.001 0 0 +8.02979 0.001 0 0 0 0.001 0 0 +8.03467 0.001 0 0 0 0.001 0 0 +8.03955 0.001 0 0 0 0.001 0 0 +8.04443 0.001 0 0 0 0.001 0 0 +8.04932 0.001 0 0 0 0.001 0 0 +8.0542 0.001 0 0 0 0.001 0 0 +8.05908 0.001 0 0 0 0.001 0 0 +8.06396 0.001 0 0 0 0.001 0 0 +8.06885 0.001 0 0 0 0.001 0 0 +8.07373 0.001 0 0 0 0.001 0 0 +8.07861 0.001 0 0 0 0.001 0 0 +8.0835 0.001 0 0 0 0.001 0 0 +8.08838 0.001 0 0 0 0.001 0 0 +8.09326 0.001 0 0 0 0.001 0 0 +8.09814 0.001 0 0 0 0.001 0 0 +8.10303 0.001 0 0 0 0.001 0 0 +8.10791 0.001 0 0 0 0.001 0 0 +8.11279 0.001 0 0 0 0.001 0 0 +8.11768 0.001 0 0 0 0.001 0 0 +8.12256 0.001 0 0 0 0.001 0 0 +8.12744 0.001 0 0 0 0.001 0 0 +8.13232 0.001 0 0 0 0.001 0 0 +8.13721 0.001 0 0 0 0.001 0 0 +8.14209 0.001 0 0 0 0.001 0 0 +8.14697 0.001 0 0 0 0.001 0 0 +8.15186 0.001 0 0 0 0.001 0 0 +8.15674 0.001 0 0 0 0.001 0 0 +8.16162 0.001 0 0 0 0.001 0 0 +8.1665 0.001 0 0 0 0.001 0 0 +8.17139 0.001 0 0 0 0.001 0 0 +8.17627 0.001 0 0 0 0.001 0 0 +8.18115 0.001 0 0 0 0.001 0 0 +8.18604 0.001 0 0 0 0.001 0 0 +8.19092 0.001 0 0 0 0.001 0 0 +8.1958 0.001 0 0 0 0.001 0 0 +8.20068 0.001 0 0 0 0.001 0 0 +8.20557 0.001 0 0 0 0.001 0 0 +8.21045 0.001 0 0 0 0.001 0 0 +8.21533 0.001 0 0 0 0.001 0 0 +8.22021 0.001 0 0 0 0.001 0 0 +8.2251 0.001 0 0 0 0.001 0 0 +8.22998 0.001 0 0 0 0.001 0 0 +8.23486 0.001 0 0 0 0.001 0 0 +8.23975 0.001 0 0 0 0.001 0 0 +8.24463 0.001 0 0 0 0.001 0 0 +8.24951 0.001 0 0 0 0.001 0 0 +8.25439 0.001 0 0 0 0.001 0 0 +8.25928 0.001 0 0 0 0.001 0 0 +8.26416 0.001 0 0 0 0.001 0 0 +8.26904 0.001 0 0 0 0.001 0 0 +8.27393 0.001 0 0 0 0.001 0 0 +8.27881 0.001 0 0 0 0.001 0 0 +8.28369 0.001 0 0 0 0.001 0 0 +8.28857 0.001 0 0 0 0.001 0 0 +8.29346 0.001 0 0 0 0.001 0 0 +8.29834 0.001 0 0 0 0.001 0 0 +8.30322 0.001 0 0 0 0.001 0 0 +8.30811 0.001 0 0 0 0.001 0 0 +8.31299 0.001 0 0 0 0.001 0 0 +8.31787 0.001 0 0 0 0.001 0 0 +8.32275 0.001 0 0 0 0.001 0 0 +8.32764 0.001 0 0 0 0.001 0 0 +8.33252 0.001 0 0 0 0.001 0 0 +8.3374 0.001 0 0 0 0.001 0 0 +8.34229 0.001 0 0 0 0.001 0 0 +8.34717 0.001 0 0 0 0.001 0 0 +8.35205 0.001 0 0 0 0.001 0 0 +8.35693 0.001 0 0 0 0.001 0 0 +8.36182 0.001 0 0 0 0.001 0 0 +8.3667 0.001 0 0 0 0.001 0 0 +8.37158 0.001 0 0 0 0.001 0 0 +8.37646 0.001 0 0 0 0.001 0 0 +8.38135 0.001 0 0 0 0.001 0 0 +8.38623 0.001 0 0 0 0.001 0 0 +8.39111 0.001 0 0 0 0.001 0 0 +8.396 0.001 0 0 0 0.001 0 0 +8.40088 0.001 0 0 0 0.001 0 0 +8.40576 0.001 0 0 0 0.001 0 0 +8.41064 0.001 0 0 0 0.001 0 0 +8.41553 0.001 0 0 0 0.001 0 0 +8.42041 0.001 0 0 0 0.001 0 0 +8.42529 0.001 0 0 0 0.001 0 0 +8.43018 0.001 0 0 0 0.001 0 0 +8.43506 0.001 0 0 0 0.001 0 0 +8.43994 0.001 0 0 0 0.001 0 0 +8.44482 0.001 0 0 0 0.001 0 0 +8.44971 0.001 0 0 0 0.001 0 0 +8.45459 0.001 0 0 0 0.001 0 0 +8.45947 0.001 0 0 0 0.001 0 0 +8.46436 0.001 0 0 0 0.001 0 0 +8.46924 0.001 0 0 0 0.001 0 0 +8.47412 0.001 0 0 0 0.001 0 0 +8.479 0.001 0 0 0 0.001 0 0 +8.48389 0.001 0 0 0 0.001 0 0 +8.48877 0.001 0 0 0 0.001 0 0 +8.49365 0.001 0 0 0 0.001 0 0 +8.49854 0.001 0 0 0 0.001 0 0 +8.50342 0.001 0 0 0 0.001 0 0 +8.5083 0.001 0 0 0 0.001 0 0 +8.51318 0.001 0 0 0 0.001 0 0 +8.51807 0.001 0 0 0 0.001 0 0 +8.52295 0.001 0 0 0 0.001 0 0 +8.52783 0.001 0 0 0 0.001 0 0 +8.53271 0.001 0 0 0 0.001 0 0 +8.5376 0.001 0 0 0 0.001 0 0 +8.54248 0.001 0 0 0 0.001 0 0 +8.54736 0.001 0 0 0 0.001 0 0 +8.55225 0.001 0 0 0 0.001 0 0 +8.55713 0.001 0 0 0 0.001 0 0 +8.56201 0.001 0 0 0 0.001 0 0 +8.56689 0.001 0 0 0 0.001 0 0 +8.57178 0.001 0 0 0 0.001 0 0 +8.57666 0.001 0 0 0 0.001 0 0 +8.58154 0.001 0 0 0 0.001 0 0 +8.58643 0.001 0 0 0 0.001 0 0 +8.59131 0.001 0 0 0 0.001 0 0 +8.59619 0.001 0 0 0 0.001 0 0 +8.60107 0.001 0 0 0 0.001 0 0 +8.60596 0.001 0 0 0 0.001 0 0 +8.61084 0.001 0 0 0 0.001 0 0 +8.61572 0.001 0 0 0 0.001 0 0 +8.62061 0.001 0 0 0 0.001 0 0 +8.62549 0.001 0 0 0 0.001 0 0 +8.63037 0.001 0 0 0 0.001 0 0 +8.63525 0.001 0 0 0 0.001 0 0 +8.64014 0.001 0 0 0 0.001 0 0 +8.64502 0.001 0 0 0 0.001 0 0 +8.6499 0.001 0 0 0 0.001 0 0 +8.65479 0.001 0 0 0 0.001 0 0 +8.65967 0.001 0 0 0 0.001 0 0 +8.66455 0.001 0 0 0 0.001 0 0 +8.66943 0.001 0 0 0 0.001 0 0 +8.67432 0.001 0 0 0 0.001 0 0 +8.6792 0.001 0 0 0 0.001 0 0 +8.68408 0.001 0 0 0 0.001 0 0 +8.68896 0.001 0 0 0 0.001 0 0 +8.69385 0.001 0 0 0 0.001 0 0 +8.69873 0.001 0 0 0 0.001 0 0 +8.70361 0.001 0 0 0 0.001 0 0 +8.7085 0.001 0 0 0 0.001 0 0 +8.71338 0.001 0 0 0 0.001 0 0 +8.71826 0.001 0 0 0 0.001 0 0 +8.72314 0.001 0 0 0 0.001 0 0 +8.72803 0.001 0 0 0 0.001 0 0 +8.73291 0.001 0 0 0 0.001 0 0 +8.73779 0.001 0 0 0 0.001 0 0 +8.74268 0.001 0 0 0 0.001 0 0 +8.74756 0.001 0 0 0 0.001 0 0 +8.75244 0.001 0 0 0 0.001 0 0 +8.75732 0.001 0 0 0 0.001 0 0 +8.76221 0.001 0 0 0 0.001 0 0 +8.76709 0.001 0 0 0 0.001 0 0 +8.77197 0.001 0 0 0 0.001 0 0 +8.77686 0.001 0 0 0 0.001 0 0 +8.78174 0.001 0 0 0 0.001 0 0 +8.78662 0.001 0 0 0 0.001 0 0 +8.7915 0.001 0 0 0 0.001 0 0 +8.79639 0.001 0 0 0 0.001 0 0 +8.80127 0.001 0 0 0 0.001 0 0 +8.80615 0.001 0 0 0 0.001 0 0 +8.81104 0.001 0 0 0 0.001 0 0 +8.81592 0.001 0 0 0 0.001 0 0 +8.8208 0.001 0 0 0 0.001 0 0 +8.82568 0.001 0 0 0 0.001 0 0 +8.83057 0.001 0 0 0 0.001 0 0 +8.83545 0.001 0 0 0 0.001 0 0 +8.84033 0.001 0 0 0 0.001 0 0 +8.84521 0.001 0 0 0 0.001 0 0 +8.8501 0.001 0 0 0 0.001 0 0 +8.85498 0.001 0 0 0 0.001 0 0 +8.85986 0.001 0 0 0 0.001 0 0 +8.86475 0.001 0 0 0 0.001 0 0 +8.86963 0.001 0 0 0 0.001 0 0 +8.87451 0.001 0 0 0 0.001 0 0 +8.87939 0.001 0 0 0 0.001 0 0 +8.88428 0.001 0 0 0 0.001 0 0 +8.88916 0.001 0 0 0 0.001 0 0 +8.89404 0.001 0 0 0 0.001 0 0 +8.89893 0.001 0 0 0 0.001 0 0 +8.90381 0.001 0 0 0 0.001 0 0 +8.90869 0.001 0 0 0 0.001 0 0 +8.91357 0.001 0 0 0 0.001 0 0 +8.91846 0.001 0 0 0 0.001 0 0 +8.92334 0.001 0 0 0 0.001 0 0 +8.92822 0.001 0 0 0 0.001 0 0 +8.93311 0.001 0 0 0 0.001 0 0 +8.93799 0.001 0 0 0 0.001 0 0 +8.94287 0.001 0 0 0 0.001 0 0 +8.94775 0.001 0 0 0 0.001 0 0 +8.95264 0.001 0 0 0 0.001 0 0 +8.95752 0.001 0 0 0 0.001 0 0 +8.9624 0.001 0 0 0 0.001 0 0 +8.96729 0.001 0 0 0 0.001 0 0 +8.97217 0.001 0 0 0 0.001 0 0 +8.97705 0.001 0 0 0 0.001 0 0 +8.98193 0.001 0 0 0 0.001 0 0 +8.98682 0.001 0 0 0 0.001 0 0 +8.9917 0.001 0 0 0 0.001 0 0 +8.99658 0.001 0 0 0 0.001 0 0 +9.00146 0.001 0 0 0 0.001 0 0 +9.00635 0.001 0 0 0 0.001 0 0 +9.01123 0.001 0 0 0 0.001 0 0 +9.01611 0.001 0 0 0 0.001 0 0 +9.021 0.001 0 0 0 0.001 0 0 +9.02588 0.001 0 0 0 0.001 0 0 +9.03076 0.001 0 0 0 0.001 0 0 +9.03564 0.001 0 0 0 0.001 0 0 +9.04053 0.001 0 0 0 0.001 0 0 +9.04541 0.001 0 0 0 0.001 0 0 +9.05029 0.001 0 0 0 0.001 0 0 +9.05518 0.001 0 0 0 0.001 0 0 +9.06006 0.001 0 0 0 0.001 0 0 +9.06494 0.001 0 0 0 0.001 0 0 +9.06982 0.001 0 0 0 0.001 0 0 +9.07471 0.001 0 0 0 0.001 0 0 +9.07959 0.001 0 0 0 0.001 0 0 +9.08447 0.001 0 0 0 0.001 0 0 +9.08936 0.001 0 0 0 0.001 0 0 +9.09424 0.001 0 0 0 0.001 0 0 +9.09912 0.001 0 0 0 0.001 0 0 +9.104 0.001 0 0 0 0.001 0 0 +9.10889 0.001 0 0 0 0.001 0 0 +9.11377 0.001 0 0 0 0.001 0 0 +9.11865 0.001 0 0 0 0.001 0 0 +9.12354 0.001 0 0 0 0.001 0 0 +9.12842 0.001 0 0 0 0.001 0 0 +9.1333 0.001 0 0 0 0.001 0 0 +9.13818 0.001 0 0 0 0.001 0 0 +9.14307 0.001 0 0 0 0.001 0 0 +9.14795 0.001 0 0 0 0.001 0 0 +9.15283 0.001 0 0 0 0.001 0 0 +9.15771 0.001 0 0 0 0.001 0 0 +9.1626 0.001 0 0 0 0.001 0 0 +9.16748 0.001 0 0 0 0.001 0 0 +9.17236 0.001 0 0 0 0.001 0 0 +9.17725 0.001 0 0 0 0.001 0 0 +9.18213 0.001 0 0 0 0.001 0 0 +9.18701 0.001 0 0 0 0.001 0 0 +9.19189 0.001 0 0 0 0.001 0 0 +9.19678 0.001 0 0 0 0.001 0 0 +9.20166 0.001 0 0 0 0.001 0 0 +9.20654 0.001 0 0 0 0.001 0 0 +9.21143 0.001 0 0 0 0.001 0 0 +9.21631 0.001 0 0 0 0.001 0 0 +9.22119 0.001 0 0 0 0.001 0 0 +9.22607 0.001 0 0 0 0.001 0 0 +9.23096 0.001 0 0 0 0.001 0 0 +9.23584 0.001 0 0 0 0.001 0 0 +9.24072 0.001 0 0 0 0.001 0 0 +9.24561 0.001 0 0 0 0.001 0 0 +9.25049 0.001 0 0 0 0.001 0 0 +9.25537 0.001 0 0 0 0.001 0 0 +9.26025 0.001 0 0 0 0.001 0 0 +9.26514 0.001 0 0 0 0.001 0 0 +9.27002 0.001 0 0 0 0.001 0 0 +9.2749 0.001 0 0 0 0.001 0 0 +9.27979 0.001 0 0 0 0.001 0 0 +9.28467 0.001 0 0 0 0.001 0 0 +9.28955 0.001 0 0 0 0.001 0 0 +9.29443 0.001 0 0 0 0.001 0 0 +9.29932 0.001 0 0 0 0.001 0 0 +9.3042 0.001 0 0 0 0.001 0 0 +9.30908 0.001 0 0 0 0.001 0 0 +9.31396 0.001 0 0 0 0.001 0 0 +9.31885 0.001 0 0 0 0.001 0 0 +9.32373 0.001 0 0 0 0.001 0 0 +9.32861 0.001 0 0 0 0.001 0 0 +9.3335 0.001 0 0 0 0.001 0 0 +9.33838 0.001 0 0 0 0.001 0 0 +9.34326 0.001 0 0 0 0.001 0 0 +9.34814 0.001 0 0 0 0.001 0 0 +9.35303 0.001 0 0 0 0.001 0 0 +9.35791 0.001 0 0 0 0.001 0 0 +9.36279 0.001 0 0 0 0.001 0 0 +9.36768 0.001 0 0 0 0.001 0 0 +9.37256 0.001 0 0 0 0.001 0 0 +9.37744 0.001 0 0 0 0.001 0 0 +9.38232 0.001 0 0 0 0.001 0 0 +9.38721 0.001 0 0 0 0.001 0 0 +9.39209 0.001 0 0 0 0.001 0 0 +9.39697 0.001 0 0 0 0.001 0 0 +9.40186 0.001 0 0 0 0.001 0 0 +9.40674 0.001 0 0 0 0.001 0 0 +9.41162 0.001 0 0 0 0.001 0 0 +9.4165 0.001 0 0 0 0.001 0 0 +9.42139 0.001 0 0 0 0.001 0 0 +9.42627 0.001 0 0 0 0.001 0 0 +9.43115 0.001 0 0 0 0.001 0 0 +9.43604 0.001 0 0 0 0.001 0 0 +9.44092 0.001 0 0 0 0.001 0 0 +9.4458 0.001 0 0 0 0.001 0 0 +9.45068 0.001 0 0 0 0.001 0 0 +9.45557 0.001 0 0 0 0.001 0 0 +9.46045 0.001 0 0 0 0.001 0 0 +9.46533 0.001 0 0 0 0.001 0 0 +9.47021 0.001 0 0 0 0.001 0 0 +9.4751 0.001 0 0 0 0.001 0 0 +9.47998 0.001 0 0 0 0.001 0 0 +9.48486 0.001 0 0 0 0.001 0 0 +9.48975 0.001 0 0 0 0.001 0 0 +9.49463 0.001 0 0 0 0.001 0 0 +9.49951 0.001 0 0 0 0.001 0 0 +9.50439 0.001 0 0 0 0.001 0 0 +9.50928 0.001 0 0 0 0.001 0 0 +9.51416 0.001 0 0 0 0.001 0 0 +9.51904 0.001 0 0 0 0.001 0 0 +9.52393 0.001 0 0 0 0.001 0 0 +9.52881 0.001 0 0 0 0.001 0 0 +9.53369 0.001 0 0 0 0.001 0 0 +9.53857 0.001 0 0 0 0.001 0 0 +9.54346 0.001 0 0 0 0.001 0 0 +9.54834 0.001 0 0 0 0.001 0 0 +9.55322 0.001 0 0 0 0.001 0 0 +9.55811 0.001 0 0 0 0.001 0 0 +9.56299 0.001 0 0 0 0.001 0 0 +9.56787 0.001 0 0 0 0.001 0 0 +9.57275 0.001 0 0 0 0.001 0 0 +9.57764 0.001 0 0 0 0.001 0 0 +9.58252 0.001 0 0 0 0.001 0 0 +9.5874 0.001 0 0 0 0.001 0 0 +9.59229 0.001 0 0 0 0.001 0 0 +9.59717 0.001 0 0 0 0.001 0 0 +9.60205 0.001 0 0 0 0.001 0 0 +9.60693 0.001 0 0 0 0.001 0 0 +9.61182 0.001 0 0 0 0.001 0 0 +9.6167 0.001 0 0 0 0.001 0 0 +9.62158 0.001 0 0 0 0.001 0 0 +9.62646 0.001 0 0 0 0.001 0 0 +9.63135 0.001 0 0 0 0.001 0 0 +9.63623 0.001 0 0 0 0.001 0 0 +9.64111 0.001 0 0 0 0.001 0 0 +9.646 0.001 0 0 0 0.001 0 0 +9.65088 0.001 0 0 0 0.001 0 0 +9.65576 0.001 0 0 0 0.001 0 0 +9.66064 0.001 0 0 0 0.001 0 0 +9.66553 0.001 0 0 0 0.001 0 0 +9.67041 0.001 0 0 0 0.001 0 0 +9.67529 0.001 0 0 0 0.001 0 0 +9.68018 0.001 0 0 0 0.001 0 0 +9.68506 0.001 0 0 0 0.001 0 0 +9.68994 0.001 0 0 0 0.001 0 0 +9.69482 0.001 0 0 0 0.001 0 0 +9.69971 0.001 0 0 0 0.001 0 0 +9.70459 0.001 0 0 0 0.001 0 0 +9.70947 0.001 0 0 0 0.001 0 0 +9.71436 0.001 0 0 0 0.001 0 0 +9.71924 0.001 0 0 0 0.001 0 0 +9.72412 0.001 0 0 0 0.001 0 0 +9.729 0.001 0 0 0 0.001 0 0 +9.73389 0.001 0 0 0 0.001 0 0 +9.73877 0.001 0 0 0 0.001 0 0 +9.74365 0.001 0 0 0 0.001 0 0 +9.74854 0.001 0 0 0 0.001 0 0 +9.75342 0.001 0 0 0 0.001 0 0 +9.7583 0.001 0 0 0 0.001 0 0 +9.76318 0.001 0 0 0 0.001 0 0 +9.76807 0.001 0 0 0 0.001 0 0 +9.77295 0.001 0 0 0 0.001 0 0 +9.77783 0.001 0 0 0 0.001 0 0 +9.78271 0.001 0 0 0 0.001 0 0 +9.7876 0.001 0 0 0 0.001 0 0 +9.79248 0.001 0 0 0 0.001 0 0 +9.79736 0.001 0 0 0 0.001 0 0 +9.80225 0.001 0 0 0 0.001 0 0 +9.80713 0.001 0 0 0 0.001 0 0 +9.81201 0.001 0 0 0 0.001 0 0 +9.81689 0.001 0 0 0 0.001 0 0 +9.82178 0.001 0 0 0 0.001 0 0 +9.82666 0.001 0 0 0 0.001 0 0 +9.83154 0.001 0 0 0 0.001 0 0 +9.83643 0.001 0 0 0 0.001 0 0 +9.84131 0.001 0 0 0 0.001 0 0 +9.84619 0.001 0 0 0 0.001 0 0 +9.85107 0.001 0 0 0 0.001 0 0 +9.85596 0.001 0 0 0 0.001 0 0 +9.86084 0.001 0 0 0 0.001 0 0 +9.86572 0.001 0 0 0 0.001 0 0 +9.87061 0.001 0 0 0 0.001 0 0 +9.87549 0.001 0 0 0 0.001 0 0 +9.88037 0.001 0 0 0 0.001 0 0 +9.88525 0.001 0 0 0 0.001 0 0 +9.89014 0.001 0 0 0 0.001 0 0 +9.89502 0.001 0 0 0 0.001 0 0 +9.8999 0.001 0 0 0 0.001 0 0 +9.90479 0.001 0 0 0 0.001 0 0 +9.90967 0.001 0 0 0 0.001 0 0 +9.91455 0.001 0 0 0 0.001 0 0 +9.91943 0.001 0 0 0 0.001 0 0 +9.92432 0.001 0 0 0 0.001 0 0 +9.9292 0.001 0 0 0 0.001 0 0 +9.93408 0.001 0 0 0 0.001 0 0 +9.93896 0.001 0 0 0 0.001 0 0 +9.94385 0.001 0 0 0 0.001 0 0 +9.94873 0.001 0 0 0 0.001 0 0 +9.95361 0.001 0 0 0 0.001 0 0 +9.9585 0.001 0 0 0 0.001 0 0 +9.96338 0.001 0 0 0 0.001 0 0 +9.96826 0.001 0 0 0 0.001 0 0 +9.97314 0.001 0 0 0 0.001 0 0 +9.97803 0.001 0 0 0 0.001 0 0 +9.98291 0.001 0 0 0 0.001 0 0 +9.98779 0.001 0 0 0 0.001 0 0 +9.99268 0.001 0 0 0 0.001 0 0 +9.99756 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=256.csv b/reference/swashes_1_nx=256.csv new file mode 100644 index 0000000..65a55ba --- /dev/null +++ b/reference/swashes_1_nx=256.csv @@ -0,0 +1,274 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.0390625 meters +# Number of cells: 256 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0195312 0.005 0 0 0 0.005 0 0 +0.0585938 0.005 0 0 0 0.005 0 0 +0.0976562 0.005 0 0 0 0.005 0 0 +0.136719 0.005 0 0 0 0.005 0 0 +0.175781 0.005 0 0 0 0.005 0 0 +0.214844 0.005 0 0 0 0.005 0 0 +0.253906 0.005 0 0 0 0.005 0 0 +0.292969 0.005 0 0 0 0.005 0 0 +0.332031 0.005 0 0 0 0.005 0 0 +0.371094 0.005 0 0 0 0.005 0 0 +0.410156 0.005 0 0 0 0.005 0 0 +0.449219 0.005 0 0 0 0.005 0 0 +0.488281 0.005 0 0 0 0.005 0 0 +0.527344 0.005 0 0 0 0.005 0 0 +0.566406 0.005 0 0 0 0.005 0 0 +0.605469 0.005 0 0 0 0.005 0 0 +0.644531 0.005 0 0 0 0.005 0 0 +0.683594 0.005 0 0 0 0.005 0 0 +0.722656 0.005 0 0 0 0.005 0 0 +0.761719 0.005 0 0 0 0.005 0 0 +0.800781 0.005 0 0 0 0.005 0 0 +0.839844 0.005 0 0 0 0.005 0 0 +0.878906 0.005 0 0 0 0.005 0 0 +0.917969 0.005 0 0 0 0.005 0 0 +0.957031 0.005 0 0 0 0.005 0 0 +0.996094 0.005 0 0 0 0.005 0 0 +1.03516 0.005 0 0 0 0.005 0 0 +1.07422 0.005 0 0 0 0.005 0 0 +1.11328 0.005 0 0 0 0.005 0 0 +1.15234 0.005 0 0 0 0.005 0 0 +1.19141 0.005 0 0 0 0.005 0 0 +1.23047 0.005 0 0 0 0.005 0 0 +1.26953 0.005 0 0 0 0.005 0 0 +1.30859 0.005 0 0 0 0.005 0 0 +1.34766 0.005 0 0 0 0.005 0 0 +1.38672 0.005 0 0 0 0.005 0 0 +1.42578 0.005 0 0 0 0.005 0 0 +1.46484 0.005 0 0 0 0.005 0 0 +1.50391 0.005 0 0 0 0.005 0 0 +1.54297 0.005 0 0 0 0.005 0 0 +1.58203 0.005 0 0 0 0.005 0 0 +1.62109 0.005 0 0 0 0.005 0 0 +1.66016 0.005 0 0 0 0.005 0 0 +1.69922 0.005 0 0 0 0.005 0 0 +1.73828 0.005 0 0 0 0.005 0 0 +1.77734 0.005 0 0 0 0.005 0 0 +1.81641 0.005 0 0 0 0.005 0 0 +1.85547 0.005 0 0 0 0.005 0 0 +1.89453 0.005 0 0 0 0.005 0 0 +1.93359 0.005 0 0 0 0.005 0 0 +1.97266 0.005 0 0 0 0.005 0 0 +2.01172 0.005 0 0 0 0.005 0 0 +2.05078 0.005 0 0 0 0.005 0 0 +2.08984 0.005 0 0 0 0.005 0 0 +2.12891 0.005 0 0 0 0.005 0 0 +2.16797 0.005 0 0 0 0.005 0 0 +2.20703 0.005 0 0 0 0.005 0 0 +2.24609 0.005 0 0 0 0.005 0 0 +2.28516 0.005 0 0 0 0.005 0 0 +2.32422 0.005 0 0 0 0.005 0 0 +2.36328 0.005 0 0 0 0.005 0 0 +2.40234 0.005 0 0 0 0.005 0 0 +2.44141 0.005 0 0 0 0.005 0 0 +2.48047 0.005 0 0 0 0.005 0 0 +2.51953 0.005 0 0 0 0.005 0 0 +2.55859 0.005 0 0 0 0.005 0 0 +2.59766 0.005 0 0 0 0.005 0 0 +2.63672 0.005 0 0 0 0.005 0 0 +2.67578 0.005 0 0 0 0.005 0 0 +2.71484 0.005 0 0 0 0.005 0 0 +2.75391 0.005 0 0 0 0.005 0 0 +2.79297 0.005 0 0 0 0.005 0 0 +2.83203 0.005 0 0 0 0.005 0 0 +2.87109 0.005 0 0 0 0.005 0 0 +2.91016 0.005 0 0 0 0.005 0 0 +2.94922 0.005 0 0 0 0.005 0 0 +2.98828 0.005 0 0 0 0.005 0 0 +3.02734 0.005 0 0 0 0.005 0 0 +3.06641 0.005 0 0 0 0.005 0 0 +3.10547 0.005 0 0 0 0.005 0 0 +3.14453 0.005 0 0 0 0.005 0 0 +3.18359 0.005 0 0 0 0.005 0 0 +3.22266 0.005 0 0 0 0.005 0 0 +3.26172 0.005 0 0 0 0.005 0 0 +3.30078 0.005 0 0 0 0.005 0 0 +3.33984 0.005 0 0 0 0.005 0 0 +3.37891 0.005 0 0 0 0.005 0 0 +3.41797 0.005 0 0 0 0.005 0 0 +3.45703 0.005 0 0 0 0.005 0 0 +3.49609 0.005 0 0 0 0.005 0 0 +3.53516 0.005 0 0 0 0.005 0 0 +3.57422 0.005 0 0 0 0.005 0 0 +3.61328 0.005 0 0 0 0.005 0 0 +3.65234 0.005 0 0 0 0.005 0 0 +3.69141 0.00494936 0.00224893 0 1.11307e-005 0.00494936 0.0102062 0.000232877 +3.73047 0.00485235 0.0065892 0 3.19731e-005 0.00485235 0.0302011 0.00047058 +3.76953 0.0047563 0.0109295 0 5.19839e-005 0.0047563 0.0505977 0.000650663 +3.80859 0.00466121 0.0152698 0 7.11755e-005 0.00466121 0.0714082 0.000802289 +3.84766 0.00456708 0.01961 0 8.95606e-005 0.00456708 0.0926456 0.000935093 +3.88672 0.00447391 0.0239503 0 0.000107152 0.00447391 0.114323 0.00105384 +3.92578 0.0043817 0.0282906 0 0.000123961 0.0043817 0.136454 0.00116136 +3.96484 0.00429045 0.0326309 0 0.000140001 0.00429045 0.159053 0.0012595 +4.00391 0.00420017 0.0369711 0 0.000155285 0.00420017 0.182136 0.00134957 +4.04297 0.00411084 0.0413114 0 0.000169825 0.00411084 0.205717 0.00143255 +4.08203 0.00402247 0.0456517 0 0.000183633 0.00402247 0.229814 0.00150919 +4.12109 0.00393506 0.049992 0 0.000196722 0.00393506 0.254443 0.00158008 +4.16016 0.00384861 0.0543323 0 0.000209104 0.00384861 0.279622 0.0016457 +4.19922 0.00376313 0.0586725 0 0.000220792 0.00376313 0.30537 0.00170647 +4.23828 0.0036786 0.0630128 0 0.000231799 0.0036786 0.331706 0.00176272 +4.27734 0.00359503 0.0673531 0 0.000242137 0.00359503 0.358651 0.00181475 +4.31641 0.00351242 0.0716934 0 0.000251818 0.00351242 0.386226 0.00186281 +4.35547 0.00343078 0.0760336 0 0.000260855 0.00343078 0.414453 0.00190711 +4.39453 0.00335009 0.0803739 0 0.00026926 0.00335009 0.443356 0.00194786 +4.43359 0.00327036 0.0847142 0 0.000277046 0.00327036 0.472959 0.00198523 +4.47266 0.0031916 0.0890545 0 0.000284226 0.0031916 0.503289 0.00201939 +4.51172 0.00311379 0.0933948 0 0.000290812 0.00311379 0.534371 0.00205046 +4.55078 0.00303694 0.097735 0 0.000296816 0.00303694 0.566236 0.00207859 +4.58984 0.00296106 0.102075 0 0.000302251 0.00296106 0.598912 0.00210389 +4.62891 0.00288613 0.106416 0 0.000307129 0.00288613 0.63243 0.00212646 +4.66797 0.00281217 0.110756 0 0.000311464 0.00281217 0.666825 0.00214642 +4.70703 0.00273916 0.115096 0 0.000315267 0.00273916 0.70213 0.00216386 +4.74609 0.00266712 0.119436 0 0.000318551 0.00266712 0.738383 0.00217886 +4.78516 0.00259603 0.123777 0 0.000321328 0.00259603 0.775621 0.00219151 +4.82422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01953 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05859 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09766 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13672 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17578 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21484 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25391 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29297 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33203 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37109 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41016 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44922 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48828 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52734 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56641 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60547 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64453 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68359 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72266 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76172 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80078 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83984 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87891 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91797 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95703 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99609 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03516 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26953 0.001 0 0 0 0.001 0 0 +6.30859 0.001 0 0 0 0.001 0 0 +6.34766 0.001 0 0 0 0.001 0 0 +6.38672 0.001 0 0 0 0.001 0 0 +6.42578 0.001 0 0 0 0.001 0 0 +6.46484 0.001 0 0 0 0.001 0 0 +6.50391 0.001 0 0 0 0.001 0 0 +6.54297 0.001 0 0 0 0.001 0 0 +6.58203 0.001 0 0 0 0.001 0 0 +6.62109 0.001 0 0 0 0.001 0 0 +6.66016 0.001 0 0 0 0.001 0 0 +6.69922 0.001 0 0 0 0.001 0 0 +6.73828 0.001 0 0 0 0.001 0 0 +6.77734 0.001 0 0 0 0.001 0 0 +6.81641 0.001 0 0 0 0.001 0 0 +6.85547 0.001 0 0 0 0.001 0 0 +6.89453 0.001 0 0 0 0.001 0 0 +6.93359 0.001 0 0 0 0.001 0 0 +6.97266 0.001 0 0 0 0.001 0 0 +7.01172 0.001 0 0 0 0.001 0 0 +7.05078 0.001 0 0 0 0.001 0 0 +7.08984 0.001 0 0 0 0.001 0 0 +7.12891 0.001 0 0 0 0.001 0 0 +7.16797 0.001 0 0 0 0.001 0 0 +7.20703 0.001 0 0 0 0.001 0 0 +7.24609 0.001 0 0 0 0.001 0 0 +7.28516 0.001 0 0 0 0.001 0 0 +7.32422 0.001 0 0 0 0.001 0 0 +7.36328 0.001 0 0 0 0.001 0 0 +7.40234 0.001 0 0 0 0.001 0 0 +7.44141 0.001 0 0 0 0.001 0 0 +7.48047 0.001 0 0 0 0.001 0 0 +7.51953 0.001 0 0 0 0.001 0 0 +7.55859 0.001 0 0 0 0.001 0 0 +7.59766 0.001 0 0 0 0.001 0 0 +7.63672 0.001 0 0 0 0.001 0 0 +7.67578 0.001 0 0 0 0.001 0 0 +7.71484 0.001 0 0 0 0.001 0 0 +7.75391 0.001 0 0 0 0.001 0 0 +7.79297 0.001 0 0 0 0.001 0 0 +7.83203 0.001 0 0 0 0.001 0 0 +7.87109 0.001 0 0 0 0.001 0 0 +7.91016 0.001 0 0 0 0.001 0 0 +7.94922 0.001 0 0 0 0.001 0 0 +7.98828 0.001 0 0 0 0.001 0 0 +8.02734 0.001 0 0 0 0.001 0 0 +8.06641 0.001 0 0 0 0.001 0 0 +8.10547 0.001 0 0 0 0.001 0 0 +8.14453 0.001 0 0 0 0.001 0 0 +8.18359 0.001 0 0 0 0.001 0 0 +8.22266 0.001 0 0 0 0.001 0 0 +8.26172 0.001 0 0 0 0.001 0 0 +8.30078 0.001 0 0 0 0.001 0 0 +8.33984 0.001 0 0 0 0.001 0 0 +8.37891 0.001 0 0 0 0.001 0 0 +8.41797 0.001 0 0 0 0.001 0 0 +8.45703 0.001 0 0 0 0.001 0 0 +8.49609 0.001 0 0 0 0.001 0 0 +8.53516 0.001 0 0 0 0.001 0 0 +8.57422 0.001 0 0 0 0.001 0 0 +8.61328 0.001 0 0 0 0.001 0 0 +8.65234 0.001 0 0 0 0.001 0 0 +8.69141 0.001 0 0 0 0.001 0 0 +8.73047 0.001 0 0 0 0.001 0 0 +8.76953 0.001 0 0 0 0.001 0 0 +8.80859 0.001 0 0 0 0.001 0 0 +8.84766 0.001 0 0 0 0.001 0 0 +8.88672 0.001 0 0 0 0.001 0 0 +8.92578 0.001 0 0 0 0.001 0 0 +8.96484 0.001 0 0 0 0.001 0 0 +9.00391 0.001 0 0 0 0.001 0 0 +9.04297 0.001 0 0 0 0.001 0 0 +9.08203 0.001 0 0 0 0.001 0 0 +9.12109 0.001 0 0 0 0.001 0 0 +9.16016 0.001 0 0 0 0.001 0 0 +9.19922 0.001 0 0 0 0.001 0 0 +9.23828 0.001 0 0 0 0.001 0 0 +9.27734 0.001 0 0 0 0.001 0 0 +9.31641 0.001 0 0 0 0.001 0 0 +9.35547 0.001 0 0 0 0.001 0 0 +9.39453 0.001 0 0 0 0.001 0 0 +9.43359 0.001 0 0 0 0.001 0 0 +9.47266 0.001 0 0 0 0.001 0 0 +9.51172 0.001 0 0 0 0.001 0 0 +9.55078 0.001 0 0 0 0.001 0 0 +9.58984 0.001 0 0 0 0.001 0 0 +9.62891 0.001 0 0 0 0.001 0 0 +9.66797 0.001 0 0 0 0.001 0 0 +9.70703 0.001 0 0 0 0.001 0 0 +9.74609 0.001 0 0 0 0.001 0 0 +9.78516 0.001 0 0 0 0.001 0 0 +9.82422 0.001 0 0 0 0.001 0 0 +9.86328 0.001 0 0 0 0.001 0 0 +9.90234 0.001 0 0 0 0.001 0 0 +9.94141 0.001 0 0 0 0.001 0 0 +9.98047 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=4096.csv b/reference/swashes_1_nx=4096.csv new file mode 100644 index 0000000..be77bf7 --- /dev/null +++ b/reference/swashes_1_nx=4096.csv @@ -0,0 +1,4114 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00244141 meters +# Number of cells: 4096 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0012207 0.005 0 0 0 0.005 0 0 +0.00366211 0.005 0 0 0 0.005 0 0 +0.00610352 0.005 0 0 0 0.005 0 0 +0.00854492 0.005 0 0 0 0.005 0 0 +0.0109863 0.005 0 0 0 0.005 0 0 +0.0134277 0.005 0 0 0 0.005 0 0 +0.0158691 0.005 0 0 0 0.005 0 0 +0.0183105 0.005 0 0 0 0.005 0 0 +0.020752 0.005 0 0 0 0.005 0 0 +0.0231934 0.005 0 0 0 0.005 0 0 +0.0256348 0.005 0 0 0 0.005 0 0 +0.0280762 0.005 0 0 0 0.005 0 0 +0.0305176 0.005 0 0 0 0.005 0 0 +0.032959 0.005 0 0 0 0.005 0 0 +0.0354004 0.005 0 0 0 0.005 0 0 +0.0378418 0.005 0 0 0 0.005 0 0 +0.0402832 0.005 0 0 0 0.005 0 0 +0.0427246 0.005 0 0 0 0.005 0 0 +0.045166 0.005 0 0 0 0.005 0 0 +0.0476074 0.005 0 0 0 0.005 0 0 +0.0500488 0.005 0 0 0 0.005 0 0 +0.0524902 0.005 0 0 0 0.005 0 0 +0.0549316 0.005 0 0 0 0.005 0 0 +0.057373 0.005 0 0 0 0.005 0 0 +0.0598145 0.005 0 0 0 0.005 0 0 +0.0622559 0.005 0 0 0 0.005 0 0 +0.0646973 0.005 0 0 0 0.005 0 0 +0.0671387 0.005 0 0 0 0.005 0 0 +0.0695801 0.005 0 0 0 0.005 0 0 +0.0720215 0.005 0 0 0 0.005 0 0 +0.0744629 0.005 0 0 0 0.005 0 0 +0.0769043 0.005 0 0 0 0.005 0 0 +0.0793457 0.005 0 0 0 0.005 0 0 +0.0817871 0.005 0 0 0 0.005 0 0 +0.0842285 0.005 0 0 0 0.005 0 0 +0.0866699 0.005 0 0 0 0.005 0 0 +0.0891113 0.005 0 0 0 0.005 0 0 +0.0915527 0.005 0 0 0 0.005 0 0 +0.0939941 0.005 0 0 0 0.005 0 0 +0.0964355 0.005 0 0 0 0.005 0 0 +0.098877 0.005 0 0 0 0.005 0 0 +0.101318 0.005 0 0 0 0.005 0 0 +0.10376 0.005 0 0 0 0.005 0 0 +0.106201 0.005 0 0 0 0.005 0 0 +0.108643 0.005 0 0 0 0.005 0 0 +0.111084 0.005 0 0 0 0.005 0 0 +0.113525 0.005 0 0 0 0.005 0 0 +0.115967 0.005 0 0 0 0.005 0 0 +0.118408 0.005 0 0 0 0.005 0 0 +0.12085 0.005 0 0 0 0.005 0 0 +0.123291 0.005 0 0 0 0.005 0 0 +0.125732 0.005 0 0 0 0.005 0 0 +0.128174 0.005 0 0 0 0.005 0 0 +0.130615 0.005 0 0 0 0.005 0 0 +0.133057 0.005 0 0 0 0.005 0 0 +0.135498 0.005 0 0 0 0.005 0 0 +0.137939 0.005 0 0 0 0.005 0 0 +0.140381 0.005 0 0 0 0.005 0 0 +0.142822 0.005 0 0 0 0.005 0 0 +0.145264 0.005 0 0 0 0.005 0 0 +0.147705 0.005 0 0 0 0.005 0 0 +0.150146 0.005 0 0 0 0.005 0 0 +0.152588 0.005 0 0 0 0.005 0 0 +0.155029 0.005 0 0 0 0.005 0 0 +0.157471 0.005 0 0 0 0.005 0 0 +0.159912 0.005 0 0 0 0.005 0 0 +0.162354 0.005 0 0 0 0.005 0 0 +0.164795 0.005 0 0 0 0.005 0 0 +0.167236 0.005 0 0 0 0.005 0 0 +0.169678 0.005 0 0 0 0.005 0 0 +0.172119 0.005 0 0 0 0.005 0 0 +0.174561 0.005 0 0 0 0.005 0 0 +0.177002 0.005 0 0 0 0.005 0 0 +0.179443 0.005 0 0 0 0.005 0 0 +0.181885 0.005 0 0 0 0.005 0 0 +0.184326 0.005 0 0 0 0.005 0 0 +0.186768 0.005 0 0 0 0.005 0 0 +0.189209 0.005 0 0 0 0.005 0 0 +0.19165 0.005 0 0 0 0.005 0 0 +0.194092 0.005 0 0 0 0.005 0 0 +0.196533 0.005 0 0 0 0.005 0 0 +0.198975 0.005 0 0 0 0.005 0 0 +0.201416 0.005 0 0 0 0.005 0 0 +0.203857 0.005 0 0 0 0.005 0 0 +0.206299 0.005 0 0 0 0.005 0 0 +0.20874 0.005 0 0 0 0.005 0 0 +0.211182 0.005 0 0 0 0.005 0 0 +0.213623 0.005 0 0 0 0.005 0 0 +0.216064 0.005 0 0 0 0.005 0 0 +0.218506 0.005 0 0 0 0.005 0 0 +0.220947 0.005 0 0 0 0.005 0 0 +0.223389 0.005 0 0 0 0.005 0 0 +0.22583 0.005 0 0 0 0.005 0 0 +0.228271 0.005 0 0 0 0.005 0 0 +0.230713 0.005 0 0 0 0.005 0 0 +0.233154 0.005 0 0 0 0.005 0 0 +0.235596 0.005 0 0 0 0.005 0 0 +0.238037 0.005 0 0 0 0.005 0 0 +0.240479 0.005 0 0 0 0.005 0 0 +0.24292 0.005 0 0 0 0.005 0 0 +0.245361 0.005 0 0 0 0.005 0 0 +0.247803 0.005 0 0 0 0.005 0 0 +0.250244 0.005 0 0 0 0.005 0 0 +0.252686 0.005 0 0 0 0.005 0 0 +0.255127 0.005 0 0 0 0.005 0 0 +0.257568 0.005 0 0 0 0.005 0 0 +0.26001 0.005 0 0 0 0.005 0 0 +0.262451 0.005 0 0 0 0.005 0 0 +0.264893 0.005 0 0 0 0.005 0 0 +0.267334 0.005 0 0 0 0.005 0 0 +0.269775 0.005 0 0 0 0.005 0 0 +0.272217 0.005 0 0 0 0.005 0 0 +0.274658 0.005 0 0 0 0.005 0 0 +0.2771 0.005 0 0 0 0.005 0 0 +0.279541 0.005 0 0 0 0.005 0 0 +0.281982 0.005 0 0 0 0.005 0 0 +0.284424 0.005 0 0 0 0.005 0 0 +0.286865 0.005 0 0 0 0.005 0 0 +0.289307 0.005 0 0 0 0.005 0 0 +0.291748 0.005 0 0 0 0.005 0 0 +0.294189 0.005 0 0 0 0.005 0 0 +0.296631 0.005 0 0 0 0.005 0 0 +0.299072 0.005 0 0 0 0.005 0 0 +0.301514 0.005 0 0 0 0.005 0 0 +0.303955 0.005 0 0 0 0.005 0 0 +0.306396 0.005 0 0 0 0.005 0 0 +0.308838 0.005 0 0 0 0.005 0 0 +0.311279 0.005 0 0 0 0.005 0 0 +0.313721 0.005 0 0 0 0.005 0 0 +0.316162 0.005 0 0 0 0.005 0 0 +0.318604 0.005 0 0 0 0.005 0 0 +0.321045 0.005 0 0 0 0.005 0 0 +0.323486 0.005 0 0 0 0.005 0 0 +0.325928 0.005 0 0 0 0.005 0 0 +0.328369 0.005 0 0 0 0.005 0 0 +0.330811 0.005 0 0 0 0.005 0 0 +0.333252 0.005 0 0 0 0.005 0 0 +0.335693 0.005 0 0 0 0.005 0 0 +0.338135 0.005 0 0 0 0.005 0 0 +0.340576 0.005 0 0 0 0.005 0 0 +0.343018 0.005 0 0 0 0.005 0 0 +0.345459 0.005 0 0 0 0.005 0 0 +0.3479 0.005 0 0 0 0.005 0 0 +0.350342 0.005 0 0 0 0.005 0 0 +0.352783 0.005 0 0 0 0.005 0 0 +0.355225 0.005 0 0 0 0.005 0 0 +0.357666 0.005 0 0 0 0.005 0 0 +0.360107 0.005 0 0 0 0.005 0 0 +0.362549 0.005 0 0 0 0.005 0 0 +0.36499 0.005 0 0 0 0.005 0 0 +0.367432 0.005 0 0 0 0.005 0 0 +0.369873 0.005 0 0 0 0.005 0 0 +0.372314 0.005 0 0 0 0.005 0 0 +0.374756 0.005 0 0 0 0.005 0 0 +0.377197 0.005 0 0 0 0.005 0 0 +0.379639 0.005 0 0 0 0.005 0 0 +0.38208 0.005 0 0 0 0.005 0 0 +0.384521 0.005 0 0 0 0.005 0 0 +0.386963 0.005 0 0 0 0.005 0 0 +0.389404 0.005 0 0 0 0.005 0 0 +0.391846 0.005 0 0 0 0.005 0 0 +0.394287 0.005 0 0 0 0.005 0 0 +0.396729 0.005 0 0 0 0.005 0 0 +0.39917 0.005 0 0 0 0.005 0 0 +0.401611 0.005 0 0 0 0.005 0 0 +0.404053 0.005 0 0 0 0.005 0 0 +0.406494 0.005 0 0 0 0.005 0 0 +0.408936 0.005 0 0 0 0.005 0 0 +0.411377 0.005 0 0 0 0.005 0 0 +0.413818 0.005 0 0 0 0.005 0 0 +0.41626 0.005 0 0 0 0.005 0 0 +0.418701 0.005 0 0 0 0.005 0 0 +0.421143 0.005 0 0 0 0.005 0 0 +0.423584 0.005 0 0 0 0.005 0 0 +0.426025 0.005 0 0 0 0.005 0 0 +0.428467 0.005 0 0 0 0.005 0 0 +0.430908 0.005 0 0 0 0.005 0 0 +0.43335 0.005 0 0 0 0.005 0 0 +0.435791 0.005 0 0 0 0.005 0 0 +0.438232 0.005 0 0 0 0.005 0 0 +0.440674 0.005 0 0 0 0.005 0 0 +0.443115 0.005 0 0 0 0.005 0 0 +0.445557 0.005 0 0 0 0.005 0 0 +0.447998 0.005 0 0 0 0.005 0 0 +0.450439 0.005 0 0 0 0.005 0 0 +0.452881 0.005 0 0 0 0.005 0 0 +0.455322 0.005 0 0 0 0.005 0 0 +0.457764 0.005 0 0 0 0.005 0 0 +0.460205 0.005 0 0 0 0.005 0 0 +0.462646 0.005 0 0 0 0.005 0 0 +0.465088 0.005 0 0 0 0.005 0 0 +0.467529 0.005 0 0 0 0.005 0 0 +0.469971 0.005 0 0 0 0.005 0 0 +0.472412 0.005 0 0 0 0.005 0 0 +0.474854 0.005 0 0 0 0.005 0 0 +0.477295 0.005 0 0 0 0.005 0 0 +0.479736 0.005 0 0 0 0.005 0 0 +0.482178 0.005 0 0 0 0.005 0 0 +0.484619 0.005 0 0 0 0.005 0 0 +0.487061 0.005 0 0 0 0.005 0 0 +0.489502 0.005 0 0 0 0.005 0 0 +0.491943 0.005 0 0 0 0.005 0 0 +0.494385 0.005 0 0 0 0.005 0 0 +0.496826 0.005 0 0 0 0.005 0 0 +0.499268 0.005 0 0 0 0.005 0 0 +0.501709 0.005 0 0 0 0.005 0 0 +0.50415 0.005 0 0 0 0.005 0 0 +0.506592 0.005 0 0 0 0.005 0 0 +0.509033 0.005 0 0 0 0.005 0 0 +0.511475 0.005 0 0 0 0.005 0 0 +0.513916 0.005 0 0 0 0.005 0 0 +0.516357 0.005 0 0 0 0.005 0 0 +0.518799 0.005 0 0 0 0.005 0 0 +0.52124 0.005 0 0 0 0.005 0 0 +0.523682 0.005 0 0 0 0.005 0 0 +0.526123 0.005 0 0 0 0.005 0 0 +0.528564 0.005 0 0 0 0.005 0 0 +0.531006 0.005 0 0 0 0.005 0 0 +0.533447 0.005 0 0 0 0.005 0 0 +0.535889 0.005 0 0 0 0.005 0 0 +0.53833 0.005 0 0 0 0.005 0 0 +0.540771 0.005 0 0 0 0.005 0 0 +0.543213 0.005 0 0 0 0.005 0 0 +0.545654 0.005 0 0 0 0.005 0 0 +0.548096 0.005 0 0 0 0.005 0 0 +0.550537 0.005 0 0 0 0.005 0 0 +0.552979 0.005 0 0 0 0.005 0 0 +0.55542 0.005 0 0 0 0.005 0 0 +0.557861 0.005 0 0 0 0.005 0 0 +0.560303 0.005 0 0 0 0.005 0 0 +0.562744 0.005 0 0 0 0.005 0 0 +0.565186 0.005 0 0 0 0.005 0 0 +0.567627 0.005 0 0 0 0.005 0 0 +0.570068 0.005 0 0 0 0.005 0 0 +0.57251 0.005 0 0 0 0.005 0 0 +0.574951 0.005 0 0 0 0.005 0 0 +0.577393 0.005 0 0 0 0.005 0 0 +0.579834 0.005 0 0 0 0.005 0 0 +0.582275 0.005 0 0 0 0.005 0 0 +0.584717 0.005 0 0 0 0.005 0 0 +0.587158 0.005 0 0 0 0.005 0 0 +0.5896 0.005 0 0 0 0.005 0 0 +0.592041 0.005 0 0 0 0.005 0 0 +0.594482 0.005 0 0 0 0.005 0 0 +0.596924 0.005 0 0 0 0.005 0 0 +0.599365 0.005 0 0 0 0.005 0 0 +0.601807 0.005 0 0 0 0.005 0 0 +0.604248 0.005 0 0 0 0.005 0 0 +0.606689 0.005 0 0 0 0.005 0 0 +0.609131 0.005 0 0 0 0.005 0 0 +0.611572 0.005 0 0 0 0.005 0 0 +0.614014 0.005 0 0 0 0.005 0 0 +0.616455 0.005 0 0 0 0.005 0 0 +0.618896 0.005 0 0 0 0.005 0 0 +0.621338 0.005 0 0 0 0.005 0 0 +0.623779 0.005 0 0 0 0.005 0 0 +0.626221 0.005 0 0 0 0.005 0 0 +0.628662 0.005 0 0 0 0.005 0 0 +0.631104 0.005 0 0 0 0.005 0 0 +0.633545 0.005 0 0 0 0.005 0 0 +0.635986 0.005 0 0 0 0.005 0 0 +0.638428 0.005 0 0 0 0.005 0 0 +0.640869 0.005 0 0 0 0.005 0 0 +0.643311 0.005 0 0 0 0.005 0 0 +0.645752 0.005 0 0 0 0.005 0 0 +0.648193 0.005 0 0 0 0.005 0 0 +0.650635 0.005 0 0 0 0.005 0 0 +0.653076 0.005 0 0 0 0.005 0 0 +0.655518 0.005 0 0 0 0.005 0 0 +0.657959 0.005 0 0 0 0.005 0 0 +0.6604 0.005 0 0 0 0.005 0 0 +0.662842 0.005 0 0 0 0.005 0 0 +0.665283 0.005 0 0 0 0.005 0 0 +0.667725 0.005 0 0 0 0.005 0 0 +0.670166 0.005 0 0 0 0.005 0 0 +0.672607 0.005 0 0 0 0.005 0 0 +0.675049 0.005 0 0 0 0.005 0 0 +0.67749 0.005 0 0 0 0.005 0 0 +0.679932 0.005 0 0 0 0.005 0 0 +0.682373 0.005 0 0 0 0.005 0 0 +0.684814 0.005 0 0 0 0.005 0 0 +0.687256 0.005 0 0 0 0.005 0 0 +0.689697 0.005 0 0 0 0.005 0 0 +0.692139 0.005 0 0 0 0.005 0 0 +0.69458 0.005 0 0 0 0.005 0 0 +0.697021 0.005 0 0 0 0.005 0 0 +0.699463 0.005 0 0 0 0.005 0 0 +0.701904 0.005 0 0 0 0.005 0 0 +0.704346 0.005 0 0 0 0.005 0 0 +0.706787 0.005 0 0 0 0.005 0 0 +0.709229 0.005 0 0 0 0.005 0 0 +0.71167 0.005 0 0 0 0.005 0 0 +0.714111 0.005 0 0 0 0.005 0 0 +0.716553 0.005 0 0 0 0.005 0 0 +0.718994 0.005 0 0 0 0.005 0 0 +0.721436 0.005 0 0 0 0.005 0 0 +0.723877 0.005 0 0 0 0.005 0 0 +0.726318 0.005 0 0 0 0.005 0 0 +0.72876 0.005 0 0 0 0.005 0 0 +0.731201 0.005 0 0 0 0.005 0 0 +0.733643 0.005 0 0 0 0.005 0 0 +0.736084 0.005 0 0 0 0.005 0 0 +0.738525 0.005 0 0 0 0.005 0 0 +0.740967 0.005 0 0 0 0.005 0 0 +0.743408 0.005 0 0 0 0.005 0 0 +0.74585 0.005 0 0 0 0.005 0 0 +0.748291 0.005 0 0 0 0.005 0 0 +0.750732 0.005 0 0 0 0.005 0 0 +0.753174 0.005 0 0 0 0.005 0 0 +0.755615 0.005 0 0 0 0.005 0 0 +0.758057 0.005 0 0 0 0.005 0 0 +0.760498 0.005 0 0 0 0.005 0 0 +0.762939 0.005 0 0 0 0.005 0 0 +0.765381 0.005 0 0 0 0.005 0 0 +0.767822 0.005 0 0 0 0.005 0 0 +0.770264 0.005 0 0 0 0.005 0 0 +0.772705 0.005 0 0 0 0.005 0 0 +0.775146 0.005 0 0 0 0.005 0 0 +0.777588 0.005 0 0 0 0.005 0 0 +0.780029 0.005 0 0 0 0.005 0 0 +0.782471 0.005 0 0 0 0.005 0 0 +0.784912 0.005 0 0 0 0.005 0 0 +0.787354 0.005 0 0 0 0.005 0 0 +0.789795 0.005 0 0 0 0.005 0 0 +0.792236 0.005 0 0 0 0.005 0 0 +0.794678 0.005 0 0 0 0.005 0 0 +0.797119 0.005 0 0 0 0.005 0 0 +0.799561 0.005 0 0 0 0.005 0 0 +0.802002 0.005 0 0 0 0.005 0 0 +0.804443 0.005 0 0 0 0.005 0 0 +0.806885 0.005 0 0 0 0.005 0 0 +0.809326 0.005 0 0 0 0.005 0 0 +0.811768 0.005 0 0 0 0.005 0 0 +0.814209 0.005 0 0 0 0.005 0 0 +0.81665 0.005 0 0 0 0.005 0 0 +0.819092 0.005 0 0 0 0.005 0 0 +0.821533 0.005 0 0 0 0.005 0 0 +0.823975 0.005 0 0 0 0.005 0 0 +0.826416 0.005 0 0 0 0.005 0 0 +0.828857 0.005 0 0 0 0.005 0 0 +0.831299 0.005 0 0 0 0.005 0 0 +0.83374 0.005 0 0 0 0.005 0 0 +0.836182 0.005 0 0 0 0.005 0 0 +0.838623 0.005 0 0 0 0.005 0 0 +0.841064 0.005 0 0 0 0.005 0 0 +0.843506 0.005 0 0 0 0.005 0 0 +0.845947 0.005 0 0 0 0.005 0 0 +0.848389 0.005 0 0 0 0.005 0 0 +0.85083 0.005 0 0 0 0.005 0 0 +0.853271 0.005 0 0 0 0.005 0 0 +0.855713 0.005 0 0 0 0.005 0 0 +0.858154 0.005 0 0 0 0.005 0 0 +0.860596 0.005 0 0 0 0.005 0 0 +0.863037 0.005 0 0 0 0.005 0 0 +0.865479 0.005 0 0 0 0.005 0 0 +0.86792 0.005 0 0 0 0.005 0 0 +0.870361 0.005 0 0 0 0.005 0 0 +0.872803 0.005 0 0 0 0.005 0 0 +0.875244 0.005 0 0 0 0.005 0 0 +0.877686 0.005 0 0 0 0.005 0 0 +0.880127 0.005 0 0 0 0.005 0 0 +0.882568 0.005 0 0 0 0.005 0 0 +0.88501 0.005 0 0 0 0.005 0 0 +0.887451 0.005 0 0 0 0.005 0 0 +0.889893 0.005 0 0 0 0.005 0 0 +0.892334 0.005 0 0 0 0.005 0 0 +0.894775 0.005 0 0 0 0.005 0 0 +0.897217 0.005 0 0 0 0.005 0 0 +0.899658 0.005 0 0 0 0.005 0 0 +0.9021 0.005 0 0 0 0.005 0 0 +0.904541 0.005 0 0 0 0.005 0 0 +0.906982 0.005 0 0 0 0.005 0 0 +0.909424 0.005 0 0 0 0.005 0 0 +0.911865 0.005 0 0 0 0.005 0 0 +0.914307 0.005 0 0 0 0.005 0 0 +0.916748 0.005 0 0 0 0.005 0 0 +0.919189 0.005 0 0 0 0.005 0 0 +0.921631 0.005 0 0 0 0.005 0 0 +0.924072 0.005 0 0 0 0.005 0 0 +0.926514 0.005 0 0 0 0.005 0 0 +0.928955 0.005 0 0 0 0.005 0 0 +0.931396 0.005 0 0 0 0.005 0 0 +0.933838 0.005 0 0 0 0.005 0 0 +0.936279 0.005 0 0 0 0.005 0 0 +0.938721 0.005 0 0 0 0.005 0 0 +0.941162 0.005 0 0 0 0.005 0 0 +0.943604 0.005 0 0 0 0.005 0 0 +0.946045 0.005 0 0 0 0.005 0 0 +0.948486 0.005 0 0 0 0.005 0 0 +0.950928 0.005 0 0 0 0.005 0 0 +0.953369 0.005 0 0 0 0.005 0 0 +0.955811 0.005 0 0 0 0.005 0 0 +0.958252 0.005 0 0 0 0.005 0 0 +0.960693 0.005 0 0 0 0.005 0 0 +0.963135 0.005 0 0 0 0.005 0 0 +0.965576 0.005 0 0 0 0.005 0 0 +0.968018 0.005 0 0 0 0.005 0 0 +0.970459 0.005 0 0 0 0.005 0 0 +0.9729 0.005 0 0 0 0.005 0 0 +0.975342 0.005 0 0 0 0.005 0 0 +0.977783 0.005 0 0 0 0.005 0 0 +0.980225 0.005 0 0 0 0.005 0 0 +0.982666 0.005 0 0 0 0.005 0 0 +0.985107 0.005 0 0 0 0.005 0 0 +0.987549 0.005 0 0 0 0.005 0 0 +0.98999 0.005 0 0 0 0.005 0 0 +0.992432 0.005 0 0 0 0.005 0 0 +0.994873 0.005 0 0 0 0.005 0 0 +0.997314 0.005 0 0 0 0.005 0 0 +0.999756 0.005 0 0 0 0.005 0 0 +1.0022 0.005 0 0 0 0.005 0 0 +1.00464 0.005 0 0 0 0.005 0 0 +1.00708 0.005 0 0 0 0.005 0 0 +1.00952 0.005 0 0 0 0.005 0 0 +1.01196 0.005 0 0 0 0.005 0 0 +1.0144 0.005 0 0 0 0.005 0 0 +1.01685 0.005 0 0 0 0.005 0 0 +1.01929 0.005 0 0 0 0.005 0 0 +1.02173 0.005 0 0 0 0.005 0 0 +1.02417 0.005 0 0 0 0.005 0 0 +1.02661 0.005 0 0 0 0.005 0 0 +1.02905 0.005 0 0 0 0.005 0 0 +1.03149 0.005 0 0 0 0.005 0 0 +1.03394 0.005 0 0 0 0.005 0 0 +1.03638 0.005 0 0 0 0.005 0 0 +1.03882 0.005 0 0 0 0.005 0 0 +1.04126 0.005 0 0 0 0.005 0 0 +1.0437 0.005 0 0 0 0.005 0 0 +1.04614 0.005 0 0 0 0.005 0 0 +1.04858 0.005 0 0 0 0.005 0 0 +1.05103 0.005 0 0 0 0.005 0 0 +1.05347 0.005 0 0 0 0.005 0 0 +1.05591 0.005 0 0 0 0.005 0 0 +1.05835 0.005 0 0 0 0.005 0 0 +1.06079 0.005 0 0 0 0.005 0 0 +1.06323 0.005 0 0 0 0.005 0 0 +1.06567 0.005 0 0 0 0.005 0 0 +1.06812 0.005 0 0 0 0.005 0 0 +1.07056 0.005 0 0 0 0.005 0 0 +1.073 0.005 0 0 0 0.005 0 0 +1.07544 0.005 0 0 0 0.005 0 0 +1.07788 0.005 0 0 0 0.005 0 0 +1.08032 0.005 0 0 0 0.005 0 0 +1.08276 0.005 0 0 0 0.005 0 0 +1.08521 0.005 0 0 0 0.005 0 0 +1.08765 0.005 0 0 0 0.005 0 0 +1.09009 0.005 0 0 0 0.005 0 0 +1.09253 0.005 0 0 0 0.005 0 0 +1.09497 0.005 0 0 0 0.005 0 0 +1.09741 0.005 0 0 0 0.005 0 0 +1.09985 0.005 0 0 0 0.005 0 0 +1.10229 0.005 0 0 0 0.005 0 0 +1.10474 0.005 0 0 0 0.005 0 0 +1.10718 0.005 0 0 0 0.005 0 0 +1.10962 0.005 0 0 0 0.005 0 0 +1.11206 0.005 0 0 0 0.005 0 0 +1.1145 0.005 0 0 0 0.005 0 0 +1.11694 0.005 0 0 0 0.005 0 0 +1.11938 0.005 0 0 0 0.005 0 0 +1.12183 0.005 0 0 0 0.005 0 0 +1.12427 0.005 0 0 0 0.005 0 0 +1.12671 0.005 0 0 0 0.005 0 0 +1.12915 0.005 0 0 0 0.005 0 0 +1.13159 0.005 0 0 0 0.005 0 0 +1.13403 0.005 0 0 0 0.005 0 0 +1.13647 0.005 0 0 0 0.005 0 0 +1.13892 0.005 0 0 0 0.005 0 0 +1.14136 0.005 0 0 0 0.005 0 0 +1.1438 0.005 0 0 0 0.005 0 0 +1.14624 0.005 0 0 0 0.005 0 0 +1.14868 0.005 0 0 0 0.005 0 0 +1.15112 0.005 0 0 0 0.005 0 0 +1.15356 0.005 0 0 0 0.005 0 0 +1.15601 0.005 0 0 0 0.005 0 0 +1.15845 0.005 0 0 0 0.005 0 0 +1.16089 0.005 0 0 0 0.005 0 0 +1.16333 0.005 0 0 0 0.005 0 0 +1.16577 0.005 0 0 0 0.005 0 0 +1.16821 0.005 0 0 0 0.005 0 0 +1.17065 0.005 0 0 0 0.005 0 0 +1.1731 0.005 0 0 0 0.005 0 0 +1.17554 0.005 0 0 0 0.005 0 0 +1.17798 0.005 0 0 0 0.005 0 0 +1.18042 0.005 0 0 0 0.005 0 0 +1.18286 0.005 0 0 0 0.005 0 0 +1.1853 0.005 0 0 0 0.005 0 0 +1.18774 0.005 0 0 0 0.005 0 0 +1.19019 0.005 0 0 0 0.005 0 0 +1.19263 0.005 0 0 0 0.005 0 0 +1.19507 0.005 0 0 0 0.005 0 0 +1.19751 0.005 0 0 0 0.005 0 0 +1.19995 0.005 0 0 0 0.005 0 0 +1.20239 0.005 0 0 0 0.005 0 0 +1.20483 0.005 0 0 0 0.005 0 0 +1.20728 0.005 0 0 0 0.005 0 0 +1.20972 0.005 0 0 0 0.005 0 0 +1.21216 0.005 0 0 0 0.005 0 0 +1.2146 0.005 0 0 0 0.005 0 0 +1.21704 0.005 0 0 0 0.005 0 0 +1.21948 0.005 0 0 0 0.005 0 0 +1.22192 0.005 0 0 0 0.005 0 0 +1.22437 0.005 0 0 0 0.005 0 0 +1.22681 0.005 0 0 0 0.005 0 0 +1.22925 0.005 0 0 0 0.005 0 0 +1.23169 0.005 0 0 0 0.005 0 0 +1.23413 0.005 0 0 0 0.005 0 0 +1.23657 0.005 0 0 0 0.005 0 0 +1.23901 0.005 0 0 0 0.005 0 0 +1.24146 0.005 0 0 0 0.005 0 0 +1.2439 0.005 0 0 0 0.005 0 0 +1.24634 0.005 0 0 0 0.005 0 0 +1.24878 0.005 0 0 0 0.005 0 0 +1.25122 0.005 0 0 0 0.005 0 0 +1.25366 0.005 0 0 0 0.005 0 0 +1.2561 0.005 0 0 0 0.005 0 0 +1.25854 0.005 0 0 0 0.005 0 0 +1.26099 0.005 0 0 0 0.005 0 0 +1.26343 0.005 0 0 0 0.005 0 0 +1.26587 0.005 0 0 0 0.005 0 0 +1.26831 0.005 0 0 0 0.005 0 0 +1.27075 0.005 0 0 0 0.005 0 0 +1.27319 0.005 0 0 0 0.005 0 0 +1.27563 0.005 0 0 0 0.005 0 0 +1.27808 0.005 0 0 0 0.005 0 0 +1.28052 0.005 0 0 0 0.005 0 0 +1.28296 0.005 0 0 0 0.005 0 0 +1.2854 0.005 0 0 0 0.005 0 0 +1.28784 0.005 0 0 0 0.005 0 0 +1.29028 0.005 0 0 0 0.005 0 0 +1.29272 0.005 0 0 0 0.005 0 0 +1.29517 0.005 0 0 0 0.005 0 0 +1.29761 0.005 0 0 0 0.005 0 0 +1.30005 0.005 0 0 0 0.005 0 0 +1.30249 0.005 0 0 0 0.005 0 0 +1.30493 0.005 0 0 0 0.005 0 0 +1.30737 0.005 0 0 0 0.005 0 0 +1.30981 0.005 0 0 0 0.005 0 0 +1.31226 0.005 0 0 0 0.005 0 0 +1.3147 0.005 0 0 0 0.005 0 0 +1.31714 0.005 0 0 0 0.005 0 0 +1.31958 0.005 0 0 0 0.005 0 0 +1.32202 0.005 0 0 0 0.005 0 0 +1.32446 0.005 0 0 0 0.005 0 0 +1.3269 0.005 0 0 0 0.005 0 0 +1.32935 0.005 0 0 0 0.005 0 0 +1.33179 0.005 0 0 0 0.005 0 0 +1.33423 0.005 0 0 0 0.005 0 0 +1.33667 0.005 0 0 0 0.005 0 0 +1.33911 0.005 0 0 0 0.005 0 0 +1.34155 0.005 0 0 0 0.005 0 0 +1.34399 0.005 0 0 0 0.005 0 0 +1.34644 0.005 0 0 0 0.005 0 0 +1.34888 0.005 0 0 0 0.005 0 0 +1.35132 0.005 0 0 0 0.005 0 0 +1.35376 0.005 0 0 0 0.005 0 0 +1.3562 0.005 0 0 0 0.005 0 0 +1.35864 0.005 0 0 0 0.005 0 0 +1.36108 0.005 0 0 0 0.005 0 0 +1.36353 0.005 0 0 0 0.005 0 0 +1.36597 0.005 0 0 0 0.005 0 0 +1.36841 0.005 0 0 0 0.005 0 0 +1.37085 0.005 0 0 0 0.005 0 0 +1.37329 0.005 0 0 0 0.005 0 0 +1.37573 0.005 0 0 0 0.005 0 0 +1.37817 0.005 0 0 0 0.005 0 0 +1.38062 0.005 0 0 0 0.005 0 0 +1.38306 0.005 0 0 0 0.005 0 0 +1.3855 0.005 0 0 0 0.005 0 0 +1.38794 0.005 0 0 0 0.005 0 0 +1.39038 0.005 0 0 0 0.005 0 0 +1.39282 0.005 0 0 0 0.005 0 0 +1.39526 0.005 0 0 0 0.005 0 0 +1.39771 0.005 0 0 0 0.005 0 0 +1.40015 0.005 0 0 0 0.005 0 0 +1.40259 0.005 0 0 0 0.005 0 0 +1.40503 0.005 0 0 0 0.005 0 0 +1.40747 0.005 0 0 0 0.005 0 0 +1.40991 0.005 0 0 0 0.005 0 0 +1.41235 0.005 0 0 0 0.005 0 0 +1.41479 0.005 0 0 0 0.005 0 0 +1.41724 0.005 0 0 0 0.005 0 0 +1.41968 0.005 0 0 0 0.005 0 0 +1.42212 0.005 0 0 0 0.005 0 0 +1.42456 0.005 0 0 0 0.005 0 0 +1.427 0.005 0 0 0 0.005 0 0 +1.42944 0.005 0 0 0 0.005 0 0 +1.43188 0.005 0 0 0 0.005 0 0 +1.43433 0.005 0 0 0 0.005 0 0 +1.43677 0.005 0 0 0 0.005 0 0 +1.43921 0.005 0 0 0 0.005 0 0 +1.44165 0.005 0 0 0 0.005 0 0 +1.44409 0.005 0 0 0 0.005 0 0 +1.44653 0.005 0 0 0 0.005 0 0 +1.44897 0.005 0 0 0 0.005 0 0 +1.45142 0.005 0 0 0 0.005 0 0 +1.45386 0.005 0 0 0 0.005 0 0 +1.4563 0.005 0 0 0 0.005 0 0 +1.45874 0.005 0 0 0 0.005 0 0 +1.46118 0.005 0 0 0 0.005 0 0 +1.46362 0.005 0 0 0 0.005 0 0 +1.46606 0.005 0 0 0 0.005 0 0 +1.46851 0.005 0 0 0 0.005 0 0 +1.47095 0.005 0 0 0 0.005 0 0 +1.47339 0.005 0 0 0 0.005 0 0 +1.47583 0.005 0 0 0 0.005 0 0 +1.47827 0.005 0 0 0 0.005 0 0 +1.48071 0.005 0 0 0 0.005 0 0 +1.48315 0.005 0 0 0 0.005 0 0 +1.4856 0.005 0 0 0 0.005 0 0 +1.48804 0.005 0 0 0 0.005 0 0 +1.49048 0.005 0 0 0 0.005 0 0 +1.49292 0.005 0 0 0 0.005 0 0 +1.49536 0.005 0 0 0 0.005 0 0 +1.4978 0.005 0 0 0 0.005 0 0 +1.50024 0.005 0 0 0 0.005 0 0 +1.50269 0.005 0 0 0 0.005 0 0 +1.50513 0.005 0 0 0 0.005 0 0 +1.50757 0.005 0 0 0 0.005 0 0 +1.51001 0.005 0 0 0 0.005 0 0 +1.51245 0.005 0 0 0 0.005 0 0 +1.51489 0.005 0 0 0 0.005 0 0 +1.51733 0.005 0 0 0 0.005 0 0 +1.51978 0.005 0 0 0 0.005 0 0 +1.52222 0.005 0 0 0 0.005 0 0 +1.52466 0.005 0 0 0 0.005 0 0 +1.5271 0.005 0 0 0 0.005 0 0 +1.52954 0.005 0 0 0 0.005 0 0 +1.53198 0.005 0 0 0 0.005 0 0 +1.53442 0.005 0 0 0 0.005 0 0 +1.53687 0.005 0 0 0 0.005 0 0 +1.53931 0.005 0 0 0 0.005 0 0 +1.54175 0.005 0 0 0 0.005 0 0 +1.54419 0.005 0 0 0 0.005 0 0 +1.54663 0.005 0 0 0 0.005 0 0 +1.54907 0.005 0 0 0 0.005 0 0 +1.55151 0.005 0 0 0 0.005 0 0 +1.55396 0.005 0 0 0 0.005 0 0 +1.5564 0.005 0 0 0 0.005 0 0 +1.55884 0.005 0 0 0 0.005 0 0 +1.56128 0.005 0 0 0 0.005 0 0 +1.56372 0.005 0 0 0 0.005 0 0 +1.56616 0.005 0 0 0 0.005 0 0 +1.5686 0.005 0 0 0 0.005 0 0 +1.57104 0.005 0 0 0 0.005 0 0 +1.57349 0.005 0 0 0 0.005 0 0 +1.57593 0.005 0 0 0 0.005 0 0 +1.57837 0.005 0 0 0 0.005 0 0 +1.58081 0.005 0 0 0 0.005 0 0 +1.58325 0.005 0 0 0 0.005 0 0 +1.58569 0.005 0 0 0 0.005 0 0 +1.58813 0.005 0 0 0 0.005 0 0 +1.59058 0.005 0 0 0 0.005 0 0 +1.59302 0.005 0 0 0 0.005 0 0 +1.59546 0.005 0 0 0 0.005 0 0 +1.5979 0.005 0 0 0 0.005 0 0 +1.60034 0.005 0 0 0 0.005 0 0 +1.60278 0.005 0 0 0 0.005 0 0 +1.60522 0.005 0 0 0 0.005 0 0 +1.60767 0.005 0 0 0 0.005 0 0 +1.61011 0.005 0 0 0 0.005 0 0 +1.61255 0.005 0 0 0 0.005 0 0 +1.61499 0.005 0 0 0 0.005 0 0 +1.61743 0.005 0 0 0 0.005 0 0 +1.61987 0.005 0 0 0 0.005 0 0 +1.62231 0.005 0 0 0 0.005 0 0 +1.62476 0.005 0 0 0 0.005 0 0 +1.6272 0.005 0 0 0 0.005 0 0 +1.62964 0.005 0 0 0 0.005 0 0 +1.63208 0.005 0 0 0 0.005 0 0 +1.63452 0.005 0 0 0 0.005 0 0 +1.63696 0.005 0 0 0 0.005 0 0 +1.6394 0.005 0 0 0 0.005 0 0 +1.64185 0.005 0 0 0 0.005 0 0 +1.64429 0.005 0 0 0 0.005 0 0 +1.64673 0.005 0 0 0 0.005 0 0 +1.64917 0.005 0 0 0 0.005 0 0 +1.65161 0.005 0 0 0 0.005 0 0 +1.65405 0.005 0 0 0 0.005 0 0 +1.65649 0.005 0 0 0 0.005 0 0 +1.65894 0.005 0 0 0 0.005 0 0 +1.66138 0.005 0 0 0 0.005 0 0 +1.66382 0.005 0 0 0 0.005 0 0 +1.66626 0.005 0 0 0 0.005 0 0 +1.6687 0.005 0 0 0 0.005 0 0 +1.67114 0.005 0 0 0 0.005 0 0 +1.67358 0.005 0 0 0 0.005 0 0 +1.67603 0.005 0 0 0 0.005 0 0 +1.67847 0.005 0 0 0 0.005 0 0 +1.68091 0.005 0 0 0 0.005 0 0 +1.68335 0.005 0 0 0 0.005 0 0 +1.68579 0.005 0 0 0 0.005 0 0 +1.68823 0.005 0 0 0 0.005 0 0 +1.69067 0.005 0 0 0 0.005 0 0 +1.69312 0.005 0 0 0 0.005 0 0 +1.69556 0.005 0 0 0 0.005 0 0 +1.698 0.005 0 0 0 0.005 0 0 +1.70044 0.005 0 0 0 0.005 0 0 +1.70288 0.005 0 0 0 0.005 0 0 +1.70532 0.005 0 0 0 0.005 0 0 +1.70776 0.005 0 0 0 0.005 0 0 +1.71021 0.005 0 0 0 0.005 0 0 +1.71265 0.005 0 0 0 0.005 0 0 +1.71509 0.005 0 0 0 0.005 0 0 +1.71753 0.005 0 0 0 0.005 0 0 +1.71997 0.005 0 0 0 0.005 0 0 +1.72241 0.005 0 0 0 0.005 0 0 +1.72485 0.005 0 0 0 0.005 0 0 +1.72729 0.005 0 0 0 0.005 0 0 +1.72974 0.005 0 0 0 0.005 0 0 +1.73218 0.005 0 0 0 0.005 0 0 +1.73462 0.005 0 0 0 0.005 0 0 +1.73706 0.005 0 0 0 0.005 0 0 +1.7395 0.005 0 0 0 0.005 0 0 +1.74194 0.005 0 0 0 0.005 0 0 +1.74438 0.005 0 0 0 0.005 0 0 +1.74683 0.005 0 0 0 0.005 0 0 +1.74927 0.005 0 0 0 0.005 0 0 +1.75171 0.005 0 0 0 0.005 0 0 +1.75415 0.005 0 0 0 0.005 0 0 +1.75659 0.005 0 0 0 0.005 0 0 +1.75903 0.005 0 0 0 0.005 0 0 +1.76147 0.005 0 0 0 0.005 0 0 +1.76392 0.005 0 0 0 0.005 0 0 +1.76636 0.005 0 0 0 0.005 0 0 +1.7688 0.005 0 0 0 0.005 0 0 +1.77124 0.005 0 0 0 0.005 0 0 +1.77368 0.005 0 0 0 0.005 0 0 +1.77612 0.005 0 0 0 0.005 0 0 +1.77856 0.005 0 0 0 0.005 0 0 +1.78101 0.005 0 0 0 0.005 0 0 +1.78345 0.005 0 0 0 0.005 0 0 +1.78589 0.005 0 0 0 0.005 0 0 +1.78833 0.005 0 0 0 0.005 0 0 +1.79077 0.005 0 0 0 0.005 0 0 +1.79321 0.005 0 0 0 0.005 0 0 +1.79565 0.005 0 0 0 0.005 0 0 +1.7981 0.005 0 0 0 0.005 0 0 +1.80054 0.005 0 0 0 0.005 0 0 +1.80298 0.005 0 0 0 0.005 0 0 +1.80542 0.005 0 0 0 0.005 0 0 +1.80786 0.005 0 0 0 0.005 0 0 +1.8103 0.005 0 0 0 0.005 0 0 +1.81274 0.005 0 0 0 0.005 0 0 +1.81519 0.005 0 0 0 0.005 0 0 +1.81763 0.005 0 0 0 0.005 0 0 +1.82007 0.005 0 0 0 0.005 0 0 +1.82251 0.005 0 0 0 0.005 0 0 +1.82495 0.005 0 0 0 0.005 0 0 +1.82739 0.005 0 0 0 0.005 0 0 +1.82983 0.005 0 0 0 0.005 0 0 +1.83228 0.005 0 0 0 0.005 0 0 +1.83472 0.005 0 0 0 0.005 0 0 +1.83716 0.005 0 0 0 0.005 0 0 +1.8396 0.005 0 0 0 0.005 0 0 +1.84204 0.005 0 0 0 0.005 0 0 +1.84448 0.005 0 0 0 0.005 0 0 +1.84692 0.005 0 0 0 0.005 0 0 +1.84937 0.005 0 0 0 0.005 0 0 +1.85181 0.005 0 0 0 0.005 0 0 +1.85425 0.005 0 0 0 0.005 0 0 +1.85669 0.005 0 0 0 0.005 0 0 +1.85913 0.005 0 0 0 0.005 0 0 +1.86157 0.005 0 0 0 0.005 0 0 +1.86401 0.005 0 0 0 0.005 0 0 +1.86646 0.005 0 0 0 0.005 0 0 +1.8689 0.005 0 0 0 0.005 0 0 +1.87134 0.005 0 0 0 0.005 0 0 +1.87378 0.005 0 0 0 0.005 0 0 +1.87622 0.005 0 0 0 0.005 0 0 +1.87866 0.005 0 0 0 0.005 0 0 +1.8811 0.005 0 0 0 0.005 0 0 +1.88354 0.005 0 0 0 0.005 0 0 +1.88599 0.005 0 0 0 0.005 0 0 +1.88843 0.005 0 0 0 0.005 0 0 +1.89087 0.005 0 0 0 0.005 0 0 +1.89331 0.005 0 0 0 0.005 0 0 +1.89575 0.005 0 0 0 0.005 0 0 +1.89819 0.005 0 0 0 0.005 0 0 +1.90063 0.005 0 0 0 0.005 0 0 +1.90308 0.005 0 0 0 0.005 0 0 +1.90552 0.005 0 0 0 0.005 0 0 +1.90796 0.005 0 0 0 0.005 0 0 +1.9104 0.005 0 0 0 0.005 0 0 +1.91284 0.005 0 0 0 0.005 0 0 +1.91528 0.005 0 0 0 0.005 0 0 +1.91772 0.005 0 0 0 0.005 0 0 +1.92017 0.005 0 0 0 0.005 0 0 +1.92261 0.005 0 0 0 0.005 0 0 +1.92505 0.005 0 0 0 0.005 0 0 +1.92749 0.005 0 0 0 0.005 0 0 +1.92993 0.005 0 0 0 0.005 0 0 +1.93237 0.005 0 0 0 0.005 0 0 +1.93481 0.005 0 0 0 0.005 0 0 +1.93726 0.005 0 0 0 0.005 0 0 +1.9397 0.005 0 0 0 0.005 0 0 +1.94214 0.005 0 0 0 0.005 0 0 +1.94458 0.005 0 0 0 0.005 0 0 +1.94702 0.005 0 0 0 0.005 0 0 +1.94946 0.005 0 0 0 0.005 0 0 +1.9519 0.005 0 0 0 0.005 0 0 +1.95435 0.005 0 0 0 0.005 0 0 +1.95679 0.005 0 0 0 0.005 0 0 +1.95923 0.005 0 0 0 0.005 0 0 +1.96167 0.005 0 0 0 0.005 0 0 +1.96411 0.005 0 0 0 0.005 0 0 +1.96655 0.005 0 0 0 0.005 0 0 +1.96899 0.005 0 0 0 0.005 0 0 +1.97144 0.005 0 0 0 0.005 0 0 +1.97388 0.005 0 0 0 0.005 0 0 +1.97632 0.005 0 0 0 0.005 0 0 +1.97876 0.005 0 0 0 0.005 0 0 +1.9812 0.005 0 0 0 0.005 0 0 +1.98364 0.005 0 0 0 0.005 0 0 +1.98608 0.005 0 0 0 0.005 0 0 +1.98853 0.005 0 0 0 0.005 0 0 +1.99097 0.005 0 0 0 0.005 0 0 +1.99341 0.005 0 0 0 0.005 0 0 +1.99585 0.005 0 0 0 0.005 0 0 +1.99829 0.005 0 0 0 0.005 0 0 +2.00073 0.005 0 0 0 0.005 0 0 +2.00317 0.005 0 0 0 0.005 0 0 +2.00562 0.005 0 0 0 0.005 0 0 +2.00806 0.005 0 0 0 0.005 0 0 +2.0105 0.005 0 0 0 0.005 0 0 +2.01294 0.005 0 0 0 0.005 0 0 +2.01538 0.005 0 0 0 0.005 0 0 +2.01782 0.005 0 0 0 0.005 0 0 +2.02026 0.005 0 0 0 0.005 0 0 +2.02271 0.005 0 0 0 0.005 0 0 +2.02515 0.005 0 0 0 0.005 0 0 +2.02759 0.005 0 0 0 0.005 0 0 +2.03003 0.005 0 0 0 0.005 0 0 +2.03247 0.005 0 0 0 0.005 0 0 +2.03491 0.005 0 0 0 0.005 0 0 +2.03735 0.005 0 0 0 0.005 0 0 +2.03979 0.005 0 0 0 0.005 0 0 +2.04224 0.005 0 0 0 0.005 0 0 +2.04468 0.005 0 0 0 0.005 0 0 +2.04712 0.005 0 0 0 0.005 0 0 +2.04956 0.005 0 0 0 0.005 0 0 +2.052 0.005 0 0 0 0.005 0 0 +2.05444 0.005 0 0 0 0.005 0 0 +2.05688 0.005 0 0 0 0.005 0 0 +2.05933 0.005 0 0 0 0.005 0 0 +2.06177 0.005 0 0 0 0.005 0 0 +2.06421 0.005 0 0 0 0.005 0 0 +2.06665 0.005 0 0 0 0.005 0 0 +2.06909 0.005 0 0 0 0.005 0 0 +2.07153 0.005 0 0 0 0.005 0 0 +2.07397 0.005 0 0 0 0.005 0 0 +2.07642 0.005 0 0 0 0.005 0 0 +2.07886 0.005 0 0 0 0.005 0 0 +2.0813 0.005 0 0 0 0.005 0 0 +2.08374 0.005 0 0 0 0.005 0 0 +2.08618 0.005 0 0 0 0.005 0 0 +2.08862 0.005 0 0 0 0.005 0 0 +2.09106 0.005 0 0 0 0.005 0 0 +2.09351 0.005 0 0 0 0.005 0 0 +2.09595 0.005 0 0 0 0.005 0 0 +2.09839 0.005 0 0 0 0.005 0 0 +2.10083 0.005 0 0 0 0.005 0 0 +2.10327 0.005 0 0 0 0.005 0 0 +2.10571 0.005 0 0 0 0.005 0 0 +2.10815 0.005 0 0 0 0.005 0 0 +2.1106 0.005 0 0 0 0.005 0 0 +2.11304 0.005 0 0 0 0.005 0 0 +2.11548 0.005 0 0 0 0.005 0 0 +2.11792 0.005 0 0 0 0.005 0 0 +2.12036 0.005 0 0 0 0.005 0 0 +2.1228 0.005 0 0 0 0.005 0 0 +2.12524 0.005 0 0 0 0.005 0 0 +2.12769 0.005 0 0 0 0.005 0 0 +2.13013 0.005 0 0 0 0.005 0 0 +2.13257 0.005 0 0 0 0.005 0 0 +2.13501 0.005 0 0 0 0.005 0 0 +2.13745 0.005 0 0 0 0.005 0 0 +2.13989 0.005 0 0 0 0.005 0 0 +2.14233 0.005 0 0 0 0.005 0 0 +2.14478 0.005 0 0 0 0.005 0 0 +2.14722 0.005 0 0 0 0.005 0 0 +2.14966 0.005 0 0 0 0.005 0 0 +2.1521 0.005 0 0 0 0.005 0 0 +2.15454 0.005 0 0 0 0.005 0 0 +2.15698 0.005 0 0 0 0.005 0 0 +2.15942 0.005 0 0 0 0.005 0 0 +2.16187 0.005 0 0 0 0.005 0 0 +2.16431 0.005 0 0 0 0.005 0 0 +2.16675 0.005 0 0 0 0.005 0 0 +2.16919 0.005 0 0 0 0.005 0 0 +2.17163 0.005 0 0 0 0.005 0 0 +2.17407 0.005 0 0 0 0.005 0 0 +2.17651 0.005 0 0 0 0.005 0 0 +2.17896 0.005 0 0 0 0.005 0 0 +2.1814 0.005 0 0 0 0.005 0 0 +2.18384 0.005 0 0 0 0.005 0 0 +2.18628 0.005 0 0 0 0.005 0 0 +2.18872 0.005 0 0 0 0.005 0 0 +2.19116 0.005 0 0 0 0.005 0 0 +2.1936 0.005 0 0 0 0.005 0 0 +2.19604 0.005 0 0 0 0.005 0 0 +2.19849 0.005 0 0 0 0.005 0 0 +2.20093 0.005 0 0 0 0.005 0 0 +2.20337 0.005 0 0 0 0.005 0 0 +2.20581 0.005 0 0 0 0.005 0 0 +2.20825 0.005 0 0 0 0.005 0 0 +2.21069 0.005 0 0 0 0.005 0 0 +2.21313 0.005 0 0 0 0.005 0 0 +2.21558 0.005 0 0 0 0.005 0 0 +2.21802 0.005 0 0 0 0.005 0 0 +2.22046 0.005 0 0 0 0.005 0 0 +2.2229 0.005 0 0 0 0.005 0 0 +2.22534 0.005 0 0 0 0.005 0 0 +2.22778 0.005 0 0 0 0.005 0 0 +2.23022 0.005 0 0 0 0.005 0 0 +2.23267 0.005 0 0 0 0.005 0 0 +2.23511 0.005 0 0 0 0.005 0 0 +2.23755 0.005 0 0 0 0.005 0 0 +2.23999 0.005 0 0 0 0.005 0 0 +2.24243 0.005 0 0 0 0.005 0 0 +2.24487 0.005 0 0 0 0.005 0 0 +2.24731 0.005 0 0 0 0.005 0 0 +2.24976 0.005 0 0 0 0.005 0 0 +2.2522 0.005 0 0 0 0.005 0 0 +2.25464 0.005 0 0 0 0.005 0 0 +2.25708 0.005 0 0 0 0.005 0 0 +2.25952 0.005 0 0 0 0.005 0 0 +2.26196 0.005 0 0 0 0.005 0 0 +2.2644 0.005 0 0 0 0.005 0 0 +2.26685 0.005 0 0 0 0.005 0 0 +2.26929 0.005 0 0 0 0.005 0 0 +2.27173 0.005 0 0 0 0.005 0 0 +2.27417 0.005 0 0 0 0.005 0 0 +2.27661 0.005 0 0 0 0.005 0 0 +2.27905 0.005 0 0 0 0.005 0 0 +2.28149 0.005 0 0 0 0.005 0 0 +2.28394 0.005 0 0 0 0.005 0 0 +2.28638 0.005 0 0 0 0.005 0 0 +2.28882 0.005 0 0 0 0.005 0 0 +2.29126 0.005 0 0 0 0.005 0 0 +2.2937 0.005 0 0 0 0.005 0 0 +2.29614 0.005 0 0 0 0.005 0 0 +2.29858 0.005 0 0 0 0.005 0 0 +2.30103 0.005 0 0 0 0.005 0 0 +2.30347 0.005 0 0 0 0.005 0 0 +2.30591 0.005 0 0 0 0.005 0 0 +2.30835 0.005 0 0 0 0.005 0 0 +2.31079 0.005 0 0 0 0.005 0 0 +2.31323 0.005 0 0 0 0.005 0 0 +2.31567 0.005 0 0 0 0.005 0 0 +2.31812 0.005 0 0 0 0.005 0 0 +2.32056 0.005 0 0 0 0.005 0 0 +2.323 0.005 0 0 0 0.005 0 0 +2.32544 0.005 0 0 0 0.005 0 0 +2.32788 0.005 0 0 0 0.005 0 0 +2.33032 0.005 0 0 0 0.005 0 0 +2.33276 0.005 0 0 0 0.005 0 0 +2.33521 0.005 0 0 0 0.005 0 0 +2.33765 0.005 0 0 0 0.005 0 0 +2.34009 0.005 0 0 0 0.005 0 0 +2.34253 0.005 0 0 0 0.005 0 0 +2.34497 0.005 0 0 0 0.005 0 0 +2.34741 0.005 0 0 0 0.005 0 0 +2.34985 0.005 0 0 0 0.005 0 0 +2.35229 0.005 0 0 0 0.005 0 0 +2.35474 0.005 0 0 0 0.005 0 0 +2.35718 0.005 0 0 0 0.005 0 0 +2.35962 0.005 0 0 0 0.005 0 0 +2.36206 0.005 0 0 0 0.005 0 0 +2.3645 0.005 0 0 0 0.005 0 0 +2.36694 0.005 0 0 0 0.005 0 0 +2.36938 0.005 0 0 0 0.005 0 0 +2.37183 0.005 0 0 0 0.005 0 0 +2.37427 0.005 0 0 0 0.005 0 0 +2.37671 0.005 0 0 0 0.005 0 0 +2.37915 0.005 0 0 0 0.005 0 0 +2.38159 0.005 0 0 0 0.005 0 0 +2.38403 0.005 0 0 0 0.005 0 0 +2.38647 0.005 0 0 0 0.005 0 0 +2.38892 0.005 0 0 0 0.005 0 0 +2.39136 0.005 0 0 0 0.005 0 0 +2.3938 0.005 0 0 0 0.005 0 0 +2.39624 0.005 0 0 0 0.005 0 0 +2.39868 0.005 0 0 0 0.005 0 0 +2.40112 0.005 0 0 0 0.005 0 0 +2.40356 0.005 0 0 0 0.005 0 0 +2.40601 0.005 0 0 0 0.005 0 0 +2.40845 0.005 0 0 0 0.005 0 0 +2.41089 0.005 0 0 0 0.005 0 0 +2.41333 0.005 0 0 0 0.005 0 0 +2.41577 0.005 0 0 0 0.005 0 0 +2.41821 0.005 0 0 0 0.005 0 0 +2.42065 0.005 0 0 0 0.005 0 0 +2.4231 0.005 0 0 0 0.005 0 0 +2.42554 0.005 0 0 0 0.005 0 0 +2.42798 0.005 0 0 0 0.005 0 0 +2.43042 0.005 0 0 0 0.005 0 0 +2.43286 0.005 0 0 0 0.005 0 0 +2.4353 0.005 0 0 0 0.005 0 0 +2.43774 0.005 0 0 0 0.005 0 0 +2.44019 0.005 0 0 0 0.005 0 0 +2.44263 0.005 0 0 0 0.005 0 0 +2.44507 0.005 0 0 0 0.005 0 0 +2.44751 0.005 0 0 0 0.005 0 0 +2.44995 0.005 0 0 0 0.005 0 0 +2.45239 0.005 0 0 0 0.005 0 0 +2.45483 0.005 0 0 0 0.005 0 0 +2.45728 0.005 0 0 0 0.005 0 0 +2.45972 0.005 0 0 0 0.005 0 0 +2.46216 0.005 0 0 0 0.005 0 0 +2.4646 0.005 0 0 0 0.005 0 0 +2.46704 0.005 0 0 0 0.005 0 0 +2.46948 0.005 0 0 0 0.005 0 0 +2.47192 0.005 0 0 0 0.005 0 0 +2.47437 0.005 0 0 0 0.005 0 0 +2.47681 0.005 0 0 0 0.005 0 0 +2.47925 0.005 0 0 0 0.005 0 0 +2.48169 0.005 0 0 0 0.005 0 0 +2.48413 0.005 0 0 0 0.005 0 0 +2.48657 0.005 0 0 0 0.005 0 0 +2.48901 0.005 0 0 0 0.005 0 0 +2.49146 0.005 0 0 0 0.005 0 0 +2.4939 0.005 0 0 0 0.005 0 0 +2.49634 0.005 0 0 0 0.005 0 0 +2.49878 0.005 0 0 0 0.005 0 0 +2.50122 0.005 0 0 0 0.005 0 0 +2.50366 0.005 0 0 0 0.005 0 0 +2.5061 0.005 0 0 0 0.005 0 0 +2.50854 0.005 0 0 0 0.005 0 0 +2.51099 0.005 0 0 0 0.005 0 0 +2.51343 0.005 0 0 0 0.005 0 0 +2.51587 0.005 0 0 0 0.005 0 0 +2.51831 0.005 0 0 0 0.005 0 0 +2.52075 0.005 0 0 0 0.005 0 0 +2.52319 0.005 0 0 0 0.005 0 0 +2.52563 0.005 0 0 0 0.005 0 0 +2.52808 0.005 0 0 0 0.005 0 0 +2.53052 0.005 0 0 0 0.005 0 0 +2.53296 0.005 0 0 0 0.005 0 0 +2.5354 0.005 0 0 0 0.005 0 0 +2.53784 0.005 0 0 0 0.005 0 0 +2.54028 0.005 0 0 0 0.005 0 0 +2.54272 0.005 0 0 0 0.005 0 0 +2.54517 0.005 0 0 0 0.005 0 0 +2.54761 0.005 0 0 0 0.005 0 0 +2.55005 0.005 0 0 0 0.005 0 0 +2.55249 0.005 0 0 0 0.005 0 0 +2.55493 0.005 0 0 0 0.005 0 0 +2.55737 0.005 0 0 0 0.005 0 0 +2.55981 0.005 0 0 0 0.005 0 0 +2.56226 0.005 0 0 0 0.005 0 0 +2.5647 0.005 0 0 0 0.005 0 0 +2.56714 0.005 0 0 0 0.005 0 0 +2.56958 0.005 0 0 0 0.005 0 0 +2.57202 0.005 0 0 0 0.005 0 0 +2.57446 0.005 0 0 0 0.005 0 0 +2.5769 0.005 0 0 0 0.005 0 0 +2.57935 0.005 0 0 0 0.005 0 0 +2.58179 0.005 0 0 0 0.005 0 0 +2.58423 0.005 0 0 0 0.005 0 0 +2.58667 0.005 0 0 0 0.005 0 0 +2.58911 0.005 0 0 0 0.005 0 0 +2.59155 0.005 0 0 0 0.005 0 0 +2.59399 0.005 0 0 0 0.005 0 0 +2.59644 0.005 0 0 0 0.005 0 0 +2.59888 0.005 0 0 0 0.005 0 0 +2.60132 0.005 0 0 0 0.005 0 0 +2.60376 0.005 0 0 0 0.005 0 0 +2.6062 0.005 0 0 0 0.005 0 0 +2.60864 0.005 0 0 0 0.005 0 0 +2.61108 0.005 0 0 0 0.005 0 0 +2.61353 0.005 0 0 0 0.005 0 0 +2.61597 0.005 0 0 0 0.005 0 0 +2.61841 0.005 0 0 0 0.005 0 0 +2.62085 0.005 0 0 0 0.005 0 0 +2.62329 0.005 0 0 0 0.005 0 0 +2.62573 0.005 0 0 0 0.005 0 0 +2.62817 0.005 0 0 0 0.005 0 0 +2.63062 0.005 0 0 0 0.005 0 0 +2.63306 0.005 0 0 0 0.005 0 0 +2.6355 0.005 0 0 0 0.005 0 0 +2.63794 0.005 0 0 0 0.005 0 0 +2.64038 0.005 0 0 0 0.005 0 0 +2.64282 0.005 0 0 0 0.005 0 0 +2.64526 0.005 0 0 0 0.005 0 0 +2.64771 0.005 0 0 0 0.005 0 0 +2.65015 0.005 0 0 0 0.005 0 0 +2.65259 0.005 0 0 0 0.005 0 0 +2.65503 0.005 0 0 0 0.005 0 0 +2.65747 0.005 0 0 0 0.005 0 0 +2.65991 0.005 0 0 0 0.005 0 0 +2.66235 0.005 0 0 0 0.005 0 0 +2.66479 0.005 0 0 0 0.005 0 0 +2.66724 0.005 0 0 0 0.005 0 0 +2.66968 0.005 0 0 0 0.005 0 0 +2.67212 0.005 0 0 0 0.005 0 0 +2.67456 0.005 0 0 0 0.005 0 0 +2.677 0.005 0 0 0 0.005 0 0 +2.67944 0.005 0 0 0 0.005 0 0 +2.68188 0.005 0 0 0 0.005 0 0 +2.68433 0.005 0 0 0 0.005 0 0 +2.68677 0.005 0 0 0 0.005 0 0 +2.68921 0.005 0 0 0 0.005 0 0 +2.69165 0.005 0 0 0 0.005 0 0 +2.69409 0.005 0 0 0 0.005 0 0 +2.69653 0.005 0 0 0 0.005 0 0 +2.69897 0.005 0 0 0 0.005 0 0 +2.70142 0.005 0 0 0 0.005 0 0 +2.70386 0.005 0 0 0 0.005 0 0 +2.7063 0.005 0 0 0 0.005 0 0 +2.70874 0.005 0 0 0 0.005 0 0 +2.71118 0.005 0 0 0 0.005 0 0 +2.71362 0.005 0 0 0 0.005 0 0 +2.71606 0.005 0 0 0 0.005 0 0 +2.71851 0.005 0 0 0 0.005 0 0 +2.72095 0.005 0 0 0 0.005 0 0 +2.72339 0.005 0 0 0 0.005 0 0 +2.72583 0.005 0 0 0 0.005 0 0 +2.72827 0.005 0 0 0 0.005 0 0 +2.73071 0.005 0 0 0 0.005 0 0 +2.73315 0.005 0 0 0 0.005 0 0 +2.7356 0.005 0 0 0 0.005 0 0 +2.73804 0.005 0 0 0 0.005 0 0 +2.74048 0.005 0 0 0 0.005 0 0 +2.74292 0.005 0 0 0 0.005 0 0 +2.74536 0.005 0 0 0 0.005 0 0 +2.7478 0.005 0 0 0 0.005 0 0 +2.75024 0.005 0 0 0 0.005 0 0 +2.75269 0.005 0 0 0 0.005 0 0 +2.75513 0.005 0 0 0 0.005 0 0 +2.75757 0.005 0 0 0 0.005 0 0 +2.76001 0.005 0 0 0 0.005 0 0 +2.76245 0.005 0 0 0 0.005 0 0 +2.76489 0.005 0 0 0 0.005 0 0 +2.76733 0.005 0 0 0 0.005 0 0 +2.76978 0.005 0 0 0 0.005 0 0 +2.77222 0.005 0 0 0 0.005 0 0 +2.77466 0.005 0 0 0 0.005 0 0 +2.7771 0.005 0 0 0 0.005 0 0 +2.77954 0.005 0 0 0 0.005 0 0 +2.78198 0.005 0 0 0 0.005 0 0 +2.78442 0.005 0 0 0 0.005 0 0 +2.78687 0.005 0 0 0 0.005 0 0 +2.78931 0.005 0 0 0 0.005 0 0 +2.79175 0.005 0 0 0 0.005 0 0 +2.79419 0.005 0 0 0 0.005 0 0 +2.79663 0.005 0 0 0 0.005 0 0 +2.79907 0.005 0 0 0 0.005 0 0 +2.80151 0.005 0 0 0 0.005 0 0 +2.80396 0.005 0 0 0 0.005 0 0 +2.8064 0.005 0 0 0 0.005 0 0 +2.80884 0.005 0 0 0 0.005 0 0 +2.81128 0.005 0 0 0 0.005 0 0 +2.81372 0.005 0 0 0 0.005 0 0 +2.81616 0.005 0 0 0 0.005 0 0 +2.8186 0.005 0 0 0 0.005 0 0 +2.82104 0.005 0 0 0 0.005 0 0 +2.82349 0.005 0 0 0 0.005 0 0 +2.82593 0.005 0 0 0 0.005 0 0 +2.82837 0.005 0 0 0 0.005 0 0 +2.83081 0.005 0 0 0 0.005 0 0 +2.83325 0.005 0 0 0 0.005 0 0 +2.83569 0.005 0 0 0 0.005 0 0 +2.83813 0.005 0 0 0 0.005 0 0 +2.84058 0.005 0 0 0 0.005 0 0 +2.84302 0.005 0 0 0 0.005 0 0 +2.84546 0.005 0 0 0 0.005 0 0 +2.8479 0.005 0 0 0 0.005 0 0 +2.85034 0.005 0 0 0 0.005 0 0 +2.85278 0.005 0 0 0 0.005 0 0 +2.85522 0.005 0 0 0 0.005 0 0 +2.85767 0.005 0 0 0 0.005 0 0 +2.86011 0.005 0 0 0 0.005 0 0 +2.86255 0.005 0 0 0 0.005 0 0 +2.86499 0.005 0 0 0 0.005 0 0 +2.86743 0.005 0 0 0 0.005 0 0 +2.86987 0.005 0 0 0 0.005 0 0 +2.87231 0.005 0 0 0 0.005 0 0 +2.87476 0.005 0 0 0 0.005 0 0 +2.8772 0.005 0 0 0 0.005 0 0 +2.87964 0.005 0 0 0 0.005 0 0 +2.88208 0.005 0 0 0 0.005 0 0 +2.88452 0.005 0 0 0 0.005 0 0 +2.88696 0.005 0 0 0 0.005 0 0 +2.8894 0.005 0 0 0 0.005 0 0 +2.89185 0.005 0 0 0 0.005 0 0 +2.89429 0.005 0 0 0 0.005 0 0 +2.89673 0.005 0 0 0 0.005 0 0 +2.89917 0.005 0 0 0 0.005 0 0 +2.90161 0.005 0 0 0 0.005 0 0 +2.90405 0.005 0 0 0 0.005 0 0 +2.90649 0.005 0 0 0 0.005 0 0 +2.90894 0.005 0 0 0 0.005 0 0 +2.91138 0.005 0 0 0 0.005 0 0 +2.91382 0.005 0 0 0 0.005 0 0 +2.91626 0.005 0 0 0 0.005 0 0 +2.9187 0.005 0 0 0 0.005 0 0 +2.92114 0.005 0 0 0 0.005 0 0 +2.92358 0.005 0 0 0 0.005 0 0 +2.92603 0.005 0 0 0 0.005 0 0 +2.92847 0.005 0 0 0 0.005 0 0 +2.93091 0.005 0 0 0 0.005 0 0 +2.93335 0.005 0 0 0 0.005 0 0 +2.93579 0.005 0 0 0 0.005 0 0 +2.93823 0.005 0 0 0 0.005 0 0 +2.94067 0.005 0 0 0 0.005 0 0 +2.94312 0.005 0 0 0 0.005 0 0 +2.94556 0.005 0 0 0 0.005 0 0 +2.948 0.005 0 0 0 0.005 0 0 +2.95044 0.005 0 0 0 0.005 0 0 +2.95288 0.005 0 0 0 0.005 0 0 +2.95532 0.005 0 0 0 0.005 0 0 +2.95776 0.005 0 0 0 0.005 0 0 +2.96021 0.005 0 0 0 0.005 0 0 +2.96265 0.005 0 0 0 0.005 0 0 +2.96509 0.005 0 0 0 0.005 0 0 +2.96753 0.005 0 0 0 0.005 0 0 +2.96997 0.005 0 0 0 0.005 0 0 +2.97241 0.005 0 0 0 0.005 0 0 +2.97485 0.005 0 0 0 0.005 0 0 +2.97729 0.005 0 0 0 0.005 0 0 +2.97974 0.005 0 0 0 0.005 0 0 +2.98218 0.005 0 0 0 0.005 0 0 +2.98462 0.005 0 0 0 0.005 0 0 +2.98706 0.005 0 0 0 0.005 0 0 +2.9895 0.005 0 0 0 0.005 0 0 +2.99194 0.005 0 0 0 0.005 0 0 +2.99438 0.005 0 0 0 0.005 0 0 +2.99683 0.005 0 0 0 0.005 0 0 +2.99927 0.005 0 0 0 0.005 0 0 +3.00171 0.005 0 0 0 0.005 0 0 +3.00415 0.005 0 0 0 0.005 0 0 +3.00659 0.005 0 0 0 0.005 0 0 +3.00903 0.005 0 0 0 0.005 0 0 +3.01147 0.005 0 0 0 0.005 0 0 +3.01392 0.005 0 0 0 0.005 0 0 +3.01636 0.005 0 0 0 0.005 0 0 +3.0188 0.005 0 0 0 0.005 0 0 +3.02124 0.005 0 0 0 0.005 0 0 +3.02368 0.005 0 0 0 0.005 0 0 +3.02612 0.005 0 0 0 0.005 0 0 +3.02856 0.005 0 0 0 0.005 0 0 +3.03101 0.005 0 0 0 0.005 0 0 +3.03345 0.005 0 0 0 0.005 0 0 +3.03589 0.005 0 0 0 0.005 0 0 +3.03833 0.005 0 0 0 0.005 0 0 +3.04077 0.005 0 0 0 0.005 0 0 +3.04321 0.005 0 0 0 0.005 0 0 +3.04565 0.005 0 0 0 0.005 0 0 +3.0481 0.005 0 0 0 0.005 0 0 +3.05054 0.005 0 0 0 0.005 0 0 +3.05298 0.005 0 0 0 0.005 0 0 +3.05542 0.005 0 0 0 0.005 0 0 +3.05786 0.005 0 0 0 0.005 0 0 +3.0603 0.005 0 0 0 0.005 0 0 +3.06274 0.005 0 0 0 0.005 0 0 +3.06519 0.005 0 0 0 0.005 0 0 +3.06763 0.005 0 0 0 0.005 0 0 +3.07007 0.005 0 0 0 0.005 0 0 +3.07251 0.005 0 0 0 0.005 0 0 +3.07495 0.005 0 0 0 0.005 0 0 +3.07739 0.005 0 0 0 0.005 0 0 +3.07983 0.005 0 0 0 0.005 0 0 +3.08228 0.005 0 0 0 0.005 0 0 +3.08472 0.005 0 0 0 0.005 0 0 +3.08716 0.005 0 0 0 0.005 0 0 +3.0896 0.005 0 0 0 0.005 0 0 +3.09204 0.005 0 0 0 0.005 0 0 +3.09448 0.005 0 0 0 0.005 0 0 +3.09692 0.005 0 0 0 0.005 0 0 +3.09937 0.005 0 0 0 0.005 0 0 +3.10181 0.005 0 0 0 0.005 0 0 +3.10425 0.005 0 0 0 0.005 0 0 +3.10669 0.005 0 0 0 0.005 0 0 +3.10913 0.005 0 0 0 0.005 0 0 +3.11157 0.005 0 0 0 0.005 0 0 +3.11401 0.005 0 0 0 0.005 0 0 +3.11646 0.005 0 0 0 0.005 0 0 +3.1189 0.005 0 0 0 0.005 0 0 +3.12134 0.005 0 0 0 0.005 0 0 +3.12378 0.005 0 0 0 0.005 0 0 +3.12622 0.005 0 0 0 0.005 0 0 +3.12866 0.005 0 0 0 0.005 0 0 +3.1311 0.005 0 0 0 0.005 0 0 +3.13354 0.005 0 0 0 0.005 0 0 +3.13599 0.005 0 0 0 0.005 0 0 +3.13843 0.005 0 0 0 0.005 0 0 +3.14087 0.005 0 0 0 0.005 0 0 +3.14331 0.005 0 0 0 0.005 0 0 +3.14575 0.005 0 0 0 0.005 0 0 +3.14819 0.005 0 0 0 0.005 0 0 +3.15063 0.005 0 0 0 0.005 0 0 +3.15308 0.005 0 0 0 0.005 0 0 +3.15552 0.005 0 0 0 0.005 0 0 +3.15796 0.005 0 0 0 0.005 0 0 +3.1604 0.005 0 0 0 0.005 0 0 +3.16284 0.005 0 0 0 0.005 0 0 +3.16528 0.005 0 0 0 0.005 0 0 +3.16772 0.005 0 0 0 0.005 0 0 +3.17017 0.005 0 0 0 0.005 0 0 +3.17261 0.005 0 0 0 0.005 0 0 +3.17505 0.005 0 0 0 0.005 0 0 +3.17749 0.005 0 0 0 0.005 0 0 +3.17993 0.005 0 0 0 0.005 0 0 +3.18237 0.005 0 0 0 0.005 0 0 +3.18481 0.005 0 0 0 0.005 0 0 +3.18726 0.005 0 0 0 0.005 0 0 +3.1897 0.005 0 0 0 0.005 0 0 +3.19214 0.005 0 0 0 0.005 0 0 +3.19458 0.005 0 0 0 0.005 0 0 +3.19702 0.005 0 0 0 0.005 0 0 +3.19946 0.005 0 0 0 0.005 0 0 +3.2019 0.005 0 0 0 0.005 0 0 +3.20435 0.005 0 0 0 0.005 0 0 +3.20679 0.005 0 0 0 0.005 0 0 +3.20923 0.005 0 0 0 0.005 0 0 +3.21167 0.005 0 0 0 0.005 0 0 +3.21411 0.005 0 0 0 0.005 0 0 +3.21655 0.005 0 0 0 0.005 0 0 +3.21899 0.005 0 0 0 0.005 0 0 +3.22144 0.005 0 0 0 0.005 0 0 +3.22388 0.005 0 0 0 0.005 0 0 +3.22632 0.005 0 0 0 0.005 0 0 +3.22876 0.005 0 0 0 0.005 0 0 +3.2312 0.005 0 0 0 0.005 0 0 +3.23364 0.005 0 0 0 0.005 0 0 +3.23608 0.005 0 0 0 0.005 0 0 +3.23853 0.005 0 0 0 0.005 0 0 +3.24097 0.005 0 0 0 0.005 0 0 +3.24341 0.005 0 0 0 0.005 0 0 +3.24585 0.005 0 0 0 0.005 0 0 +3.24829 0.005 0 0 0 0.005 0 0 +3.25073 0.005 0 0 0 0.005 0 0 +3.25317 0.005 0 0 0 0.005 0 0 +3.25562 0.005 0 0 0 0.005 0 0 +3.25806 0.005 0 0 0 0.005 0 0 +3.2605 0.005 0 0 0 0.005 0 0 +3.26294 0.005 0 0 0 0.005 0 0 +3.26538 0.005 0 0 0 0.005 0 0 +3.26782 0.005 0 0 0 0.005 0 0 +3.27026 0.005 0 0 0 0.005 0 0 +3.27271 0.005 0 0 0 0.005 0 0 +3.27515 0.005 0 0 0 0.005 0 0 +3.27759 0.005 0 0 0 0.005 0 0 +3.28003 0.005 0 0 0 0.005 0 0 +3.28247 0.005 0 0 0 0.005 0 0 +3.28491 0.005 0 0 0 0.005 0 0 +3.28735 0.005 0 0 0 0.005 0 0 +3.28979 0.005 0 0 0 0.005 0 0 +3.29224 0.005 0 0 0 0.005 0 0 +3.29468 0.005 0 0 0 0.005 0 0 +3.29712 0.005 0 0 0 0.005 0 0 +3.29956 0.005 0 0 0 0.005 0 0 +3.302 0.005 0 0 0 0.005 0 0 +3.30444 0.005 0 0 0 0.005 0 0 +3.30688 0.005 0 0 0 0.005 0 0 +3.30933 0.005 0 0 0 0.005 0 0 +3.31177 0.005 0 0 0 0.005 0 0 +3.31421 0.005 0 0 0 0.005 0 0 +3.31665 0.005 0 0 0 0.005 0 0 +3.31909 0.005 0 0 0 0.005 0 0 +3.32153 0.005 0 0 0 0.005 0 0 +3.32397 0.005 0 0 0 0.005 0 0 +3.32642 0.005 0 0 0 0.005 0 0 +3.32886 0.005 0 0 0 0.005 0 0 +3.3313 0.005 0 0 0 0.005 0 0 +3.33374 0.005 0 0 0 0.005 0 0 +3.33618 0.005 0 0 0 0.005 0 0 +3.33862 0.005 0 0 0 0.005 0 0 +3.34106 0.005 0 0 0 0.005 0 0 +3.34351 0.005 0 0 0 0.005 0 0 +3.34595 0.005 0 0 0 0.005 0 0 +3.34839 0.005 0 0 0 0.005 0 0 +3.35083 0.005 0 0 0 0.005 0 0 +3.35327 0.005 0 0 0 0.005 0 0 +3.35571 0.005 0 0 0 0.005 0 0 +3.35815 0.005 0 0 0 0.005 0 0 +3.3606 0.005 0 0 0 0.005 0 0 +3.36304 0.005 0 0 0 0.005 0 0 +3.36548 0.005 0 0 0 0.005 0 0 +3.36792 0.005 0 0 0 0.005 0 0 +3.37036 0.005 0 0 0 0.005 0 0 +3.3728 0.005 0 0 0 0.005 0 0 +3.37524 0.005 0 0 0 0.005 0 0 +3.37769 0.005 0 0 0 0.005 0 0 +3.38013 0.005 0 0 0 0.005 0 0 +3.38257 0.005 0 0 0 0.005 0 0 +3.38501 0.005 0 0 0 0.005 0 0 +3.38745 0.005 0 0 0 0.005 0 0 +3.38989 0.005 0 0 0 0.005 0 0 +3.39233 0.005 0 0 0 0.005 0 0 +3.39478 0.005 0 0 0 0.005 0 0 +3.39722 0.005 0 0 0 0.005 0 0 +3.39966 0.005 0 0 0 0.005 0 0 +3.4021 0.005 0 0 0 0.005 0 0 +3.40454 0.005 0 0 0 0.005 0 0 +3.40698 0.005 0 0 0 0.005 0 0 +3.40942 0.005 0 0 0 0.005 0 0 +3.41187 0.005 0 0 0 0.005 0 0 +3.41431 0.005 0 0 0 0.005 0 0 +3.41675 0.005 0 0 0 0.005 0 0 +3.41919 0.005 0 0 0 0.005 0 0 +3.42163 0.005 0 0 0 0.005 0 0 +3.42407 0.005 0 0 0 0.005 0 0 +3.42651 0.005 0 0 0 0.005 0 0 +3.42896 0.005 0 0 0 0.005 0 0 +3.4314 0.005 0 0 0 0.005 0 0 +3.43384 0.005 0 0 0 0.005 0 0 +3.43628 0.005 0 0 0 0.005 0 0 +3.43872 0.005 0 0 0 0.005 0 0 +3.44116 0.005 0 0 0 0.005 0 0 +3.4436 0.005 0 0 0 0.005 0 0 +3.44604 0.005 0 0 0 0.005 0 0 +3.44849 0.005 0 0 0 0.005 0 0 +3.45093 0.005 0 0 0 0.005 0 0 +3.45337 0.005 0 0 0 0.005 0 0 +3.45581 0.005 0 0 0 0.005 0 0 +3.45825 0.005 0 0 0 0.005 0 0 +3.46069 0.005 0 0 0 0.005 0 0 +3.46313 0.005 0 0 0 0.005 0 0 +3.46558 0.005 0 0 0 0.005 0 0 +3.46802 0.005 0 0 0 0.005 0 0 +3.47046 0.005 0 0 0 0.005 0 0 +3.4729 0.005 0 0 0 0.005 0 0 +3.47534 0.005 0 0 0 0.005 0 0 +3.47778 0.005 0 0 0 0.005 0 0 +3.48022 0.005 0 0 0 0.005 0 0 +3.48267 0.005 0 0 0 0.005 0 0 +3.48511 0.005 0 0 0 0.005 0 0 +3.48755 0.005 0 0 0 0.005 0 0 +3.48999 0.005 0 0 0 0.005 0 0 +3.49243 0.005 0 0 0 0.005 0 0 +3.49487 0.005 0 0 0 0.005 0 0 +3.49731 0.005 0 0 0 0.005 0 0 +3.49976 0.005 0 0 0 0.005 0 0 +3.5022 0.005 0 0 0 0.005 0 0 +3.50464 0.005 0 0 0 0.005 0 0 +3.50708 0.005 0 0 0 0.005 0 0 +3.50952 0.005 0 0 0 0.005 0 0 +3.51196 0.005 0 0 0 0.005 0 0 +3.5144 0.005 0 0 0 0.005 0 0 +3.51685 0.005 0 0 0 0.005 0 0 +3.51929 0.005 0 0 0 0.005 0 0 +3.52173 0.005 0 0 0 0.005 0 0 +3.52417 0.005 0 0 0 0.005 0 0 +3.52661 0.005 0 0 0 0.005 0 0 +3.52905 0.005 0 0 0 0.005 0 0 +3.53149 0.005 0 0 0 0.005 0 0 +3.53394 0.005 0 0 0 0.005 0 0 +3.53638 0.005 0 0 0 0.005 0 0 +3.53882 0.005 0 0 0 0.005 0 0 +3.54126 0.005 0 0 0 0.005 0 0 +3.5437 0.005 0 0 0 0.005 0 0 +3.54614 0.005 0 0 0 0.005 0 0 +3.54858 0.005 0 0 0 0.005 0 0 +3.55103 0.005 0 0 0 0.005 0 0 +3.55347 0.005 0 0 0 0.005 0 0 +3.55591 0.005 0 0 0 0.005 0 0 +3.55835 0.005 0 0 0 0.005 0 0 +3.56079 0.005 0 0 0 0.005 0 0 +3.56323 0.005 0 0 0 0.005 0 0 +3.56567 0.005 0 0 0 0.005 0 0 +3.56812 0.005 0 0 0 0.005 0 0 +3.57056 0.005 0 0 0 0.005 0 0 +3.573 0.005 0 0 0 0.005 0 0 +3.57544 0.005 0 0 0 0.005 0 0 +3.57788 0.005 0 0 0 0.005 0 0 +3.58032 0.005 0 0 0 0.005 0 0 +3.58276 0.005 0 0 0 0.005 0 0 +3.58521 0.005 0 0 0 0.005 0 0 +3.58765 0.005 0 0 0 0.005 0 0 +3.59009 0.005 0 0 0 0.005 0 0 +3.59253 0.005 0 0 0 0.005 0 0 +3.59497 0.005 0 0 0 0.005 0 0 +3.59741 0.005 0 0 0 0.005 0 0 +3.59985 0.005 0 0 0 0.005 0 0 +3.60229 0.005 0 0 0 0.005 0 0 +3.60474 0.005 0 0 0 0.005 0 0 +3.60718 0.005 0 0 0 0.005 0 0 +3.60962 0.005 0 0 0 0.005 0 0 +3.61206 0.005 0 0 0 0.005 0 0 +3.6145 0.005 0 0 0 0.005 0 0 +3.61694 0.005 0 0 0 0.005 0 0 +3.61938 0.005 0 0 0 0.005 0 0 +3.62183 0.005 0 0 0 0.005 0 0 +3.62427 0.005 0 0 0 0.005 0 0 +3.62671 0.005 0 0 0 0.005 0 0 +3.62915 0.005 0 0 0 0.005 0 0 +3.63159 0.005 0 0 0 0.005 0 0 +3.63403 0.005 0 0 0 0.005 0 0 +3.63647 0.005 0 0 0 0.005 0 0 +3.63892 0.005 0 0 0 0.005 0 0 +3.64136 0.005 0 0 0 0.005 0 0 +3.6438 0.005 0 0 0 0.005 0 0 +3.64624 0.005 0 0 0 0.005 0 0 +3.64868 0.005 0 0 0 0.005 0 0 +3.65112 0.005 0 0 0 0.005 0 0 +3.65356 0.005 0 0 0 0.005 0 0 +3.65601 0.005 0 0 0 0.005 0 0 +3.65845 0.005 0 0 0 0.005 0 0 +3.66089 0.005 0 0 0 0.005 0 0 +3.66333 0.005 0 0 0 0.005 0 0 +3.66577 0.005 0 0 0 0.005 0 0 +3.66821 0.005 0 0 0 0.005 0 0 +3.67065 0.005 0 0 0 0.005 0 0 +3.6731 0.00499516 0.00021442 0 1.07106e-006 0.00499516 0.000968625 4.89012e-005 +3.67554 0.00498904 0.000485687 0 2.42311e-006 0.00498904 0.0021954 8.42738e-005 +3.67798 0.00498293 0.000756955 0 3.77185e-006 0.00498293 0.00342368 0.000113191 +3.68042 0.00497681 0.00102822 0 5.11727e-006 0.00497681 0.00465347 0.000138719 +3.68286 0.00497071 0.00129949 0 6.45938e-006 0.00497071 0.00588477 0.000162021 +3.6853 0.0049646 0.00157076 0 7.79818e-006 0.0049646 0.00711758 0.000183699 +3.68774 0.0049585 0.00184202 0 9.13368e-006 0.0049585 0.00835191 0.000204115 +3.69019 0.0049524 0.00211329 0 1.04659e-005 0.0049524 0.00958775 0.000223508 +3.69263 0.00494631 0.00238456 0 1.17948e-005 0.00494631 0.0108251 0.000242049 +3.69507 0.00494022 0.00265583 0 1.31204e-005 0.00494022 0.012064 0.000259861 +3.69751 0.00493414 0.00292709 0 1.44427e-005 0.00493414 0.0133044 0.00027704 +3.69995 0.00492805 0.00319836 0 1.57617e-005 0.00492805 0.0145464 0.000293661 +3.70239 0.00492198 0.00346963 0 1.70774e-005 0.00492198 0.0157899 0.000309784 +3.70483 0.0049159 0.0037409 0 1.83899e-005 0.0049159 0.0170349 0.000325459 +3.70728 0.00490983 0.00401216 0 1.9699e-005 0.00490983 0.0182815 0.000340727 +3.70972 0.00490376 0.00428343 0 2.10049e-005 0.00490376 0.0195296 0.000355624 +3.71216 0.0048977 0.0045547 0 2.23075e-005 0.0048977 0.0207792 0.000370179 +3.7146 0.00489164 0.00482596 0 2.36069e-005 0.00489164 0.0220304 0.000384417 +3.71704 0.00488559 0.00509723 0 2.4903e-005 0.00488559 0.0232831 0.000398362 +3.71948 0.00487953 0.0053685 0 2.61958e-005 0.00487953 0.0245374 0.000412032 +3.72192 0.00487349 0.00563977 0 2.74853e-005 0.00487349 0.0257933 0.000425446 +3.72437 0.00486744 0.00591103 0 2.87716e-005 0.00486744 0.0270507 0.000438618 +3.72681 0.0048614 0.0061823 0 3.00546e-005 0.0048614 0.0283097 0.000451563 +3.72925 0.00485536 0.00645357 0 3.13344e-005 0.00485536 0.0295702 0.000464292 +3.73169 0.00484933 0.00672484 0 3.2611e-005 0.00484933 0.0308323 0.000476818 +3.73413 0.0048433 0.0069961 0 3.38842e-005 0.0048433 0.032096 0.00048915 +3.73657 0.00483728 0.00726737 0 3.51543e-005 0.00483728 0.0333613 0.000501298 +3.73901 0.00483125 0.00753864 0 3.64211e-005 0.00483125 0.0346281 0.00051327 +3.74146 0.00482524 0.00780991 0 3.76846e-005 0.00482524 0.0358965 0.000525073 +3.7439 0.00481922 0.00808117 0 3.8945e-005 0.00481922 0.0371665 0.000536716 +3.74634 0.00481321 0.00835244 0 4.02021e-005 0.00481321 0.0384381 0.000548204 +3.74878 0.0048072 0.00862371 0 4.14559e-005 0.0048072 0.0397112 0.000559545 +3.75122 0.0048012 0.00889498 0 4.27066e-005 0.0048012 0.040986 0.000570742 +3.75366 0.0047952 0.00916624 0 4.3954e-005 0.0047952 0.0422623 0.000581803 +3.7561 0.00478921 0.00943751 0 4.51982e-005 0.00478921 0.0435403 0.000592731 +3.75854 0.00478322 0.00970878 0 4.64392e-005 0.00478322 0.0448198 0.000603532 +3.76099 0.00477723 0.00998004 0 4.76769e-005 0.00477723 0.046101 0.000614209 +3.76343 0.00477124 0.0102513 0 4.89115e-005 0.00477124 0.0473837 0.000624766 +3.76587 0.00476526 0.0105226 0 5.01428e-005 0.00476526 0.0486681 0.000635209 +3.76831 0.00475929 0.0107938 0 5.1371e-005 0.00475929 0.0499541 0.000645539 +3.77075 0.00475331 0.0110651 0 5.25959e-005 0.00475331 0.0512417 0.00065576 +3.77319 0.00474734 0.0113364 0 5.38177e-005 0.00474734 0.0525309 0.000665877 +3.77563 0.00474138 0.0116076 0 5.50362e-005 0.00474138 0.0538217 0.00067589 +3.77808 0.00473542 0.0118789 0 5.62516e-005 0.00473542 0.0551142 0.000685804 +3.78052 0.00472946 0.0121502 0 5.74638e-005 0.00472946 0.0564083 0.000695622 +3.78296 0.0047235 0.0124215 0 5.86728e-005 0.0047235 0.057704 0.000705345 +3.7854 0.00471755 0.0126927 0 5.98786e-005 0.00471755 0.0590013 0.000714976 +3.78784 0.00471161 0.012964 0 6.10812e-005 0.00471161 0.0603003 0.000724517 +3.79028 0.00470566 0.0132353 0 6.22806e-005 0.00470566 0.0616009 0.000733971 +3.79272 0.00469972 0.0135065 0 6.34769e-005 0.00469972 0.0629032 0.00074334 +3.79517 0.00469379 0.0137778 0 6.467e-005 0.00469379 0.0642071 0.000752626 +3.79761 0.00468786 0.0140491 0 6.58599e-005 0.00468786 0.0655127 0.00076183 +3.80005 0.00468193 0.0143203 0 6.70467e-005 0.00468193 0.0668199 0.000770954 +3.80249 0.004676 0.0145916 0 6.82303e-005 0.004676 0.0681288 0.000780001 +3.80493 0.00467008 0.0148629 0 6.94108e-005 0.00467008 0.0694393 0.000788972 +3.80737 0.00466417 0.0151341 0 7.05881e-005 0.00466417 0.0707515 0.000797868 +3.80981 0.00465825 0.0154054 0 7.17622e-005 0.00465825 0.0720654 0.000806692 +3.81226 0.00465234 0.0156767 0 7.29332e-005 0.00465234 0.0733809 0.000815444 +3.8147 0.00464644 0.0159479 0 7.41011e-005 0.00464644 0.0746981 0.000824125 +3.81714 0.00464054 0.0162192 0 7.52658e-005 0.00464054 0.076017 0.000832738 +3.81958 0.00463464 0.0164905 0 7.64273e-005 0.00463464 0.0773375 0.000841284 +3.82202 0.00462874 0.0167617 0 7.75858e-005 0.00462874 0.0786598 0.000849764 +3.82446 0.00462285 0.017033 0 7.8741e-005 0.00462285 0.0799837 0.000858179 +3.8269 0.00461697 0.0173043 0 7.98932e-005 0.00461697 0.0813093 0.00086653 +3.82935 0.00461108 0.0175755 0 8.10422e-005 0.00461108 0.0826366 0.000874819 +3.83179 0.0046052 0.0178468 0 8.21882e-005 0.0046052 0.0839656 0.000883046 +3.83423 0.00459933 0.0181181 0 8.33309e-005 0.00459933 0.0852963 0.000891212 +3.83667 0.00459346 0.0183893 0 8.44706e-005 0.00459346 0.0866287 0.00089932 +3.83911 0.00458759 0.0186606 0 8.56072e-005 0.00458759 0.0879628 0.000907369 +3.84155 0.00458172 0.0189319 0 8.67406e-005 0.00458172 0.0892986 0.00091536 +3.84399 0.00457586 0.0192031 0 8.78709e-005 0.00457586 0.0906361 0.000923295 +3.84644 0.00457001 0.0194744 0 8.89982e-005 0.00457001 0.0919753 0.000931174 +3.84888 0.00456415 0.0197457 0 9.01223e-005 0.00456415 0.0933162 0.000938999 +3.85132 0.0045583 0.0200169 0 9.12433e-005 0.0045583 0.0946589 0.00094677 +3.85376 0.00455246 0.0202882 0 9.23612e-005 0.00455246 0.0960033 0.000954487 +3.8562 0.00454662 0.0205595 0 9.34761e-005 0.00454662 0.0973494 0.000962153 +3.85864 0.00454078 0.0208307 0 9.45878e-005 0.00454078 0.0986972 0.000969766 +3.86108 0.00453495 0.021102 0 9.56964e-005 0.00453495 0.100047 0.000977329 +3.86353 0.00452911 0.0213733 0 9.6802e-005 0.00452911 0.101398 0.000984842 +3.86597 0.00452329 0.0216445 0 9.79045e-005 0.00452329 0.102751 0.000992306 +3.86841 0.00451746 0.0219158 0 9.90039e-005 0.00451746 0.104106 0.00099972 +3.87085 0.00451165 0.0221871 0 0.0001001 0.00451165 0.105463 0.00100709 +3.87329 0.00450583 0.0224583 0 0.000101193 0.00450583 0.106821 0.00101441 +3.87573 0.00450002 0.0227296 0 0.000102284 0.00450002 0.108181 0.00102168 +3.87817 0.00449421 0.0230009 0 0.000103371 0.00449421 0.109543 0.00102891 +3.88062 0.00448841 0.0232721 0 0.000104455 0.00448841 0.110906 0.00103609 +3.88306 0.00448261 0.0235434 0 0.000105536 0.00448261 0.112272 0.00104322 +3.8855 0.00447681 0.0238147 0 0.000106614 0.00447681 0.113639 0.00105031 +3.88794 0.00447102 0.0240859 0 0.000107689 0.00447102 0.115007 0.00105736 +3.89038 0.00446523 0.0243572 0 0.00010876 0.00446523 0.116378 0.00106437 +3.89282 0.00445944 0.0246285 0 0.000109829 0.00445944 0.117751 0.00107133 +3.89526 0.00445366 0.0248997 0 0.000110895 0.00445366 0.119125 0.00107825 +3.89771 0.00444788 0.025171 0 0.000111958 0.00444788 0.120501 0.00108512 +3.90015 0.00444211 0.0254423 0 0.000113017 0.00444211 0.121879 0.00109196 +3.90259 0.00443634 0.0257136 0 0.000114074 0.00443634 0.123258 0.00109876 +3.90503 0.00443057 0.0259848 0 0.000115128 0.00443057 0.124639 0.00110551 +3.90747 0.00442481 0.0262561 0 0.000116178 0.00442481 0.126023 0.00111223 +3.90991 0.00441905 0.0265274 0 0.000117226 0.00441905 0.127408 0.0011189 +3.91235 0.00441329 0.0267986 0 0.00011827 0.00441329 0.128794 0.00112554 +3.91479 0.00440754 0.0270699 0 0.000119312 0.00440754 0.130183 0.00113214 +3.91724 0.00440179 0.0273412 0 0.00012035 0.00440179 0.131573 0.0011387 +3.91968 0.00439605 0.0276124 0 0.000121386 0.00439605 0.132965 0.00114522 +3.92212 0.00439031 0.0278837 0 0.000122418 0.00439031 0.134359 0.0011517 +3.92456 0.00438457 0.028155 0 0.000123447 0.00438457 0.135755 0.00115815 +3.927 0.00437884 0.0284262 0 0.000124474 0.00437884 0.137153 0.00116456 +3.92944 0.00437311 0.0286975 0 0.000125497 0.00437311 0.138553 0.00117094 +3.93188 0.00436738 0.0289688 0 0.000126518 0.00436738 0.139954 0.00117727 +3.93433 0.00436166 0.02924 0 0.000127535 0.00436166 0.141357 0.00118358 +3.93677 0.00435594 0.0295113 0 0.000128549 0.00435594 0.142762 0.00118985 +3.93921 0.00435023 0.0297826 0 0.000129561 0.00435023 0.144169 0.00119608 +3.94165 0.00434452 0.0300538 0 0.000130569 0.00434452 0.145578 0.00120228 +3.94409 0.00433881 0.0303251 0 0.000131575 0.00433881 0.146988 0.00120844 +3.94653 0.00433311 0.0305964 0 0.000132577 0.00433311 0.148401 0.00121457 +3.94897 0.00432741 0.0308676 0 0.000133577 0.00432741 0.149815 0.00122067 +3.95142 0.00432171 0.0311389 0 0.000134573 0.00432171 0.151231 0.00122673 +3.95386 0.00431602 0.0314102 0 0.000135567 0.00431602 0.152649 0.00123276 +3.9563 0.00431033 0.0316814 0 0.000136558 0.00431033 0.154069 0.00123876 +3.95874 0.00430465 0.0319527 0 0.000137545 0.00430465 0.155491 0.00124473 +3.96118 0.00429897 0.032224 0 0.00013853 0.00429897 0.156914 0.00125066 +3.96362 0.00429329 0.0324952 0 0.000139512 0.00429329 0.15834 0.00125656 +3.96606 0.00428762 0.0327665 0 0.00014049 0.00428762 0.159767 0.00126243 +3.96851 0.00428195 0.0330378 0 0.000141466 0.00428195 0.161196 0.00126827 +3.97095 0.00427628 0.033309 0 0.000142439 0.00427628 0.162628 0.00127408 +3.97339 0.00427062 0.0335803 0 0.000143409 0.00427062 0.164061 0.00127986 +3.97583 0.00426496 0.0338516 0 0.000144376 0.00426496 0.165496 0.0012856 +3.97827 0.00425931 0.0341228 0 0.00014534 0.00425931 0.166933 0.00129132 +3.98071 0.00425366 0.0343941 0 0.000146301 0.00425366 0.168371 0.001297 +3.98315 0.00424801 0.0346654 0 0.000147259 0.00424801 0.169812 0.00130266 +3.9856 0.00424237 0.0349366 0 0.000148214 0.00424237 0.171255 0.00130829 +3.98804 0.00423673 0.0352079 0 0.000149166 0.00423673 0.172699 0.00131389 +3.99048 0.00423109 0.0354792 0 0.000150116 0.00423109 0.174146 0.00131945 +3.99292 0.00422546 0.0357504 0 0.000151062 0.00422546 0.175594 0.00132499 +3.99536 0.00421983 0.0360217 0 0.000152006 0.00421983 0.177044 0.00133051 +3.9978 0.00421421 0.036293 0 0.000152946 0.00421421 0.178497 0.00133599 +4.00024 0.00420859 0.0365642 0 0.000153884 0.00420859 0.179951 0.00134144 +4.00269 0.00420297 0.0368355 0 0.000154819 0.00420297 0.181407 0.00134687 +4.00513 0.00419736 0.0371068 0 0.000155751 0.00419736 0.182865 0.00135227 +4.00757 0.00419175 0.037378 0 0.000156679 0.00419175 0.184325 0.00135764 +4.01001 0.00418615 0.0376493 0 0.000157606 0.00418615 0.185787 0.00136299 +4.01245 0.00418054 0.0379206 0 0.000158529 0.00418054 0.187251 0.0013683 +4.01489 0.00417495 0.0381919 0 0.000159449 0.00417495 0.188717 0.00137359 +4.01733 0.00416935 0.0384631 0 0.000160366 0.00416935 0.190185 0.00137886 +4.01978 0.00416376 0.0387344 0 0.000161281 0.00416376 0.191655 0.00138409 +4.02222 0.00415817 0.0390057 0 0.000162192 0.00415817 0.193126 0.0013893 +4.02466 0.00415259 0.0392769 0 0.000163101 0.00415259 0.1946 0.00139449 +4.0271 0.00414701 0.0395482 0 0.000164007 0.00414701 0.196076 0.00139965 +4.02954 0.00414144 0.0398195 0 0.00016491 0.00414144 0.197554 0.00140478 +4.03198 0.00413586 0.0400907 0 0.00016581 0.00413586 0.199034 0.00140989 +4.03442 0.0041303 0.040362 0 0.000166707 0.0041303 0.200515 0.00141497 +4.03687 0.00412473 0.0406333 0 0.000167601 0.00412473 0.201999 0.00142002 +4.03931 0.00411917 0.0409045 0 0.000168493 0.00411917 0.203485 0.00142505 +4.04175 0.00411361 0.0411758 0 0.000169381 0.00411361 0.204973 0.00143006 +4.04419 0.00410806 0.0414471 0 0.000170267 0.00410806 0.206462 0.00143504 +4.04663 0.00410251 0.0417183 0 0.00017115 0.00410251 0.207954 0.00144 +4.04907 0.00409697 0.0419896 0 0.00017203 0.00409697 0.209448 0.00144493 +4.05151 0.00409143 0.0422609 0 0.000172907 0.00409143 0.210944 0.00144984 +4.05396 0.00408589 0.0425321 0 0.000173781 0.00408589 0.212442 0.00145472 +4.0564 0.00408035 0.0428034 0 0.000174653 0.00408035 0.213941 0.00145958 +4.05884 0.00407482 0.0430747 0 0.000175522 0.00407482 0.215443 0.00146441 +4.06128 0.0040693 0.0433459 0 0.000176387 0.0040693 0.216947 0.00146923 +4.06372 0.00406377 0.0436172 0 0.00017725 0.00406377 0.218453 0.00147401 +4.06616 0.00405825 0.0438885 0 0.000178111 0.00405825 0.219961 0.00147878 +4.0686 0.00405274 0.0441597 0 0.000178968 0.00405274 0.221471 0.00148352 +4.07104 0.00404723 0.044431 0 0.000179822 0.00404723 0.222984 0.00148824 +4.07349 0.00404172 0.0447023 0 0.000180674 0.00404172 0.224498 0.00149293 +4.07593 0.00403621 0.0449735 0 0.000181523 0.00403621 0.226014 0.00149761 +4.07837 0.00403071 0.0452448 0 0.000182369 0.00403071 0.227532 0.00150226 +4.08081 0.00402522 0.0455161 0 0.000183212 0.00402522 0.229053 0.00150688 +4.08325 0.00401972 0.0457873 0 0.000184052 0.00401972 0.230575 0.00151149 +4.08569 0.00401423 0.0460586 0 0.00018489 0.00401423 0.2321 0.00151607 +4.08813 0.00400875 0.0463299 0 0.000185725 0.00400875 0.233627 0.00152063 +4.09058 0.00400327 0.0466011 0 0.000186557 0.00400327 0.235155 0.00152517 +4.09302 0.00399779 0.0468724 0 0.000187386 0.00399779 0.236686 0.00152968 +4.09546 0.00399232 0.0471437 0 0.000188212 0.00399232 0.238219 0.00153418 +4.0979 0.00398684 0.0474149 0 0.000189036 0.00398684 0.239754 0.00153865 +4.10034 0.00398138 0.0476862 0 0.000189857 0.00398138 0.241291 0.0015431 +4.10278 0.00397591 0.0479575 0 0.000190675 0.00397591 0.242831 0.00154753 +4.10522 0.00397046 0.0482287 0 0.00019149 0.00397046 0.244372 0.00155194 +4.10767 0.003965 0.0485 0 0.000192303 0.003965 0.245915 0.00155633 +4.11011 0.00395955 0.0487713 0 0.000193112 0.00395955 0.247461 0.00156069 +4.11255 0.0039541 0.0490425 0 0.000193919 0.0039541 0.249009 0.00156504 +4.11499 0.00394866 0.0493138 0 0.000194723 0.00394866 0.250559 0.00156936 +4.11743 0.00394322 0.0495851 0 0.000195525 0.00394322 0.252111 0.00157366 +4.11987 0.00393778 0.0498563 0 0.000196323 0.00393778 0.253665 0.00157794 +4.12231 0.00393235 0.0501276 0 0.000197119 0.00393235 0.255221 0.00158221 +4.12476 0.00392692 0.0503989 0 0.000197912 0.00392692 0.25678 0.00158645 +4.1272 0.00392149 0.0506701 0 0.000198703 0.00392149 0.25834 0.00159067 +4.12964 0.00391607 0.0509414 0 0.00019949 0.00391607 0.259903 0.00159487 +4.13208 0.00391065 0.0512127 0 0.000200275 0.00391065 0.261468 0.00159905 +4.13452 0.00390524 0.051484 0 0.000201057 0.00390524 0.263035 0.00160321 +4.13696 0.00389983 0.0517552 0 0.000201836 0.00389983 0.264604 0.00160735 +4.1394 0.00389442 0.0520265 0 0.000202613 0.00389442 0.266176 0.00161147 +4.14185 0.00388902 0.0522978 0 0.000203387 0.00388902 0.267749 0.00161557 +4.14429 0.00388362 0.052569 0 0.000204158 0.00388362 0.269325 0.00161965 +4.14673 0.00387822 0.0528403 0 0.000204926 0.00387822 0.270903 0.00162371 +4.14917 0.00387283 0.0531116 0 0.000205692 0.00387283 0.272484 0.00162775 +4.15161 0.00386744 0.0533828 0 0.000206455 0.00386744 0.274066 0.00163178 +4.15405 0.00386206 0.0536541 0 0.000207215 0.00386206 0.275651 0.00163578 +4.15649 0.00385668 0.0539254 0 0.000207973 0.00385668 0.277237 0.00163976 +4.15894 0.0038513 0.0541966 0 0.000208728 0.0038513 0.278826 0.00164373 +4.16138 0.00384593 0.0544679 0 0.00020948 0.00384593 0.280418 0.00164767 +4.16382 0.00384056 0.0547392 0 0.000210229 0.00384056 0.282011 0.0016516 +4.16626 0.00383519 0.0550104 0 0.000210976 0.00383519 0.283607 0.00165551 +4.1687 0.00382983 0.0552817 0 0.00021172 0.00382983 0.285205 0.0016594 +4.17114 0.00382447 0.055553 0 0.000212461 0.00382447 0.286805 0.00166327 +4.17358 0.00381912 0.0558242 0 0.000213199 0.00381912 0.288408 0.00166712 +4.17603 0.00381377 0.0560955 0 0.000213935 0.00381377 0.290012 0.00167096 +4.17847 0.00380842 0.0563668 0 0.000214668 0.00380842 0.291619 0.00167477 +4.18091 0.00380308 0.056638 0 0.000215399 0.00380308 0.293228 0.00167857 +4.18335 0.00379774 0.0569093 0 0.000216127 0.00379774 0.29484 0.00168235 +4.18579 0.0037924 0.0571806 0 0.000216852 0.0037924 0.296454 0.00168611 +4.18823 0.00378707 0.0574518 0 0.000217574 0.00378707 0.29807 0.00168985 +4.19067 0.00378175 0.0577231 0 0.000218294 0.00378175 0.299688 0.00169358 +4.19312 0.00377642 0.0579944 0 0.000219011 0.00377642 0.301308 0.00169728 +4.19556 0.0037711 0.0582656 0 0.000219726 0.0037711 0.302931 0.00170097 +4.198 0.00376578 0.0585369 0 0.000220437 0.00376578 0.304556 0.00170464 +4.20044 0.00376047 0.0588082 0 0.000221146 0.00376047 0.306184 0.0017083 +4.20288 0.00375516 0.0590794 0 0.000221853 0.00375516 0.307813 0.00171193 +4.20532 0.00374986 0.0593507 0 0.000222557 0.00374986 0.309445 0.00171555 +4.20776 0.00374455 0.059622 0 0.000223258 0.00374455 0.31108 0.00171915 +4.21021 0.00373926 0.0598932 0 0.000223956 0.00373926 0.312716 0.00172274 +4.21265 0.00373396 0.0601645 0 0.000224652 0.00373396 0.314355 0.0017263 +4.21509 0.00372867 0.0604358 0 0.000225345 0.00372867 0.315997 0.00172985 +4.21753 0.00372338 0.060707 0 0.000226036 0.00372338 0.31764 0.00173338 +4.21997 0.0037181 0.0609783 0 0.000226724 0.0037181 0.319286 0.0017369 +4.22241 0.00371282 0.0612496 0 0.000227409 0.00371282 0.320935 0.0017404 +4.22485 0.00370755 0.0615208 0 0.000228091 0.00370755 0.322585 0.00174388 +4.22729 0.00370228 0.0617921 0 0.000228771 0.00370228 0.324238 0.00174734 +4.22974 0.00369701 0.0620634 0 0.000229449 0.00369701 0.325894 0.00175079 +4.23218 0.00369174 0.0623346 0 0.000230124 0.00369174 0.327551 0.00175422 +4.23462 0.00368648 0.0626059 0 0.000230796 0.00368648 0.329211 0.00175764 +4.23706 0.00368123 0.0628772 0 0.000231465 0.00368123 0.330874 0.00176103 +4.2395 0.00367597 0.0631484 0 0.000232132 0.00367597 0.332539 0.00176441 +4.24194 0.00367072 0.0634197 0 0.000232796 0.00367072 0.334206 0.00176778 +4.24438 0.00366548 0.063691 0 0.000233458 0.00366548 0.335875 0.00177113 +4.24683 0.00366024 0.0639622 0 0.000234117 0.00366024 0.337547 0.00177446 +4.24927 0.003655 0.0642335 0 0.000234773 0.003655 0.339222 0.00177777 +4.25171 0.00364976 0.0645048 0 0.000235427 0.00364976 0.340898 0.00178107 +4.25415 0.00364453 0.0647761 0 0.000236079 0.00364453 0.342578 0.00178436 +4.25659 0.00363931 0.0650473 0 0.000236727 0.00363931 0.344259 0.00178762 +4.25903 0.00363408 0.0653186 0 0.000237373 0.00363408 0.345943 0.00179087 +4.26147 0.00362887 0.0655899 0 0.000238017 0.00362887 0.34763 0.00179411 +4.26392 0.00362365 0.0658611 0 0.000238658 0.00362365 0.349318 0.00179733 +4.26636 0.00361844 0.0661324 0 0.000239296 0.00361844 0.35101 0.00180053 +4.2688 0.00361323 0.0664037 0 0.000239932 0.00361323 0.352703 0.00180372 +4.27124 0.00360803 0.0666749 0 0.000240565 0.00360803 0.3544 0.00180689 +4.27368 0.00360283 0.0669462 0 0.000241195 0.00360283 0.356098 0.00181005 +4.27612 0.00359763 0.0672175 0 0.000241823 0.00359763 0.357799 0.00181319 +4.27856 0.00359244 0.0674887 0 0.000242449 0.00359244 0.359503 0.00181631 +4.28101 0.00358725 0.06776 0 0.000243072 0.00358725 0.361209 0.00181942 +4.28345 0.00358206 0.0680313 0 0.000243692 0.00358206 0.362917 0.00182252 +4.28589 0.00357688 0.0683025 0 0.00024431 0.00357688 0.364628 0.00182559 +4.28833 0.0035717 0.0685738 0 0.000244925 0.0035717 0.366341 0.00182866 +4.29077 0.00356653 0.0688451 0 0.000245538 0.00356653 0.368057 0.00183171 +4.29321 0.00356136 0.0691163 0 0.000246148 0.00356136 0.369776 0.00183474 +4.29565 0.00355619 0.0693876 0 0.000246755 0.00355619 0.371497 0.00183776 +4.2981 0.00355103 0.0696589 0 0.000247361 0.00355103 0.37322 0.00184076 +4.30054 0.00354587 0.0699301 0 0.000247963 0.00354587 0.374946 0.00184375 +4.30298 0.00354071 0.0702014 0 0.000248563 0.00354071 0.376674 0.00184672 +4.30542 0.00353556 0.0704727 0 0.00024916 0.00353556 0.378405 0.00184968 +4.30786 0.00353041 0.0707439 0 0.000249755 0.00353041 0.380139 0.00185262 +4.3103 0.00352527 0.0710152 0 0.000250348 0.00352527 0.381875 0.00185555 +4.31274 0.00352013 0.0712865 0 0.000250938 0.00352013 0.383613 0.00185846 +4.31519 0.00351499 0.0715577 0 0.000251525 0.00351499 0.385354 0.00186136 +4.31763 0.00350986 0.071829 0 0.00025211 0.00350986 0.387098 0.00186425 +4.32007 0.00350473 0.0721003 0 0.000252692 0.00350473 0.388844 0.00186712 +4.32251 0.0034996 0.0723715 0 0.000253272 0.0034996 0.390592 0.00186997 +4.32495 0.00349448 0.0726428 0 0.000253849 0.00349448 0.392344 0.00187281 +4.32739 0.00348936 0.0729141 0 0.000254424 0.00348936 0.394098 0.00187564 +4.32983 0.00348425 0.0731853 0 0.000254996 0.00348425 0.395854 0.00187845 +4.33228 0.00347914 0.0734566 0 0.000255566 0.00347914 0.397613 0.00188125 +4.33472 0.00347403 0.0737279 0 0.000256133 0.00347403 0.399374 0.00188403 +4.33716 0.00346893 0.0739991 0 0.000256698 0.00346893 0.401139 0.0018868 +4.3396 0.00346383 0.0742704 0 0.00025726 0.00346383 0.402905 0.00188955 +4.34204 0.00345874 0.0745417 0 0.00025782 0.00345874 0.404675 0.00189229 +4.34448 0.00345364 0.0748129 0 0.000258377 0.00345364 0.406447 0.00189502 +4.34692 0.00344856 0.0750842 0 0.000258932 0.00344856 0.408221 0.00189773 +4.34937 0.00344347 0.0753555 0 0.000259484 0.00344347 0.409998 0.00190043 +4.35181 0.00343839 0.0756267 0 0.000260034 0.00343839 0.411778 0.00190311 +4.35425 0.00343331 0.075898 0 0.000260582 0.00343331 0.413561 0.00190578 +4.35669 0.00342824 0.0761693 0 0.000261127 0.00342824 0.415346 0.00190844 +4.35913 0.00342317 0.0764405 0 0.000261669 0.00342317 0.417133 0.00191108 +4.36157 0.00341811 0.0767118 0 0.000262209 0.00341811 0.418924 0.00191371 +4.36401 0.00341305 0.0769831 0 0.000262747 0.00341305 0.420717 0.00191632 +4.36646 0.00340799 0.0772544 0 0.000263282 0.00340799 0.422512 0.00191892 +4.3689 0.00340293 0.0775256 0 0.000263815 0.00340293 0.424311 0.00192151 +4.37134 0.00339788 0.0777969 0 0.000264345 0.00339788 0.426112 0.00192408 +4.37378 0.00339284 0.0780682 0 0.000264872 0.00339284 0.427915 0.00192664 +4.37622 0.00338779 0.0783394 0 0.000265398 0.00338779 0.429722 0.00192919 +4.37866 0.00338275 0.0786107 0 0.000265921 0.00338275 0.431531 0.00193172 +4.3811 0.00337772 0.078882 0 0.000266441 0.00337772 0.433343 0.00193424 +4.38354 0.00337269 0.0791532 0 0.000266959 0.00337269 0.435157 0.00193675 +4.38599 0.00336766 0.0794245 0 0.000267475 0.00336766 0.436974 0.00193924 +4.38843 0.00336263 0.0796958 0 0.000267988 0.00336263 0.438794 0.00194172 +4.39087 0.00335761 0.079967 0 0.000268498 0.00335761 0.440617 0.00194419 +4.39331 0.0033526 0.0802383 0 0.000269007 0.0033526 0.442442 0.00194664 +4.39575 0.00334758 0.0805096 0 0.000269513 0.00334758 0.44427 0.00194908 +4.39819 0.00334258 0.0807808 0 0.000270016 0.00334258 0.446101 0.00195151 +4.40063 0.00333757 0.0810521 0 0.000270517 0.00333757 0.447935 0.00195392 +4.40308 0.00333257 0.0813234 0 0.000271016 0.00333257 0.449771 0.00195632 +4.40552 0.00332757 0.0815946 0 0.000271512 0.00332757 0.45161 0.00195871 +4.40796 0.00332258 0.0818659 0 0.000272006 0.00332258 0.453452 0.00196108 +4.4104 0.00331759 0.0821372 0 0.000272497 0.00331759 0.455296 0.00196344 +4.41284 0.0033126 0.0824084 0 0.000272986 0.0033126 0.457144 0.00196579 +4.41528 0.00330762 0.0826797 0 0.000273473 0.00330762 0.458994 0.00196813 +4.41772 0.00330264 0.082951 0 0.000273957 0.00330264 0.460847 0.00197045 +4.42017 0.00329766 0.0832222 0 0.000274439 0.00329766 0.462702 0.00197276 +4.42261 0.00329269 0.0834935 0 0.000274918 0.00329269 0.464561 0.00197506 +4.42505 0.00328772 0.0837648 0 0.000275395 0.00328772 0.466422 0.00197734 +4.42749 0.00328276 0.084036 0 0.00027587 0.00328276 0.468286 0.00197961 +4.42993 0.0032778 0.0843073 0 0.000276342 0.0032778 0.470153 0.00198187 +4.43237 0.00327284 0.0845786 0 0.000276812 0.00327284 0.472023 0.00198412 +4.43481 0.00326789 0.0848498 0 0.00027728 0.00326789 0.473896 0.00198635 +4.43726 0.00326294 0.0851211 0 0.000277745 0.00326294 0.475771 0.00198857 +4.4397 0.00325799 0.0853924 0 0.000278208 0.00325799 0.47765 0.00199078 +4.44214 0.00325305 0.0856636 0 0.000278668 0.00325305 0.479531 0.00199298 +4.44458 0.00324811 0.0859349 0 0.000279126 0.00324811 0.481415 0.00199516 +4.44702 0.00324318 0.0862062 0 0.000279582 0.00324318 0.483302 0.00199733 +4.44946 0.00323825 0.0864774 0 0.000280036 0.00323825 0.485192 0.00199949 +4.4519 0.00323332 0.0867487 0 0.000280487 0.00323332 0.487084 0.00200164 +4.45435 0.0032284 0.08702 0 0.000280935 0.0032284 0.48898 0.00200377 +4.45679 0.00322348 0.0872912 0 0.000281382 0.00322348 0.490878 0.00200589 +4.45923 0.00321857 0.0875625 0 0.000281826 0.00321857 0.492779 0.002008 +4.46167 0.00321365 0.0878338 0 0.000282267 0.00321365 0.494684 0.0020101 +4.46411 0.00320875 0.088105 0 0.000282707 0.00320875 0.496591 0.00201218 +4.46655 0.00320384 0.0883763 0 0.000283144 0.00320384 0.498501 0.00201426 +4.46899 0.00319894 0.0886476 0 0.000283578 0.00319894 0.500414 0.00201632 +4.47144 0.00319404 0.0889188 0 0.000284011 0.00319404 0.50233 0.00201837 +4.47388 0.00318915 0.0891901 0 0.000284441 0.00318915 0.504249 0.0020204 +4.47632 0.00318426 0.0894614 0 0.000284869 0.00318426 0.50617 0.00202243 +4.47876 0.00317938 0.0897326 0 0.000285294 0.00317938 0.508095 0.00202444 +4.4812 0.0031745 0.0900039 0 0.000285717 0.0031745 0.510023 0.00202644 +4.48364 0.00316962 0.0902752 0 0.000286138 0.00316962 0.511953 0.00202843 +4.48608 0.00316474 0.0905465 0 0.000286556 0.00316474 0.513887 0.00203041 +4.48853 0.00315987 0.0908177 0 0.000286972 0.00315987 0.515824 0.00203237 +4.49097 0.00315501 0.091089 0 0.000287386 0.00315501 0.517763 0.00203433 +4.49341 0.00315014 0.0913603 0 0.000287798 0.00315014 0.519706 0.00203627 +4.49585 0.00314528 0.0916315 0 0.000288207 0.00314528 0.521651 0.0020382 +4.49829 0.00314043 0.0919028 0 0.000288614 0.00314043 0.5236 0.00204012 +4.50073 0.00313558 0.0921741 0 0.000289019 0.00313558 0.525552 0.00204203 +4.50317 0.00313073 0.0924453 0 0.000289421 0.00313073 0.527506 0.00204392 +4.50562 0.00312588 0.0927166 0 0.000289821 0.00312588 0.529464 0.0020458 +4.50806 0.00312104 0.0929879 0 0.000290219 0.00312104 0.531425 0.00204768 +4.5105 0.00311621 0.0932591 0 0.000290615 0.00311621 0.533388 0.00204954 +4.51294 0.00311137 0.0935304 0 0.000291008 0.00311137 0.535355 0.00205138 +4.51538 0.00310655 0.0938017 0 0.000291399 0.00310655 0.537325 0.00205322 +4.51782 0.00310172 0.0940729 0 0.000291788 0.00310172 0.539298 0.00205505 +4.52026 0.0030969 0.0943442 0 0.000292174 0.0030969 0.541274 0.00205686 +4.52271 0.00309208 0.0946155 0 0.000292559 0.00309208 0.543253 0.00205866 +4.52515 0.00308727 0.0948867 0 0.000292941 0.00308727 0.545235 0.00206046 +4.52759 0.00308246 0.095158 0 0.00029332 0.00308246 0.54722 0.00206224 +4.53003 0.00307765 0.0954293 0 0.000293698 0.00307765 0.549209 0.00206401 +4.53247 0.00307285 0.0957005 0 0.000294073 0.00307285 0.5512 0.00206576 +4.53491 0.00306805 0.0959718 0 0.000294446 0.00306805 0.553195 0.00206751 +4.53735 0.00306325 0.0962431 0 0.000294817 0.00306325 0.555192 0.00206924 +4.53979 0.00305846 0.0965143 0 0.000295185 0.00305846 0.557193 0.00207097 +4.54224 0.00305367 0.0967856 0 0.000295552 0.00305367 0.559197 0.00207268 +4.54468 0.00304889 0.0970569 0 0.000295916 0.00304889 0.561204 0.00207438 +4.54712 0.00304411 0.0973281 0 0.000296277 0.00304411 0.563215 0.00207607 +4.54956 0.00303933 0.0975994 0 0.000296637 0.00303933 0.565228 0.00207775 +4.552 0.00303456 0.0978707 0 0.000296994 0.00303456 0.567244 0.00207942 +4.55444 0.00302979 0.0981419 0 0.000297349 0.00302979 0.569264 0.00208108 +4.55688 0.00302502 0.0984132 0 0.000297702 0.00302502 0.571287 0.00208272 +4.55933 0.00302026 0.0986845 0 0.000298053 0.00302026 0.573313 0.00208436 +4.56177 0.0030155 0.0989557 0 0.000298401 0.0030155 0.575343 0.00208598 +4.56421 0.00301075 0.099227 0 0.000298748 0.00301075 0.577375 0.0020876 +4.56665 0.003006 0.0994983 0 0.000299092 0.003006 0.579411 0.0020892 +4.56909 0.00300125 0.0997695 0 0.000299434 0.00300125 0.58145 0.00209079 +4.57153 0.00299651 0.100041 0 0.000299773 0.00299651 0.583492 0.00209237 +4.57397 0.00299177 0.100312 0 0.000300111 0.00299177 0.585537 0.00209394 +4.57642 0.00298704 0.100583 0 0.000300446 0.00298704 0.587586 0.0020955 +4.57886 0.0029823 0.100855 0 0.000300779 0.0029823 0.589638 0.00209705 +4.5813 0.00297758 0.101126 0 0.00030111 0.00297758 0.591693 0.00209859 +4.58374 0.00297285 0.101397 0 0.000301439 0.00297285 0.593751 0.00210011 +4.58618 0.00296813 0.101668 0 0.000301765 0.00296813 0.595813 0.00210163 +4.58862 0.00296342 0.10194 0 0.00030209 0.00296342 0.597878 0.00210314 +4.59106 0.0029587 0.102211 0 0.000302412 0.0029587 0.599946 0.00210463 +4.59351 0.00295399 0.102482 0 0.000302732 0.00295399 0.602018 0.00210612 +4.59595 0.00294929 0.102753 0 0.00030305 0.00294929 0.604093 0.00210759 +4.59839 0.00294459 0.103025 0 0.000303365 0.00294459 0.606171 0.00210905 +4.60083 0.00293989 0.103296 0 0.000303679 0.00293989 0.608252 0.00211051 +4.60327 0.00293519 0.103567 0 0.00030399 0.00293519 0.610337 0.00211195 +4.60571 0.0029305 0.103839 0 0.000304299 0.0029305 0.612425 0.00211338 +4.60815 0.00292582 0.10411 0 0.000304606 0.00292582 0.614517 0.0021148 +4.6106 0.00292113 0.104381 0 0.000304911 0.00292113 0.616611 0.00211621 +4.61304 0.00291646 0.104652 0 0.000305214 0.00291646 0.61871 0.00211761 +4.61548 0.00291178 0.104924 0 0.000305514 0.00291178 0.620811 0.002119 +4.61792 0.00290711 0.105195 0 0.000305813 0.00290711 0.622916 0.00212038 +4.62036 0.00290244 0.105466 0 0.000306109 0.00290244 0.625024 0.00212175 +4.6228 0.00289778 0.105737 0 0.000306403 0.00289778 0.627136 0.00212311 +4.62524 0.00289312 0.106009 0 0.000306695 0.00289312 0.629251 0.00212446 +4.62769 0.00288846 0.10628 0 0.000306985 0.00288846 0.63137 0.0021258 +4.63013 0.00288381 0.106551 0 0.000307273 0.00288381 0.633492 0.00212713 +4.63257 0.00287916 0.106822 0 0.000307559 0.00287916 0.635617 0.00212844 +4.63501 0.00287451 0.107094 0 0.000307842 0.00287451 0.637746 0.00212975 +4.63745 0.00286987 0.107365 0 0.000308124 0.00286987 0.639878 0.00213105 +4.63989 0.00286523 0.107636 0 0.000308403 0.00286523 0.642014 0.00213234 +4.64233 0.0028606 0.107908 0 0.00030868 0.0028606 0.644153 0.00213362 +4.64478 0.00285597 0.108179 0 0.000308955 0.00285597 0.646295 0.00213488 +4.64722 0.00285134 0.10845 0 0.000309228 0.00285134 0.648442 0.00213614 +4.64966 0.00284672 0.108721 0 0.000309499 0.00284672 0.650591 0.00213739 +4.6521 0.0028421 0.108993 0 0.000309768 0.0028421 0.652744 0.00213862 +4.65454 0.00283748 0.109264 0 0.000310035 0.00283748 0.654901 0.00213985 +4.65698 0.00283287 0.109535 0 0.000310299 0.00283287 0.657061 0.00214107 +4.65942 0.00282826 0.109806 0 0.000310562 0.00282826 0.659224 0.00214228 +4.66187 0.00282366 0.110078 0 0.000310822 0.00282366 0.661392 0.00214347 +4.66431 0.00281906 0.110349 0 0.00031108 0.00281906 0.663562 0.00214466 +4.66675 0.00281446 0.11062 0 0.000311337 0.00281446 0.665736 0.00214584 +4.66919 0.00280987 0.110892 0 0.000311591 0.00280987 0.667914 0.00214701 +4.67163 0.00280528 0.111163 0 0.000311843 0.00280528 0.670095 0.00214816 +4.67407 0.0028007 0.111434 0 0.000312093 0.0028007 0.67228 0.00214931 +4.67651 0.00279611 0.111705 0 0.000312341 0.00279611 0.674469 0.00215045 +4.67896 0.00279154 0.111977 0 0.000312587 0.00279154 0.676661 0.00215158 +4.6814 0.00278696 0.112248 0 0.000312831 0.00278696 0.678856 0.0021527 +4.68384 0.00278239 0.112519 0 0.000313072 0.00278239 0.681056 0.00215381 +4.68628 0.00277783 0.11279 0 0.000313312 0.00277783 0.683259 0.00215491 +4.68872 0.00277326 0.113062 0 0.00031355 0.00277326 0.685465 0.002156 +4.69116 0.0027687 0.113333 0 0.000313785 0.0027687 0.687675 0.00215708 +4.6936 0.00276415 0.113604 0 0.000314019 0.00276415 0.689889 0.00215815 +4.69604 0.0027596 0.113875 0 0.00031425 0.0027596 0.692106 0.00215921 +4.69849 0.00275505 0.114147 0 0.00031448 0.00275505 0.694327 0.00216026 +4.70093 0.0027505 0.114418 0 0.000314707 0.0027505 0.696552 0.0021613 +4.70337 0.00274596 0.114689 0 0.000314933 0.00274596 0.69878 0.00216233 +4.70581 0.00274143 0.114961 0 0.000315156 0.00274143 0.701013 0.00216335 +4.70825 0.00273689 0.115232 0 0.000315377 0.00273689 0.703248 0.00216437 +4.71069 0.00273237 0.115503 0 0.000315597 0.00273237 0.705488 0.00216537 +4.71313 0.00272784 0.115774 0 0.000315814 0.00272784 0.707731 0.00216636 +4.71558 0.00272332 0.116046 0 0.000316029 0.00272332 0.709978 0.00216735 +4.71802 0.0027188 0.116317 0 0.000316242 0.0027188 0.712229 0.00216832 +4.72046 0.00271429 0.116588 0 0.000316454 0.00271429 0.714483 0.00216929 +4.7229 0.00270978 0.116859 0 0.000316663 0.00270978 0.716741 0.00217024 +4.72534 0.00270527 0.117131 0 0.00031687 0.00270527 0.719003 0.00217119 +4.72778 0.00270077 0.117402 0 0.000317075 0.00270077 0.721269 0.00217213 +4.73022 0.00269627 0.117673 0 0.000317278 0.00269627 0.723538 0.00217305 +4.73267 0.00269177 0.117944 0 0.00031748 0.00269177 0.725811 0.00217397 +4.73511 0.00268728 0.118216 0 0.000317679 0.00268728 0.728089 0.00217488 +4.73755 0.00268279 0.118487 0 0.000317876 0.00268279 0.730369 0.00217578 +4.73999 0.00267831 0.118758 0 0.000318071 0.00267831 0.732654 0.00217667 +4.74243 0.00267383 0.11903 0 0.000318265 0.00267383 0.734943 0.00217756 +4.74487 0.00266935 0.119301 0 0.000318456 0.00266935 0.737235 0.00217843 +4.74731 0.00266488 0.119572 0 0.000318645 0.00266488 0.739531 0.00217929 +4.74976 0.00266041 0.119843 0 0.000318832 0.00266041 0.741831 0.00218014 +4.7522 0.00265594 0.120115 0 0.000319018 0.00265594 0.744135 0.00218099 +4.75464 0.00265148 0.120386 0 0.000319201 0.00265148 0.746443 0.00218182 +4.75708 0.00264703 0.120657 0 0.000319382 0.00264703 0.748754 0.00218265 +4.75952 0.00264257 0.120928 0 0.000319562 0.00264257 0.75107 0.00218347 +4.76196 0.00263812 0.1212 0 0.000319739 0.00263812 0.753389 0.00218428 +4.7644 0.00263367 0.121471 0 0.000319915 0.00263367 0.755713 0.00218508 +4.76685 0.00262923 0.121742 0 0.000320088 0.00262923 0.75804 0.00218587 +4.76929 0.00262479 0.122013 0 0.00032026 0.00262479 0.760371 0.00218665 +4.77173 0.00262036 0.122285 0 0.00032043 0.00262036 0.762707 0.00218742 +4.77417 0.00261593 0.122556 0 0.000320597 0.00261593 0.765046 0.00218818 +4.77661 0.0026115 0.122827 0 0.000320763 0.0026115 0.767389 0.00218894 +4.77905 0.00260707 0.123099 0 0.000320927 0.00260707 0.769736 0.00218968 +4.78149 0.00260265 0.12337 0 0.000321089 0.00260265 0.772087 0.00219042 +4.78394 0.00259824 0.123641 0 0.000321249 0.00259824 0.774442 0.00219115 +4.78638 0.00259382 0.123912 0 0.000321407 0.00259382 0.776801 0.00219186 +4.78882 0.00258941 0.124184 0 0.000321563 0.00258941 0.779164 0.00219257 +4.79126 0.00258501 0.124455 0 0.000321717 0.00258501 0.781531 0.00219327 +4.7937 0.00258061 0.124726 0 0.000321869 0.00258061 0.783902 0.00219397 +4.79614 0.00257621 0.124997 0 0.00032202 0.00257621 0.786278 0.00219465 +4.79858 0.00257182 0.125269 0 0.000322168 0.00257182 0.788657 0.00219532 +4.80103 0.00256743 0.12554 0 0.000322314 0.00256743 0.79104 0.00219599 +4.80347 0.00256304 0.125811 0 0.000322459 0.00256304 0.793427 0.00219665 +4.80591 0.00255866 0.126082 0 0.000322602 0.00255866 0.795819 0.00219729 +4.80835 0.00255428 0.126354 0 0.000322742 0.00255428 0.798214 0.00219793 +4.81079 0.0025499 0.126625 0 0.000322881 0.0025499 0.800614 0.00219856 +4.81323 0.00254553 0.126896 0 0.000323018 0.00254553 0.803018 0.00219918 +4.81567 0.00254116 0.127168 0 0.000323153 0.00254116 0.805426 0.0021998 +4.81812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01099 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01343 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01587 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01831 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02075 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02319 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02563 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02808 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03052 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03296 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03784 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04028 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04272 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04517 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04761 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05005 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05493 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05737 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05981 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06226 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06714 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07202 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07446 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07935 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08179 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08423 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08911 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09155 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09399 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09644 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09888 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10132 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10864 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11108 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11597 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11841 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12329 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12573 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12817 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13306 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13794 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14038 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14282 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14526 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15015 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15259 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15503 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15747 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15991 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16235 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16724 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16968 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17212 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17456 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17944 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18433 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18677 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18921 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19165 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19409 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19653 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20142 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20386 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20874 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21118 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21362 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21851 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22095 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22339 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22583 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22827 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23071 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23315 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23804 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24048 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24536 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25024 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25269 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25513 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25757 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26245 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26489 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26733 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26978 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27222 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27466 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27954 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28198 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28442 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28687 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28931 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29175 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29419 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29663 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29907 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30151 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30884 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31128 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31372 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31616 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32349 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32593 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32837 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33081 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33325 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33569 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33813 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34058 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34302 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34546 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35034 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35278 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35522 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35767 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36011 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36255 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36743 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36987 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37476 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37964 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38208 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38452 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38696 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39185 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39429 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39673 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39917 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40161 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40405 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40649 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41138 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41382 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41626 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42114 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42358 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42847 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43091 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43335 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43579 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44067 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44556 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45044 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45288 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45532 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45776 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46265 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46509 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46753 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46997 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47241 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47485 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47974 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48218 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48462 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48706 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49194 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49683 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49927 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50171 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50659 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50903 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51147 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51392 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51636 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52368 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52612 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52856 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53101 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53345 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53589 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54077 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54321 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54565 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55054 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55298 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55786 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56274 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56519 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56763 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57007 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57495 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57739 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57983 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58228 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58472 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58716 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59204 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59692 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59937 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60181 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60425 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60669 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60913 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61157 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61401 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62134 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62378 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62622 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62866 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63599 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63843 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64087 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64331 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64575 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64819 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65308 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65552 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65796 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66284 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66528 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67017 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67261 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67505 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67749 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67993 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68237 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68726 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69214 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69458 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69702 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69946 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70435 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70679 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70923 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71167 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71411 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71655 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71899 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72388 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72632 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72876 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73364 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73608 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74097 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74341 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74829 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75317 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75562 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75806 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76294 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76538 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76782 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77026 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77515 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77759 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78003 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78247 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78491 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78735 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79224 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79468 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79712 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79956 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80444 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80688 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80933 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81421 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81909 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82153 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82397 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82642 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82886 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83618 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83862 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84106 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84351 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84595 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84839 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85327 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85571 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85815 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86304 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86548 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87036 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87524 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88013 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88257 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88501 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88745 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88989 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89233 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89722 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89966 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90454 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90698 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90942 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91431 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91675 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91919 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92163 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92407 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92651 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93384 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93628 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93872 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94116 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94849 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95093 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95337 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95581 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95825 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96069 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96558 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97046 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97534 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97778 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98267 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98511 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98755 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99243 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99487 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99976 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00464 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00952 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01196 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01685 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01929 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02173 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02661 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02905 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03149 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03394 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03638 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03882 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04614 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04858 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05103 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05347 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05591 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06079 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06323 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06567 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26099 0.001 0 0 0 0.001 0 0 +6.26343 0.001 0 0 0 0.001 0 0 +6.26587 0.001 0 0 0 0.001 0 0 +6.26831 0.001 0 0 0 0.001 0 0 +6.27075 0.001 0 0 0 0.001 0 0 +6.27319 0.001 0 0 0 0.001 0 0 +6.27563 0.001 0 0 0 0.001 0 0 +6.27808 0.001 0 0 0 0.001 0 0 +6.28052 0.001 0 0 0 0.001 0 0 +6.28296 0.001 0 0 0 0.001 0 0 +6.2854 0.001 0 0 0 0.001 0 0 +6.28784 0.001 0 0 0 0.001 0 0 +6.29028 0.001 0 0 0 0.001 0 0 +6.29272 0.001 0 0 0 0.001 0 0 +6.29517 0.001 0 0 0 0.001 0 0 +6.29761 0.001 0 0 0 0.001 0 0 +6.30005 0.001 0 0 0 0.001 0 0 +6.30249 0.001 0 0 0 0.001 0 0 +6.30493 0.001 0 0 0 0.001 0 0 +6.30737 0.001 0 0 0 0.001 0 0 +6.30981 0.001 0 0 0 0.001 0 0 +6.31226 0.001 0 0 0 0.001 0 0 +6.3147 0.001 0 0 0 0.001 0 0 +6.31714 0.001 0 0 0 0.001 0 0 +6.31958 0.001 0 0 0 0.001 0 0 +6.32202 0.001 0 0 0 0.001 0 0 +6.32446 0.001 0 0 0 0.001 0 0 +6.3269 0.001 0 0 0 0.001 0 0 +6.32935 0.001 0 0 0 0.001 0 0 +6.33179 0.001 0 0 0 0.001 0 0 +6.33423 0.001 0 0 0 0.001 0 0 +6.33667 0.001 0 0 0 0.001 0 0 +6.33911 0.001 0 0 0 0.001 0 0 +6.34155 0.001 0 0 0 0.001 0 0 +6.34399 0.001 0 0 0 0.001 0 0 +6.34644 0.001 0 0 0 0.001 0 0 +6.34888 0.001 0 0 0 0.001 0 0 +6.35132 0.001 0 0 0 0.001 0 0 +6.35376 0.001 0 0 0 0.001 0 0 +6.3562 0.001 0 0 0 0.001 0 0 +6.35864 0.001 0 0 0 0.001 0 0 +6.36108 0.001 0 0 0 0.001 0 0 +6.36353 0.001 0 0 0 0.001 0 0 +6.36597 0.001 0 0 0 0.001 0 0 +6.36841 0.001 0 0 0 0.001 0 0 +6.37085 0.001 0 0 0 0.001 0 0 +6.37329 0.001 0 0 0 0.001 0 0 +6.37573 0.001 0 0 0 0.001 0 0 +6.37817 0.001 0 0 0 0.001 0 0 +6.38062 0.001 0 0 0 0.001 0 0 +6.38306 0.001 0 0 0 0.001 0 0 +6.3855 0.001 0 0 0 0.001 0 0 +6.38794 0.001 0 0 0 0.001 0 0 +6.39038 0.001 0 0 0 0.001 0 0 +6.39282 0.001 0 0 0 0.001 0 0 +6.39526 0.001 0 0 0 0.001 0 0 +6.39771 0.001 0 0 0 0.001 0 0 +6.40015 0.001 0 0 0 0.001 0 0 +6.40259 0.001 0 0 0 0.001 0 0 +6.40503 0.001 0 0 0 0.001 0 0 +6.40747 0.001 0 0 0 0.001 0 0 +6.40991 0.001 0 0 0 0.001 0 0 +6.41235 0.001 0 0 0 0.001 0 0 +6.41479 0.001 0 0 0 0.001 0 0 +6.41724 0.001 0 0 0 0.001 0 0 +6.41968 0.001 0 0 0 0.001 0 0 +6.42212 0.001 0 0 0 0.001 0 0 +6.42456 0.001 0 0 0 0.001 0 0 +6.427 0.001 0 0 0 0.001 0 0 +6.42944 0.001 0 0 0 0.001 0 0 +6.43188 0.001 0 0 0 0.001 0 0 +6.43433 0.001 0 0 0 0.001 0 0 +6.43677 0.001 0 0 0 0.001 0 0 +6.43921 0.001 0 0 0 0.001 0 0 +6.44165 0.001 0 0 0 0.001 0 0 +6.44409 0.001 0 0 0 0.001 0 0 +6.44653 0.001 0 0 0 0.001 0 0 +6.44897 0.001 0 0 0 0.001 0 0 +6.45142 0.001 0 0 0 0.001 0 0 +6.45386 0.001 0 0 0 0.001 0 0 +6.4563 0.001 0 0 0 0.001 0 0 +6.45874 0.001 0 0 0 0.001 0 0 +6.46118 0.001 0 0 0 0.001 0 0 +6.46362 0.001 0 0 0 0.001 0 0 +6.46606 0.001 0 0 0 0.001 0 0 +6.46851 0.001 0 0 0 0.001 0 0 +6.47095 0.001 0 0 0 0.001 0 0 +6.47339 0.001 0 0 0 0.001 0 0 +6.47583 0.001 0 0 0 0.001 0 0 +6.47827 0.001 0 0 0 0.001 0 0 +6.48071 0.001 0 0 0 0.001 0 0 +6.48315 0.001 0 0 0 0.001 0 0 +6.4856 0.001 0 0 0 0.001 0 0 +6.48804 0.001 0 0 0 0.001 0 0 +6.49048 0.001 0 0 0 0.001 0 0 +6.49292 0.001 0 0 0 0.001 0 0 +6.49536 0.001 0 0 0 0.001 0 0 +6.4978 0.001 0 0 0 0.001 0 0 +6.50024 0.001 0 0 0 0.001 0 0 +6.50269 0.001 0 0 0 0.001 0 0 +6.50513 0.001 0 0 0 0.001 0 0 +6.50757 0.001 0 0 0 0.001 0 0 +6.51001 0.001 0 0 0 0.001 0 0 +6.51245 0.001 0 0 0 0.001 0 0 +6.51489 0.001 0 0 0 0.001 0 0 +6.51733 0.001 0 0 0 0.001 0 0 +6.51978 0.001 0 0 0 0.001 0 0 +6.52222 0.001 0 0 0 0.001 0 0 +6.52466 0.001 0 0 0 0.001 0 0 +6.5271 0.001 0 0 0 0.001 0 0 +6.52954 0.001 0 0 0 0.001 0 0 +6.53198 0.001 0 0 0 0.001 0 0 +6.53442 0.001 0 0 0 0.001 0 0 +6.53687 0.001 0 0 0 0.001 0 0 +6.53931 0.001 0 0 0 0.001 0 0 +6.54175 0.001 0 0 0 0.001 0 0 +6.54419 0.001 0 0 0 0.001 0 0 +6.54663 0.001 0 0 0 0.001 0 0 +6.54907 0.001 0 0 0 0.001 0 0 +6.55151 0.001 0 0 0 0.001 0 0 +6.55396 0.001 0 0 0 0.001 0 0 +6.5564 0.001 0 0 0 0.001 0 0 +6.55884 0.001 0 0 0 0.001 0 0 +6.56128 0.001 0 0 0 0.001 0 0 +6.56372 0.001 0 0 0 0.001 0 0 +6.56616 0.001 0 0 0 0.001 0 0 +6.5686 0.001 0 0 0 0.001 0 0 +6.57104 0.001 0 0 0 0.001 0 0 +6.57349 0.001 0 0 0 0.001 0 0 +6.57593 0.001 0 0 0 0.001 0 0 +6.57837 0.001 0 0 0 0.001 0 0 +6.58081 0.001 0 0 0 0.001 0 0 +6.58325 0.001 0 0 0 0.001 0 0 +6.58569 0.001 0 0 0 0.001 0 0 +6.58813 0.001 0 0 0 0.001 0 0 +6.59058 0.001 0 0 0 0.001 0 0 +6.59302 0.001 0 0 0 0.001 0 0 +6.59546 0.001 0 0 0 0.001 0 0 +6.5979 0.001 0 0 0 0.001 0 0 +6.60034 0.001 0 0 0 0.001 0 0 +6.60278 0.001 0 0 0 0.001 0 0 +6.60522 0.001 0 0 0 0.001 0 0 +6.60767 0.001 0 0 0 0.001 0 0 +6.61011 0.001 0 0 0 0.001 0 0 +6.61255 0.001 0 0 0 0.001 0 0 +6.61499 0.001 0 0 0 0.001 0 0 +6.61743 0.001 0 0 0 0.001 0 0 +6.61987 0.001 0 0 0 0.001 0 0 +6.62231 0.001 0 0 0 0.001 0 0 +6.62476 0.001 0 0 0 0.001 0 0 +6.6272 0.001 0 0 0 0.001 0 0 +6.62964 0.001 0 0 0 0.001 0 0 +6.63208 0.001 0 0 0 0.001 0 0 +6.63452 0.001 0 0 0 0.001 0 0 +6.63696 0.001 0 0 0 0.001 0 0 +6.6394 0.001 0 0 0 0.001 0 0 +6.64185 0.001 0 0 0 0.001 0 0 +6.64429 0.001 0 0 0 0.001 0 0 +6.64673 0.001 0 0 0 0.001 0 0 +6.64917 0.001 0 0 0 0.001 0 0 +6.65161 0.001 0 0 0 0.001 0 0 +6.65405 0.001 0 0 0 0.001 0 0 +6.65649 0.001 0 0 0 0.001 0 0 +6.65894 0.001 0 0 0 0.001 0 0 +6.66138 0.001 0 0 0 0.001 0 0 +6.66382 0.001 0 0 0 0.001 0 0 +6.66626 0.001 0 0 0 0.001 0 0 +6.6687 0.001 0 0 0 0.001 0 0 +6.67114 0.001 0 0 0 0.001 0 0 +6.67358 0.001 0 0 0 0.001 0 0 +6.67603 0.001 0 0 0 0.001 0 0 +6.67847 0.001 0 0 0 0.001 0 0 +6.68091 0.001 0 0 0 0.001 0 0 +6.68335 0.001 0 0 0 0.001 0 0 +6.68579 0.001 0 0 0 0.001 0 0 +6.68823 0.001 0 0 0 0.001 0 0 +6.69067 0.001 0 0 0 0.001 0 0 +6.69312 0.001 0 0 0 0.001 0 0 +6.69556 0.001 0 0 0 0.001 0 0 +6.698 0.001 0 0 0 0.001 0 0 +6.70044 0.001 0 0 0 0.001 0 0 +6.70288 0.001 0 0 0 0.001 0 0 +6.70532 0.001 0 0 0 0.001 0 0 +6.70776 0.001 0 0 0 0.001 0 0 +6.71021 0.001 0 0 0 0.001 0 0 +6.71265 0.001 0 0 0 0.001 0 0 +6.71509 0.001 0 0 0 0.001 0 0 +6.71753 0.001 0 0 0 0.001 0 0 +6.71997 0.001 0 0 0 0.001 0 0 +6.72241 0.001 0 0 0 0.001 0 0 +6.72485 0.001 0 0 0 0.001 0 0 +6.72729 0.001 0 0 0 0.001 0 0 +6.72974 0.001 0 0 0 0.001 0 0 +6.73218 0.001 0 0 0 0.001 0 0 +6.73462 0.001 0 0 0 0.001 0 0 +6.73706 0.001 0 0 0 0.001 0 0 +6.7395 0.001 0 0 0 0.001 0 0 +6.74194 0.001 0 0 0 0.001 0 0 +6.74438 0.001 0 0 0 0.001 0 0 +6.74683 0.001 0 0 0 0.001 0 0 +6.74927 0.001 0 0 0 0.001 0 0 +6.75171 0.001 0 0 0 0.001 0 0 +6.75415 0.001 0 0 0 0.001 0 0 +6.75659 0.001 0 0 0 0.001 0 0 +6.75903 0.001 0 0 0 0.001 0 0 +6.76147 0.001 0 0 0 0.001 0 0 +6.76392 0.001 0 0 0 0.001 0 0 +6.76636 0.001 0 0 0 0.001 0 0 +6.7688 0.001 0 0 0 0.001 0 0 +6.77124 0.001 0 0 0 0.001 0 0 +6.77368 0.001 0 0 0 0.001 0 0 +6.77612 0.001 0 0 0 0.001 0 0 +6.77856 0.001 0 0 0 0.001 0 0 +6.78101 0.001 0 0 0 0.001 0 0 +6.78345 0.001 0 0 0 0.001 0 0 +6.78589 0.001 0 0 0 0.001 0 0 +6.78833 0.001 0 0 0 0.001 0 0 +6.79077 0.001 0 0 0 0.001 0 0 +6.79321 0.001 0 0 0 0.001 0 0 +6.79565 0.001 0 0 0 0.001 0 0 +6.7981 0.001 0 0 0 0.001 0 0 +6.80054 0.001 0 0 0 0.001 0 0 +6.80298 0.001 0 0 0 0.001 0 0 +6.80542 0.001 0 0 0 0.001 0 0 +6.80786 0.001 0 0 0 0.001 0 0 +6.8103 0.001 0 0 0 0.001 0 0 +6.81274 0.001 0 0 0 0.001 0 0 +6.81519 0.001 0 0 0 0.001 0 0 +6.81763 0.001 0 0 0 0.001 0 0 +6.82007 0.001 0 0 0 0.001 0 0 +6.82251 0.001 0 0 0 0.001 0 0 +6.82495 0.001 0 0 0 0.001 0 0 +6.82739 0.001 0 0 0 0.001 0 0 +6.82983 0.001 0 0 0 0.001 0 0 +6.83228 0.001 0 0 0 0.001 0 0 +6.83472 0.001 0 0 0 0.001 0 0 +6.83716 0.001 0 0 0 0.001 0 0 +6.8396 0.001 0 0 0 0.001 0 0 +6.84204 0.001 0 0 0 0.001 0 0 +6.84448 0.001 0 0 0 0.001 0 0 +6.84692 0.001 0 0 0 0.001 0 0 +6.84937 0.001 0 0 0 0.001 0 0 +6.85181 0.001 0 0 0 0.001 0 0 +6.85425 0.001 0 0 0 0.001 0 0 +6.85669 0.001 0 0 0 0.001 0 0 +6.85913 0.001 0 0 0 0.001 0 0 +6.86157 0.001 0 0 0 0.001 0 0 +6.86401 0.001 0 0 0 0.001 0 0 +6.86646 0.001 0 0 0 0.001 0 0 +6.8689 0.001 0 0 0 0.001 0 0 +6.87134 0.001 0 0 0 0.001 0 0 +6.87378 0.001 0 0 0 0.001 0 0 +6.87622 0.001 0 0 0 0.001 0 0 +6.87866 0.001 0 0 0 0.001 0 0 +6.8811 0.001 0 0 0 0.001 0 0 +6.88354 0.001 0 0 0 0.001 0 0 +6.88599 0.001 0 0 0 0.001 0 0 +6.88843 0.001 0 0 0 0.001 0 0 +6.89087 0.001 0 0 0 0.001 0 0 +6.89331 0.001 0 0 0 0.001 0 0 +6.89575 0.001 0 0 0 0.001 0 0 +6.89819 0.001 0 0 0 0.001 0 0 +6.90063 0.001 0 0 0 0.001 0 0 +6.90308 0.001 0 0 0 0.001 0 0 +6.90552 0.001 0 0 0 0.001 0 0 +6.90796 0.001 0 0 0 0.001 0 0 +6.9104 0.001 0 0 0 0.001 0 0 +6.91284 0.001 0 0 0 0.001 0 0 +6.91528 0.001 0 0 0 0.001 0 0 +6.91772 0.001 0 0 0 0.001 0 0 +6.92017 0.001 0 0 0 0.001 0 0 +6.92261 0.001 0 0 0 0.001 0 0 +6.92505 0.001 0 0 0 0.001 0 0 +6.92749 0.001 0 0 0 0.001 0 0 +6.92993 0.001 0 0 0 0.001 0 0 +6.93237 0.001 0 0 0 0.001 0 0 +6.93481 0.001 0 0 0 0.001 0 0 +6.93726 0.001 0 0 0 0.001 0 0 +6.9397 0.001 0 0 0 0.001 0 0 +6.94214 0.001 0 0 0 0.001 0 0 +6.94458 0.001 0 0 0 0.001 0 0 +6.94702 0.001 0 0 0 0.001 0 0 +6.94946 0.001 0 0 0 0.001 0 0 +6.9519 0.001 0 0 0 0.001 0 0 +6.95435 0.001 0 0 0 0.001 0 0 +6.95679 0.001 0 0 0 0.001 0 0 +6.95923 0.001 0 0 0 0.001 0 0 +6.96167 0.001 0 0 0 0.001 0 0 +6.96411 0.001 0 0 0 0.001 0 0 +6.96655 0.001 0 0 0 0.001 0 0 +6.96899 0.001 0 0 0 0.001 0 0 +6.97144 0.001 0 0 0 0.001 0 0 +6.97388 0.001 0 0 0 0.001 0 0 +6.97632 0.001 0 0 0 0.001 0 0 +6.97876 0.001 0 0 0 0.001 0 0 +6.9812 0.001 0 0 0 0.001 0 0 +6.98364 0.001 0 0 0 0.001 0 0 +6.98608 0.001 0 0 0 0.001 0 0 +6.98853 0.001 0 0 0 0.001 0 0 +6.99097 0.001 0 0 0 0.001 0 0 +6.99341 0.001 0 0 0 0.001 0 0 +6.99585 0.001 0 0 0 0.001 0 0 +6.99829 0.001 0 0 0 0.001 0 0 +7.00073 0.001 0 0 0 0.001 0 0 +7.00317 0.001 0 0 0 0.001 0 0 +7.00562 0.001 0 0 0 0.001 0 0 +7.00806 0.001 0 0 0 0.001 0 0 +7.0105 0.001 0 0 0 0.001 0 0 +7.01294 0.001 0 0 0 0.001 0 0 +7.01538 0.001 0 0 0 0.001 0 0 +7.01782 0.001 0 0 0 0.001 0 0 +7.02026 0.001 0 0 0 0.001 0 0 +7.02271 0.001 0 0 0 0.001 0 0 +7.02515 0.001 0 0 0 0.001 0 0 +7.02759 0.001 0 0 0 0.001 0 0 +7.03003 0.001 0 0 0 0.001 0 0 +7.03247 0.001 0 0 0 0.001 0 0 +7.03491 0.001 0 0 0 0.001 0 0 +7.03735 0.001 0 0 0 0.001 0 0 +7.03979 0.001 0 0 0 0.001 0 0 +7.04224 0.001 0 0 0 0.001 0 0 +7.04468 0.001 0 0 0 0.001 0 0 +7.04712 0.001 0 0 0 0.001 0 0 +7.04956 0.001 0 0 0 0.001 0 0 +7.052 0.001 0 0 0 0.001 0 0 +7.05444 0.001 0 0 0 0.001 0 0 +7.05688 0.001 0 0 0 0.001 0 0 +7.05933 0.001 0 0 0 0.001 0 0 +7.06177 0.001 0 0 0 0.001 0 0 +7.06421 0.001 0 0 0 0.001 0 0 +7.06665 0.001 0 0 0 0.001 0 0 +7.06909 0.001 0 0 0 0.001 0 0 +7.07153 0.001 0 0 0 0.001 0 0 +7.07397 0.001 0 0 0 0.001 0 0 +7.07642 0.001 0 0 0 0.001 0 0 +7.07886 0.001 0 0 0 0.001 0 0 +7.0813 0.001 0 0 0 0.001 0 0 +7.08374 0.001 0 0 0 0.001 0 0 +7.08618 0.001 0 0 0 0.001 0 0 +7.08862 0.001 0 0 0 0.001 0 0 +7.09106 0.001 0 0 0 0.001 0 0 +7.09351 0.001 0 0 0 0.001 0 0 +7.09595 0.001 0 0 0 0.001 0 0 +7.09839 0.001 0 0 0 0.001 0 0 +7.10083 0.001 0 0 0 0.001 0 0 +7.10327 0.001 0 0 0 0.001 0 0 +7.10571 0.001 0 0 0 0.001 0 0 +7.10815 0.001 0 0 0 0.001 0 0 +7.1106 0.001 0 0 0 0.001 0 0 +7.11304 0.001 0 0 0 0.001 0 0 +7.11548 0.001 0 0 0 0.001 0 0 +7.11792 0.001 0 0 0 0.001 0 0 +7.12036 0.001 0 0 0 0.001 0 0 +7.1228 0.001 0 0 0 0.001 0 0 +7.12524 0.001 0 0 0 0.001 0 0 +7.12769 0.001 0 0 0 0.001 0 0 +7.13013 0.001 0 0 0 0.001 0 0 +7.13257 0.001 0 0 0 0.001 0 0 +7.13501 0.001 0 0 0 0.001 0 0 +7.13745 0.001 0 0 0 0.001 0 0 +7.13989 0.001 0 0 0 0.001 0 0 +7.14233 0.001 0 0 0 0.001 0 0 +7.14478 0.001 0 0 0 0.001 0 0 +7.14722 0.001 0 0 0 0.001 0 0 +7.14966 0.001 0 0 0 0.001 0 0 +7.1521 0.001 0 0 0 0.001 0 0 +7.15454 0.001 0 0 0 0.001 0 0 +7.15698 0.001 0 0 0 0.001 0 0 +7.15942 0.001 0 0 0 0.001 0 0 +7.16187 0.001 0 0 0 0.001 0 0 +7.16431 0.001 0 0 0 0.001 0 0 +7.16675 0.001 0 0 0 0.001 0 0 +7.16919 0.001 0 0 0 0.001 0 0 +7.17163 0.001 0 0 0 0.001 0 0 +7.17407 0.001 0 0 0 0.001 0 0 +7.17651 0.001 0 0 0 0.001 0 0 +7.17896 0.001 0 0 0 0.001 0 0 +7.1814 0.001 0 0 0 0.001 0 0 +7.18384 0.001 0 0 0 0.001 0 0 +7.18628 0.001 0 0 0 0.001 0 0 +7.18872 0.001 0 0 0 0.001 0 0 +7.19116 0.001 0 0 0 0.001 0 0 +7.1936 0.001 0 0 0 0.001 0 0 +7.19604 0.001 0 0 0 0.001 0 0 +7.19849 0.001 0 0 0 0.001 0 0 +7.20093 0.001 0 0 0 0.001 0 0 +7.20337 0.001 0 0 0 0.001 0 0 +7.20581 0.001 0 0 0 0.001 0 0 +7.20825 0.001 0 0 0 0.001 0 0 +7.21069 0.001 0 0 0 0.001 0 0 +7.21313 0.001 0 0 0 0.001 0 0 +7.21558 0.001 0 0 0 0.001 0 0 +7.21802 0.001 0 0 0 0.001 0 0 +7.22046 0.001 0 0 0 0.001 0 0 +7.2229 0.001 0 0 0 0.001 0 0 +7.22534 0.001 0 0 0 0.001 0 0 +7.22778 0.001 0 0 0 0.001 0 0 +7.23022 0.001 0 0 0 0.001 0 0 +7.23267 0.001 0 0 0 0.001 0 0 +7.23511 0.001 0 0 0 0.001 0 0 +7.23755 0.001 0 0 0 0.001 0 0 +7.23999 0.001 0 0 0 0.001 0 0 +7.24243 0.001 0 0 0 0.001 0 0 +7.24487 0.001 0 0 0 0.001 0 0 +7.24731 0.001 0 0 0 0.001 0 0 +7.24976 0.001 0 0 0 0.001 0 0 +7.2522 0.001 0 0 0 0.001 0 0 +7.25464 0.001 0 0 0 0.001 0 0 +7.25708 0.001 0 0 0 0.001 0 0 +7.25952 0.001 0 0 0 0.001 0 0 +7.26196 0.001 0 0 0 0.001 0 0 +7.2644 0.001 0 0 0 0.001 0 0 +7.26685 0.001 0 0 0 0.001 0 0 +7.26929 0.001 0 0 0 0.001 0 0 +7.27173 0.001 0 0 0 0.001 0 0 +7.27417 0.001 0 0 0 0.001 0 0 +7.27661 0.001 0 0 0 0.001 0 0 +7.27905 0.001 0 0 0 0.001 0 0 +7.28149 0.001 0 0 0 0.001 0 0 +7.28394 0.001 0 0 0 0.001 0 0 +7.28638 0.001 0 0 0 0.001 0 0 +7.28882 0.001 0 0 0 0.001 0 0 +7.29126 0.001 0 0 0 0.001 0 0 +7.2937 0.001 0 0 0 0.001 0 0 +7.29614 0.001 0 0 0 0.001 0 0 +7.29858 0.001 0 0 0 0.001 0 0 +7.30103 0.001 0 0 0 0.001 0 0 +7.30347 0.001 0 0 0 0.001 0 0 +7.30591 0.001 0 0 0 0.001 0 0 +7.30835 0.001 0 0 0 0.001 0 0 +7.31079 0.001 0 0 0 0.001 0 0 +7.31323 0.001 0 0 0 0.001 0 0 +7.31567 0.001 0 0 0 0.001 0 0 +7.31812 0.001 0 0 0 0.001 0 0 +7.32056 0.001 0 0 0 0.001 0 0 +7.323 0.001 0 0 0 0.001 0 0 +7.32544 0.001 0 0 0 0.001 0 0 +7.32788 0.001 0 0 0 0.001 0 0 +7.33032 0.001 0 0 0 0.001 0 0 +7.33276 0.001 0 0 0 0.001 0 0 +7.33521 0.001 0 0 0 0.001 0 0 +7.33765 0.001 0 0 0 0.001 0 0 +7.34009 0.001 0 0 0 0.001 0 0 +7.34253 0.001 0 0 0 0.001 0 0 +7.34497 0.001 0 0 0 0.001 0 0 +7.34741 0.001 0 0 0 0.001 0 0 +7.34985 0.001 0 0 0 0.001 0 0 +7.35229 0.001 0 0 0 0.001 0 0 +7.35474 0.001 0 0 0 0.001 0 0 +7.35718 0.001 0 0 0 0.001 0 0 +7.35962 0.001 0 0 0 0.001 0 0 +7.36206 0.001 0 0 0 0.001 0 0 +7.3645 0.001 0 0 0 0.001 0 0 +7.36694 0.001 0 0 0 0.001 0 0 +7.36938 0.001 0 0 0 0.001 0 0 +7.37183 0.001 0 0 0 0.001 0 0 +7.37427 0.001 0 0 0 0.001 0 0 +7.37671 0.001 0 0 0 0.001 0 0 +7.37915 0.001 0 0 0 0.001 0 0 +7.38159 0.001 0 0 0 0.001 0 0 +7.38403 0.001 0 0 0 0.001 0 0 +7.38647 0.001 0 0 0 0.001 0 0 +7.38892 0.001 0 0 0 0.001 0 0 +7.39136 0.001 0 0 0 0.001 0 0 +7.3938 0.001 0 0 0 0.001 0 0 +7.39624 0.001 0 0 0 0.001 0 0 +7.39868 0.001 0 0 0 0.001 0 0 +7.40112 0.001 0 0 0 0.001 0 0 +7.40356 0.001 0 0 0 0.001 0 0 +7.40601 0.001 0 0 0 0.001 0 0 +7.40845 0.001 0 0 0 0.001 0 0 +7.41089 0.001 0 0 0 0.001 0 0 +7.41333 0.001 0 0 0 0.001 0 0 +7.41577 0.001 0 0 0 0.001 0 0 +7.41821 0.001 0 0 0 0.001 0 0 +7.42065 0.001 0 0 0 0.001 0 0 +7.4231 0.001 0 0 0 0.001 0 0 +7.42554 0.001 0 0 0 0.001 0 0 +7.42798 0.001 0 0 0 0.001 0 0 +7.43042 0.001 0 0 0 0.001 0 0 +7.43286 0.001 0 0 0 0.001 0 0 +7.4353 0.001 0 0 0 0.001 0 0 +7.43774 0.001 0 0 0 0.001 0 0 +7.44019 0.001 0 0 0 0.001 0 0 +7.44263 0.001 0 0 0 0.001 0 0 +7.44507 0.001 0 0 0 0.001 0 0 +7.44751 0.001 0 0 0 0.001 0 0 +7.44995 0.001 0 0 0 0.001 0 0 +7.45239 0.001 0 0 0 0.001 0 0 +7.45483 0.001 0 0 0 0.001 0 0 +7.45728 0.001 0 0 0 0.001 0 0 +7.45972 0.001 0 0 0 0.001 0 0 +7.46216 0.001 0 0 0 0.001 0 0 +7.4646 0.001 0 0 0 0.001 0 0 +7.46704 0.001 0 0 0 0.001 0 0 +7.46948 0.001 0 0 0 0.001 0 0 +7.47192 0.001 0 0 0 0.001 0 0 +7.47437 0.001 0 0 0 0.001 0 0 +7.47681 0.001 0 0 0 0.001 0 0 +7.47925 0.001 0 0 0 0.001 0 0 +7.48169 0.001 0 0 0 0.001 0 0 +7.48413 0.001 0 0 0 0.001 0 0 +7.48657 0.001 0 0 0 0.001 0 0 +7.48901 0.001 0 0 0 0.001 0 0 +7.49146 0.001 0 0 0 0.001 0 0 +7.4939 0.001 0 0 0 0.001 0 0 +7.49634 0.001 0 0 0 0.001 0 0 +7.49878 0.001 0 0 0 0.001 0 0 +7.50122 0.001 0 0 0 0.001 0 0 +7.50366 0.001 0 0 0 0.001 0 0 +7.5061 0.001 0 0 0 0.001 0 0 +7.50854 0.001 0 0 0 0.001 0 0 +7.51099 0.001 0 0 0 0.001 0 0 +7.51343 0.001 0 0 0 0.001 0 0 +7.51587 0.001 0 0 0 0.001 0 0 +7.51831 0.001 0 0 0 0.001 0 0 +7.52075 0.001 0 0 0 0.001 0 0 +7.52319 0.001 0 0 0 0.001 0 0 +7.52563 0.001 0 0 0 0.001 0 0 +7.52808 0.001 0 0 0 0.001 0 0 +7.53052 0.001 0 0 0 0.001 0 0 +7.53296 0.001 0 0 0 0.001 0 0 +7.5354 0.001 0 0 0 0.001 0 0 +7.53784 0.001 0 0 0 0.001 0 0 +7.54028 0.001 0 0 0 0.001 0 0 +7.54272 0.001 0 0 0 0.001 0 0 +7.54517 0.001 0 0 0 0.001 0 0 +7.54761 0.001 0 0 0 0.001 0 0 +7.55005 0.001 0 0 0 0.001 0 0 +7.55249 0.001 0 0 0 0.001 0 0 +7.55493 0.001 0 0 0 0.001 0 0 +7.55737 0.001 0 0 0 0.001 0 0 +7.55981 0.001 0 0 0 0.001 0 0 +7.56226 0.001 0 0 0 0.001 0 0 +7.5647 0.001 0 0 0 0.001 0 0 +7.56714 0.001 0 0 0 0.001 0 0 +7.56958 0.001 0 0 0 0.001 0 0 +7.57202 0.001 0 0 0 0.001 0 0 +7.57446 0.001 0 0 0 0.001 0 0 +7.5769 0.001 0 0 0 0.001 0 0 +7.57935 0.001 0 0 0 0.001 0 0 +7.58179 0.001 0 0 0 0.001 0 0 +7.58423 0.001 0 0 0 0.001 0 0 +7.58667 0.001 0 0 0 0.001 0 0 +7.58911 0.001 0 0 0 0.001 0 0 +7.59155 0.001 0 0 0 0.001 0 0 +7.59399 0.001 0 0 0 0.001 0 0 +7.59644 0.001 0 0 0 0.001 0 0 +7.59888 0.001 0 0 0 0.001 0 0 +7.60132 0.001 0 0 0 0.001 0 0 +7.60376 0.001 0 0 0 0.001 0 0 +7.6062 0.001 0 0 0 0.001 0 0 +7.60864 0.001 0 0 0 0.001 0 0 +7.61108 0.001 0 0 0 0.001 0 0 +7.61353 0.001 0 0 0 0.001 0 0 +7.61597 0.001 0 0 0 0.001 0 0 +7.61841 0.001 0 0 0 0.001 0 0 +7.62085 0.001 0 0 0 0.001 0 0 +7.62329 0.001 0 0 0 0.001 0 0 +7.62573 0.001 0 0 0 0.001 0 0 +7.62817 0.001 0 0 0 0.001 0 0 +7.63062 0.001 0 0 0 0.001 0 0 +7.63306 0.001 0 0 0 0.001 0 0 +7.6355 0.001 0 0 0 0.001 0 0 +7.63794 0.001 0 0 0 0.001 0 0 +7.64038 0.001 0 0 0 0.001 0 0 +7.64282 0.001 0 0 0 0.001 0 0 +7.64526 0.001 0 0 0 0.001 0 0 +7.64771 0.001 0 0 0 0.001 0 0 +7.65015 0.001 0 0 0 0.001 0 0 +7.65259 0.001 0 0 0 0.001 0 0 +7.65503 0.001 0 0 0 0.001 0 0 +7.65747 0.001 0 0 0 0.001 0 0 +7.65991 0.001 0 0 0 0.001 0 0 +7.66235 0.001 0 0 0 0.001 0 0 +7.66479 0.001 0 0 0 0.001 0 0 +7.66724 0.001 0 0 0 0.001 0 0 +7.66968 0.001 0 0 0 0.001 0 0 +7.67212 0.001 0 0 0 0.001 0 0 +7.67456 0.001 0 0 0 0.001 0 0 +7.677 0.001 0 0 0 0.001 0 0 +7.67944 0.001 0 0 0 0.001 0 0 +7.68188 0.001 0 0 0 0.001 0 0 +7.68433 0.001 0 0 0 0.001 0 0 +7.68677 0.001 0 0 0 0.001 0 0 +7.68921 0.001 0 0 0 0.001 0 0 +7.69165 0.001 0 0 0 0.001 0 0 +7.69409 0.001 0 0 0 0.001 0 0 +7.69653 0.001 0 0 0 0.001 0 0 +7.69897 0.001 0 0 0 0.001 0 0 +7.70142 0.001 0 0 0 0.001 0 0 +7.70386 0.001 0 0 0 0.001 0 0 +7.7063 0.001 0 0 0 0.001 0 0 +7.70874 0.001 0 0 0 0.001 0 0 +7.71118 0.001 0 0 0 0.001 0 0 +7.71362 0.001 0 0 0 0.001 0 0 +7.71606 0.001 0 0 0 0.001 0 0 +7.71851 0.001 0 0 0 0.001 0 0 +7.72095 0.001 0 0 0 0.001 0 0 +7.72339 0.001 0 0 0 0.001 0 0 +7.72583 0.001 0 0 0 0.001 0 0 +7.72827 0.001 0 0 0 0.001 0 0 +7.73071 0.001 0 0 0 0.001 0 0 +7.73315 0.001 0 0 0 0.001 0 0 +7.7356 0.001 0 0 0 0.001 0 0 +7.73804 0.001 0 0 0 0.001 0 0 +7.74048 0.001 0 0 0 0.001 0 0 +7.74292 0.001 0 0 0 0.001 0 0 +7.74536 0.001 0 0 0 0.001 0 0 +7.7478 0.001 0 0 0 0.001 0 0 +7.75024 0.001 0 0 0 0.001 0 0 +7.75269 0.001 0 0 0 0.001 0 0 +7.75513 0.001 0 0 0 0.001 0 0 +7.75757 0.001 0 0 0 0.001 0 0 +7.76001 0.001 0 0 0 0.001 0 0 +7.76245 0.001 0 0 0 0.001 0 0 +7.76489 0.001 0 0 0 0.001 0 0 +7.76733 0.001 0 0 0 0.001 0 0 +7.76978 0.001 0 0 0 0.001 0 0 +7.77222 0.001 0 0 0 0.001 0 0 +7.77466 0.001 0 0 0 0.001 0 0 +7.7771 0.001 0 0 0 0.001 0 0 +7.77954 0.001 0 0 0 0.001 0 0 +7.78198 0.001 0 0 0 0.001 0 0 +7.78442 0.001 0 0 0 0.001 0 0 +7.78687 0.001 0 0 0 0.001 0 0 +7.78931 0.001 0 0 0 0.001 0 0 +7.79175 0.001 0 0 0 0.001 0 0 +7.79419 0.001 0 0 0 0.001 0 0 +7.79663 0.001 0 0 0 0.001 0 0 +7.79907 0.001 0 0 0 0.001 0 0 +7.80151 0.001 0 0 0 0.001 0 0 +7.80396 0.001 0 0 0 0.001 0 0 +7.8064 0.001 0 0 0 0.001 0 0 +7.80884 0.001 0 0 0 0.001 0 0 +7.81128 0.001 0 0 0 0.001 0 0 +7.81372 0.001 0 0 0 0.001 0 0 +7.81616 0.001 0 0 0 0.001 0 0 +7.8186 0.001 0 0 0 0.001 0 0 +7.82104 0.001 0 0 0 0.001 0 0 +7.82349 0.001 0 0 0 0.001 0 0 +7.82593 0.001 0 0 0 0.001 0 0 +7.82837 0.001 0 0 0 0.001 0 0 +7.83081 0.001 0 0 0 0.001 0 0 +7.83325 0.001 0 0 0 0.001 0 0 +7.83569 0.001 0 0 0 0.001 0 0 +7.83813 0.001 0 0 0 0.001 0 0 +7.84058 0.001 0 0 0 0.001 0 0 +7.84302 0.001 0 0 0 0.001 0 0 +7.84546 0.001 0 0 0 0.001 0 0 +7.8479 0.001 0 0 0 0.001 0 0 +7.85034 0.001 0 0 0 0.001 0 0 +7.85278 0.001 0 0 0 0.001 0 0 +7.85522 0.001 0 0 0 0.001 0 0 +7.85767 0.001 0 0 0 0.001 0 0 +7.86011 0.001 0 0 0 0.001 0 0 +7.86255 0.001 0 0 0 0.001 0 0 +7.86499 0.001 0 0 0 0.001 0 0 +7.86743 0.001 0 0 0 0.001 0 0 +7.86987 0.001 0 0 0 0.001 0 0 +7.87231 0.001 0 0 0 0.001 0 0 +7.87476 0.001 0 0 0 0.001 0 0 +7.8772 0.001 0 0 0 0.001 0 0 +7.87964 0.001 0 0 0 0.001 0 0 +7.88208 0.001 0 0 0 0.001 0 0 +7.88452 0.001 0 0 0 0.001 0 0 +7.88696 0.001 0 0 0 0.001 0 0 +7.8894 0.001 0 0 0 0.001 0 0 +7.89185 0.001 0 0 0 0.001 0 0 +7.89429 0.001 0 0 0 0.001 0 0 +7.89673 0.001 0 0 0 0.001 0 0 +7.89917 0.001 0 0 0 0.001 0 0 +7.90161 0.001 0 0 0 0.001 0 0 +7.90405 0.001 0 0 0 0.001 0 0 +7.90649 0.001 0 0 0 0.001 0 0 +7.90894 0.001 0 0 0 0.001 0 0 +7.91138 0.001 0 0 0 0.001 0 0 +7.91382 0.001 0 0 0 0.001 0 0 +7.91626 0.001 0 0 0 0.001 0 0 +7.9187 0.001 0 0 0 0.001 0 0 +7.92114 0.001 0 0 0 0.001 0 0 +7.92358 0.001 0 0 0 0.001 0 0 +7.92603 0.001 0 0 0 0.001 0 0 +7.92847 0.001 0 0 0 0.001 0 0 +7.93091 0.001 0 0 0 0.001 0 0 +7.93335 0.001 0 0 0 0.001 0 0 +7.93579 0.001 0 0 0 0.001 0 0 +7.93823 0.001 0 0 0 0.001 0 0 +7.94067 0.001 0 0 0 0.001 0 0 +7.94312 0.001 0 0 0 0.001 0 0 +7.94556 0.001 0 0 0 0.001 0 0 +7.948 0.001 0 0 0 0.001 0 0 +7.95044 0.001 0 0 0 0.001 0 0 +7.95288 0.001 0 0 0 0.001 0 0 +7.95532 0.001 0 0 0 0.001 0 0 +7.95776 0.001 0 0 0 0.001 0 0 +7.96021 0.001 0 0 0 0.001 0 0 +7.96265 0.001 0 0 0 0.001 0 0 +7.96509 0.001 0 0 0 0.001 0 0 +7.96753 0.001 0 0 0 0.001 0 0 +7.96997 0.001 0 0 0 0.001 0 0 +7.97241 0.001 0 0 0 0.001 0 0 +7.97485 0.001 0 0 0 0.001 0 0 +7.97729 0.001 0 0 0 0.001 0 0 +7.97974 0.001 0 0 0 0.001 0 0 +7.98218 0.001 0 0 0 0.001 0 0 +7.98462 0.001 0 0 0 0.001 0 0 +7.98706 0.001 0 0 0 0.001 0 0 +7.9895 0.001 0 0 0 0.001 0 0 +7.99194 0.001 0 0 0 0.001 0 0 +7.99438 0.001 0 0 0 0.001 0 0 +7.99683 0.001 0 0 0 0.001 0 0 +7.99927 0.001 0 0 0 0.001 0 0 +8.00171 0.001 0 0 0 0.001 0 0 +8.00415 0.001 0 0 0 0.001 0 0 +8.00659 0.001 0 0 0 0.001 0 0 +8.00903 0.001 0 0 0 0.001 0 0 +8.01147 0.001 0 0 0 0.001 0 0 +8.01392 0.001 0 0 0 0.001 0 0 +8.01636 0.001 0 0 0 0.001 0 0 +8.0188 0.001 0 0 0 0.001 0 0 +8.02124 0.001 0 0 0 0.001 0 0 +8.02368 0.001 0 0 0 0.001 0 0 +8.02612 0.001 0 0 0 0.001 0 0 +8.02856 0.001 0 0 0 0.001 0 0 +8.03101 0.001 0 0 0 0.001 0 0 +8.03345 0.001 0 0 0 0.001 0 0 +8.03589 0.001 0 0 0 0.001 0 0 +8.03833 0.001 0 0 0 0.001 0 0 +8.04077 0.001 0 0 0 0.001 0 0 +8.04321 0.001 0 0 0 0.001 0 0 +8.04565 0.001 0 0 0 0.001 0 0 +8.0481 0.001 0 0 0 0.001 0 0 +8.05054 0.001 0 0 0 0.001 0 0 +8.05298 0.001 0 0 0 0.001 0 0 +8.05542 0.001 0 0 0 0.001 0 0 +8.05786 0.001 0 0 0 0.001 0 0 +8.0603 0.001 0 0 0 0.001 0 0 +8.06274 0.001 0 0 0 0.001 0 0 +8.06519 0.001 0 0 0 0.001 0 0 +8.06763 0.001 0 0 0 0.001 0 0 +8.07007 0.001 0 0 0 0.001 0 0 +8.07251 0.001 0 0 0 0.001 0 0 +8.07495 0.001 0 0 0 0.001 0 0 +8.07739 0.001 0 0 0 0.001 0 0 +8.07983 0.001 0 0 0 0.001 0 0 +8.08228 0.001 0 0 0 0.001 0 0 +8.08472 0.001 0 0 0 0.001 0 0 +8.08716 0.001 0 0 0 0.001 0 0 +8.0896 0.001 0 0 0 0.001 0 0 +8.09204 0.001 0 0 0 0.001 0 0 +8.09448 0.001 0 0 0 0.001 0 0 +8.09692 0.001 0 0 0 0.001 0 0 +8.09937 0.001 0 0 0 0.001 0 0 +8.10181 0.001 0 0 0 0.001 0 0 +8.10425 0.001 0 0 0 0.001 0 0 +8.10669 0.001 0 0 0 0.001 0 0 +8.10913 0.001 0 0 0 0.001 0 0 +8.11157 0.001 0 0 0 0.001 0 0 +8.11401 0.001 0 0 0 0.001 0 0 +8.11646 0.001 0 0 0 0.001 0 0 +8.1189 0.001 0 0 0 0.001 0 0 +8.12134 0.001 0 0 0 0.001 0 0 +8.12378 0.001 0 0 0 0.001 0 0 +8.12622 0.001 0 0 0 0.001 0 0 +8.12866 0.001 0 0 0 0.001 0 0 +8.1311 0.001 0 0 0 0.001 0 0 +8.13354 0.001 0 0 0 0.001 0 0 +8.13599 0.001 0 0 0 0.001 0 0 +8.13843 0.001 0 0 0 0.001 0 0 +8.14087 0.001 0 0 0 0.001 0 0 +8.14331 0.001 0 0 0 0.001 0 0 +8.14575 0.001 0 0 0 0.001 0 0 +8.14819 0.001 0 0 0 0.001 0 0 +8.15063 0.001 0 0 0 0.001 0 0 +8.15308 0.001 0 0 0 0.001 0 0 +8.15552 0.001 0 0 0 0.001 0 0 +8.15796 0.001 0 0 0 0.001 0 0 +8.1604 0.001 0 0 0 0.001 0 0 +8.16284 0.001 0 0 0 0.001 0 0 +8.16528 0.001 0 0 0 0.001 0 0 +8.16772 0.001 0 0 0 0.001 0 0 +8.17017 0.001 0 0 0 0.001 0 0 +8.17261 0.001 0 0 0 0.001 0 0 +8.17505 0.001 0 0 0 0.001 0 0 +8.17749 0.001 0 0 0 0.001 0 0 +8.17993 0.001 0 0 0 0.001 0 0 +8.18237 0.001 0 0 0 0.001 0 0 +8.18481 0.001 0 0 0 0.001 0 0 +8.18726 0.001 0 0 0 0.001 0 0 +8.1897 0.001 0 0 0 0.001 0 0 +8.19214 0.001 0 0 0 0.001 0 0 +8.19458 0.001 0 0 0 0.001 0 0 +8.19702 0.001 0 0 0 0.001 0 0 +8.19946 0.001 0 0 0 0.001 0 0 +8.2019 0.001 0 0 0 0.001 0 0 +8.20435 0.001 0 0 0 0.001 0 0 +8.20679 0.001 0 0 0 0.001 0 0 +8.20923 0.001 0 0 0 0.001 0 0 +8.21167 0.001 0 0 0 0.001 0 0 +8.21411 0.001 0 0 0 0.001 0 0 +8.21655 0.001 0 0 0 0.001 0 0 +8.21899 0.001 0 0 0 0.001 0 0 +8.22144 0.001 0 0 0 0.001 0 0 +8.22388 0.001 0 0 0 0.001 0 0 +8.22632 0.001 0 0 0 0.001 0 0 +8.22876 0.001 0 0 0 0.001 0 0 +8.2312 0.001 0 0 0 0.001 0 0 +8.23364 0.001 0 0 0 0.001 0 0 +8.23608 0.001 0 0 0 0.001 0 0 +8.23853 0.001 0 0 0 0.001 0 0 +8.24097 0.001 0 0 0 0.001 0 0 +8.24341 0.001 0 0 0 0.001 0 0 +8.24585 0.001 0 0 0 0.001 0 0 +8.24829 0.001 0 0 0 0.001 0 0 +8.25073 0.001 0 0 0 0.001 0 0 +8.25317 0.001 0 0 0 0.001 0 0 +8.25562 0.001 0 0 0 0.001 0 0 +8.25806 0.001 0 0 0 0.001 0 0 +8.2605 0.001 0 0 0 0.001 0 0 +8.26294 0.001 0 0 0 0.001 0 0 +8.26538 0.001 0 0 0 0.001 0 0 +8.26782 0.001 0 0 0 0.001 0 0 +8.27026 0.001 0 0 0 0.001 0 0 +8.27271 0.001 0 0 0 0.001 0 0 +8.27515 0.001 0 0 0 0.001 0 0 +8.27759 0.001 0 0 0 0.001 0 0 +8.28003 0.001 0 0 0 0.001 0 0 +8.28247 0.001 0 0 0 0.001 0 0 +8.28491 0.001 0 0 0 0.001 0 0 +8.28735 0.001 0 0 0 0.001 0 0 +8.28979 0.001 0 0 0 0.001 0 0 +8.29224 0.001 0 0 0 0.001 0 0 +8.29468 0.001 0 0 0 0.001 0 0 +8.29712 0.001 0 0 0 0.001 0 0 +8.29956 0.001 0 0 0 0.001 0 0 +8.302 0.001 0 0 0 0.001 0 0 +8.30444 0.001 0 0 0 0.001 0 0 +8.30688 0.001 0 0 0 0.001 0 0 +8.30933 0.001 0 0 0 0.001 0 0 +8.31177 0.001 0 0 0 0.001 0 0 +8.31421 0.001 0 0 0 0.001 0 0 +8.31665 0.001 0 0 0 0.001 0 0 +8.31909 0.001 0 0 0 0.001 0 0 +8.32153 0.001 0 0 0 0.001 0 0 +8.32397 0.001 0 0 0 0.001 0 0 +8.32642 0.001 0 0 0 0.001 0 0 +8.32886 0.001 0 0 0 0.001 0 0 +8.3313 0.001 0 0 0 0.001 0 0 +8.33374 0.001 0 0 0 0.001 0 0 +8.33618 0.001 0 0 0 0.001 0 0 +8.33862 0.001 0 0 0 0.001 0 0 +8.34106 0.001 0 0 0 0.001 0 0 +8.34351 0.001 0 0 0 0.001 0 0 +8.34595 0.001 0 0 0 0.001 0 0 +8.34839 0.001 0 0 0 0.001 0 0 +8.35083 0.001 0 0 0 0.001 0 0 +8.35327 0.001 0 0 0 0.001 0 0 +8.35571 0.001 0 0 0 0.001 0 0 +8.35815 0.001 0 0 0 0.001 0 0 +8.3606 0.001 0 0 0 0.001 0 0 +8.36304 0.001 0 0 0 0.001 0 0 +8.36548 0.001 0 0 0 0.001 0 0 +8.36792 0.001 0 0 0 0.001 0 0 +8.37036 0.001 0 0 0 0.001 0 0 +8.3728 0.001 0 0 0 0.001 0 0 +8.37524 0.001 0 0 0 0.001 0 0 +8.37769 0.001 0 0 0 0.001 0 0 +8.38013 0.001 0 0 0 0.001 0 0 +8.38257 0.001 0 0 0 0.001 0 0 +8.38501 0.001 0 0 0 0.001 0 0 +8.38745 0.001 0 0 0 0.001 0 0 +8.38989 0.001 0 0 0 0.001 0 0 +8.39233 0.001 0 0 0 0.001 0 0 +8.39478 0.001 0 0 0 0.001 0 0 +8.39722 0.001 0 0 0 0.001 0 0 +8.39966 0.001 0 0 0 0.001 0 0 +8.4021 0.001 0 0 0 0.001 0 0 +8.40454 0.001 0 0 0 0.001 0 0 +8.40698 0.001 0 0 0 0.001 0 0 +8.40942 0.001 0 0 0 0.001 0 0 +8.41187 0.001 0 0 0 0.001 0 0 +8.41431 0.001 0 0 0 0.001 0 0 +8.41675 0.001 0 0 0 0.001 0 0 +8.41919 0.001 0 0 0 0.001 0 0 +8.42163 0.001 0 0 0 0.001 0 0 +8.42407 0.001 0 0 0 0.001 0 0 +8.42651 0.001 0 0 0 0.001 0 0 +8.42896 0.001 0 0 0 0.001 0 0 +8.4314 0.001 0 0 0 0.001 0 0 +8.43384 0.001 0 0 0 0.001 0 0 +8.43628 0.001 0 0 0 0.001 0 0 +8.43872 0.001 0 0 0 0.001 0 0 +8.44116 0.001 0 0 0 0.001 0 0 +8.4436 0.001 0 0 0 0.001 0 0 +8.44604 0.001 0 0 0 0.001 0 0 +8.44849 0.001 0 0 0 0.001 0 0 +8.45093 0.001 0 0 0 0.001 0 0 +8.45337 0.001 0 0 0 0.001 0 0 +8.45581 0.001 0 0 0 0.001 0 0 +8.45825 0.001 0 0 0 0.001 0 0 +8.46069 0.001 0 0 0 0.001 0 0 +8.46313 0.001 0 0 0 0.001 0 0 +8.46558 0.001 0 0 0 0.001 0 0 +8.46802 0.001 0 0 0 0.001 0 0 +8.47046 0.001 0 0 0 0.001 0 0 +8.4729 0.001 0 0 0 0.001 0 0 +8.47534 0.001 0 0 0 0.001 0 0 +8.47778 0.001 0 0 0 0.001 0 0 +8.48022 0.001 0 0 0 0.001 0 0 +8.48267 0.001 0 0 0 0.001 0 0 +8.48511 0.001 0 0 0 0.001 0 0 +8.48755 0.001 0 0 0 0.001 0 0 +8.48999 0.001 0 0 0 0.001 0 0 +8.49243 0.001 0 0 0 0.001 0 0 +8.49487 0.001 0 0 0 0.001 0 0 +8.49731 0.001 0 0 0 0.001 0 0 +8.49976 0.001 0 0 0 0.001 0 0 +8.5022 0.001 0 0 0 0.001 0 0 +8.50464 0.001 0 0 0 0.001 0 0 +8.50708 0.001 0 0 0 0.001 0 0 +8.50952 0.001 0 0 0 0.001 0 0 +8.51196 0.001 0 0 0 0.001 0 0 +8.5144 0.001 0 0 0 0.001 0 0 +8.51685 0.001 0 0 0 0.001 0 0 +8.51929 0.001 0 0 0 0.001 0 0 +8.52173 0.001 0 0 0 0.001 0 0 +8.52417 0.001 0 0 0 0.001 0 0 +8.52661 0.001 0 0 0 0.001 0 0 +8.52905 0.001 0 0 0 0.001 0 0 +8.53149 0.001 0 0 0 0.001 0 0 +8.53394 0.001 0 0 0 0.001 0 0 +8.53638 0.001 0 0 0 0.001 0 0 +8.53882 0.001 0 0 0 0.001 0 0 +8.54126 0.001 0 0 0 0.001 0 0 +8.5437 0.001 0 0 0 0.001 0 0 +8.54614 0.001 0 0 0 0.001 0 0 +8.54858 0.001 0 0 0 0.001 0 0 +8.55103 0.001 0 0 0 0.001 0 0 +8.55347 0.001 0 0 0 0.001 0 0 +8.55591 0.001 0 0 0 0.001 0 0 +8.55835 0.001 0 0 0 0.001 0 0 +8.56079 0.001 0 0 0 0.001 0 0 +8.56323 0.001 0 0 0 0.001 0 0 +8.56567 0.001 0 0 0 0.001 0 0 +8.56812 0.001 0 0 0 0.001 0 0 +8.57056 0.001 0 0 0 0.001 0 0 +8.573 0.001 0 0 0 0.001 0 0 +8.57544 0.001 0 0 0 0.001 0 0 +8.57788 0.001 0 0 0 0.001 0 0 +8.58032 0.001 0 0 0 0.001 0 0 +8.58276 0.001 0 0 0 0.001 0 0 +8.58521 0.001 0 0 0 0.001 0 0 +8.58765 0.001 0 0 0 0.001 0 0 +8.59009 0.001 0 0 0 0.001 0 0 +8.59253 0.001 0 0 0 0.001 0 0 +8.59497 0.001 0 0 0 0.001 0 0 +8.59741 0.001 0 0 0 0.001 0 0 +8.59985 0.001 0 0 0 0.001 0 0 +8.60229 0.001 0 0 0 0.001 0 0 +8.60474 0.001 0 0 0 0.001 0 0 +8.60718 0.001 0 0 0 0.001 0 0 +8.60962 0.001 0 0 0 0.001 0 0 +8.61206 0.001 0 0 0 0.001 0 0 +8.6145 0.001 0 0 0 0.001 0 0 +8.61694 0.001 0 0 0 0.001 0 0 +8.61938 0.001 0 0 0 0.001 0 0 +8.62183 0.001 0 0 0 0.001 0 0 +8.62427 0.001 0 0 0 0.001 0 0 +8.62671 0.001 0 0 0 0.001 0 0 +8.62915 0.001 0 0 0 0.001 0 0 +8.63159 0.001 0 0 0 0.001 0 0 +8.63403 0.001 0 0 0 0.001 0 0 +8.63647 0.001 0 0 0 0.001 0 0 +8.63892 0.001 0 0 0 0.001 0 0 +8.64136 0.001 0 0 0 0.001 0 0 +8.6438 0.001 0 0 0 0.001 0 0 +8.64624 0.001 0 0 0 0.001 0 0 +8.64868 0.001 0 0 0 0.001 0 0 +8.65112 0.001 0 0 0 0.001 0 0 +8.65356 0.001 0 0 0 0.001 0 0 +8.65601 0.001 0 0 0 0.001 0 0 +8.65845 0.001 0 0 0 0.001 0 0 +8.66089 0.001 0 0 0 0.001 0 0 +8.66333 0.001 0 0 0 0.001 0 0 +8.66577 0.001 0 0 0 0.001 0 0 +8.66821 0.001 0 0 0 0.001 0 0 +8.67065 0.001 0 0 0 0.001 0 0 +8.6731 0.001 0 0 0 0.001 0 0 +8.67554 0.001 0 0 0 0.001 0 0 +8.67798 0.001 0 0 0 0.001 0 0 +8.68042 0.001 0 0 0 0.001 0 0 +8.68286 0.001 0 0 0 0.001 0 0 +8.6853 0.001 0 0 0 0.001 0 0 +8.68774 0.001 0 0 0 0.001 0 0 +8.69019 0.001 0 0 0 0.001 0 0 +8.69263 0.001 0 0 0 0.001 0 0 +8.69507 0.001 0 0 0 0.001 0 0 +8.69751 0.001 0 0 0 0.001 0 0 +8.69995 0.001 0 0 0 0.001 0 0 +8.70239 0.001 0 0 0 0.001 0 0 +8.70483 0.001 0 0 0 0.001 0 0 +8.70728 0.001 0 0 0 0.001 0 0 +8.70972 0.001 0 0 0 0.001 0 0 +8.71216 0.001 0 0 0 0.001 0 0 +8.7146 0.001 0 0 0 0.001 0 0 +8.71704 0.001 0 0 0 0.001 0 0 +8.71948 0.001 0 0 0 0.001 0 0 +8.72192 0.001 0 0 0 0.001 0 0 +8.72437 0.001 0 0 0 0.001 0 0 +8.72681 0.001 0 0 0 0.001 0 0 +8.72925 0.001 0 0 0 0.001 0 0 +8.73169 0.001 0 0 0 0.001 0 0 +8.73413 0.001 0 0 0 0.001 0 0 +8.73657 0.001 0 0 0 0.001 0 0 +8.73901 0.001 0 0 0 0.001 0 0 +8.74146 0.001 0 0 0 0.001 0 0 +8.7439 0.001 0 0 0 0.001 0 0 +8.74634 0.001 0 0 0 0.001 0 0 +8.74878 0.001 0 0 0 0.001 0 0 +8.75122 0.001 0 0 0 0.001 0 0 +8.75366 0.001 0 0 0 0.001 0 0 +8.7561 0.001 0 0 0 0.001 0 0 +8.75854 0.001 0 0 0 0.001 0 0 +8.76099 0.001 0 0 0 0.001 0 0 +8.76343 0.001 0 0 0 0.001 0 0 +8.76587 0.001 0 0 0 0.001 0 0 +8.76831 0.001 0 0 0 0.001 0 0 +8.77075 0.001 0 0 0 0.001 0 0 +8.77319 0.001 0 0 0 0.001 0 0 +8.77563 0.001 0 0 0 0.001 0 0 +8.77808 0.001 0 0 0 0.001 0 0 +8.78052 0.001 0 0 0 0.001 0 0 +8.78296 0.001 0 0 0 0.001 0 0 +8.7854 0.001 0 0 0 0.001 0 0 +8.78784 0.001 0 0 0 0.001 0 0 +8.79028 0.001 0 0 0 0.001 0 0 +8.79272 0.001 0 0 0 0.001 0 0 +8.79517 0.001 0 0 0 0.001 0 0 +8.79761 0.001 0 0 0 0.001 0 0 +8.80005 0.001 0 0 0 0.001 0 0 +8.80249 0.001 0 0 0 0.001 0 0 +8.80493 0.001 0 0 0 0.001 0 0 +8.80737 0.001 0 0 0 0.001 0 0 +8.80981 0.001 0 0 0 0.001 0 0 +8.81226 0.001 0 0 0 0.001 0 0 +8.8147 0.001 0 0 0 0.001 0 0 +8.81714 0.001 0 0 0 0.001 0 0 +8.81958 0.001 0 0 0 0.001 0 0 +8.82202 0.001 0 0 0 0.001 0 0 +8.82446 0.001 0 0 0 0.001 0 0 +8.8269 0.001 0 0 0 0.001 0 0 +8.82935 0.001 0 0 0 0.001 0 0 +8.83179 0.001 0 0 0 0.001 0 0 +8.83423 0.001 0 0 0 0.001 0 0 +8.83667 0.001 0 0 0 0.001 0 0 +8.83911 0.001 0 0 0 0.001 0 0 +8.84155 0.001 0 0 0 0.001 0 0 +8.84399 0.001 0 0 0 0.001 0 0 +8.84644 0.001 0 0 0 0.001 0 0 +8.84888 0.001 0 0 0 0.001 0 0 +8.85132 0.001 0 0 0 0.001 0 0 +8.85376 0.001 0 0 0 0.001 0 0 +8.8562 0.001 0 0 0 0.001 0 0 +8.85864 0.001 0 0 0 0.001 0 0 +8.86108 0.001 0 0 0 0.001 0 0 +8.86353 0.001 0 0 0 0.001 0 0 +8.86597 0.001 0 0 0 0.001 0 0 +8.86841 0.001 0 0 0 0.001 0 0 +8.87085 0.001 0 0 0 0.001 0 0 +8.87329 0.001 0 0 0 0.001 0 0 +8.87573 0.001 0 0 0 0.001 0 0 +8.87817 0.001 0 0 0 0.001 0 0 +8.88062 0.001 0 0 0 0.001 0 0 +8.88306 0.001 0 0 0 0.001 0 0 +8.8855 0.001 0 0 0 0.001 0 0 +8.88794 0.001 0 0 0 0.001 0 0 +8.89038 0.001 0 0 0 0.001 0 0 +8.89282 0.001 0 0 0 0.001 0 0 +8.89526 0.001 0 0 0 0.001 0 0 +8.89771 0.001 0 0 0 0.001 0 0 +8.90015 0.001 0 0 0 0.001 0 0 +8.90259 0.001 0 0 0 0.001 0 0 +8.90503 0.001 0 0 0 0.001 0 0 +8.90747 0.001 0 0 0 0.001 0 0 +8.90991 0.001 0 0 0 0.001 0 0 +8.91235 0.001 0 0 0 0.001 0 0 +8.91479 0.001 0 0 0 0.001 0 0 +8.91724 0.001 0 0 0 0.001 0 0 +8.91968 0.001 0 0 0 0.001 0 0 +8.92212 0.001 0 0 0 0.001 0 0 +8.92456 0.001 0 0 0 0.001 0 0 +8.927 0.001 0 0 0 0.001 0 0 +8.92944 0.001 0 0 0 0.001 0 0 +8.93188 0.001 0 0 0 0.001 0 0 +8.93433 0.001 0 0 0 0.001 0 0 +8.93677 0.001 0 0 0 0.001 0 0 +8.93921 0.001 0 0 0 0.001 0 0 +8.94165 0.001 0 0 0 0.001 0 0 +8.94409 0.001 0 0 0 0.001 0 0 +8.94653 0.001 0 0 0 0.001 0 0 +8.94897 0.001 0 0 0 0.001 0 0 +8.95142 0.001 0 0 0 0.001 0 0 +8.95386 0.001 0 0 0 0.001 0 0 +8.9563 0.001 0 0 0 0.001 0 0 +8.95874 0.001 0 0 0 0.001 0 0 +8.96118 0.001 0 0 0 0.001 0 0 +8.96362 0.001 0 0 0 0.001 0 0 +8.96606 0.001 0 0 0 0.001 0 0 +8.96851 0.001 0 0 0 0.001 0 0 +8.97095 0.001 0 0 0 0.001 0 0 +8.97339 0.001 0 0 0 0.001 0 0 +8.97583 0.001 0 0 0 0.001 0 0 +8.97827 0.001 0 0 0 0.001 0 0 +8.98071 0.001 0 0 0 0.001 0 0 +8.98315 0.001 0 0 0 0.001 0 0 +8.9856 0.001 0 0 0 0.001 0 0 +8.98804 0.001 0 0 0 0.001 0 0 +8.99048 0.001 0 0 0 0.001 0 0 +8.99292 0.001 0 0 0 0.001 0 0 +8.99536 0.001 0 0 0 0.001 0 0 +8.9978 0.001 0 0 0 0.001 0 0 +9.00024 0.001 0 0 0 0.001 0 0 +9.00269 0.001 0 0 0 0.001 0 0 +9.00513 0.001 0 0 0 0.001 0 0 +9.00757 0.001 0 0 0 0.001 0 0 +9.01001 0.001 0 0 0 0.001 0 0 +9.01245 0.001 0 0 0 0.001 0 0 +9.01489 0.001 0 0 0 0.001 0 0 +9.01733 0.001 0 0 0 0.001 0 0 +9.01978 0.001 0 0 0 0.001 0 0 +9.02222 0.001 0 0 0 0.001 0 0 +9.02466 0.001 0 0 0 0.001 0 0 +9.0271 0.001 0 0 0 0.001 0 0 +9.02954 0.001 0 0 0 0.001 0 0 +9.03198 0.001 0 0 0 0.001 0 0 +9.03442 0.001 0 0 0 0.001 0 0 +9.03687 0.001 0 0 0 0.001 0 0 +9.03931 0.001 0 0 0 0.001 0 0 +9.04175 0.001 0 0 0 0.001 0 0 +9.04419 0.001 0 0 0 0.001 0 0 +9.04663 0.001 0 0 0 0.001 0 0 +9.04907 0.001 0 0 0 0.001 0 0 +9.05151 0.001 0 0 0 0.001 0 0 +9.05396 0.001 0 0 0 0.001 0 0 +9.0564 0.001 0 0 0 0.001 0 0 +9.05884 0.001 0 0 0 0.001 0 0 +9.06128 0.001 0 0 0 0.001 0 0 +9.06372 0.001 0 0 0 0.001 0 0 +9.06616 0.001 0 0 0 0.001 0 0 +9.0686 0.001 0 0 0 0.001 0 0 +9.07104 0.001 0 0 0 0.001 0 0 +9.07349 0.001 0 0 0 0.001 0 0 +9.07593 0.001 0 0 0 0.001 0 0 +9.07837 0.001 0 0 0 0.001 0 0 +9.08081 0.001 0 0 0 0.001 0 0 +9.08325 0.001 0 0 0 0.001 0 0 +9.08569 0.001 0 0 0 0.001 0 0 +9.08813 0.001 0 0 0 0.001 0 0 +9.09058 0.001 0 0 0 0.001 0 0 +9.09302 0.001 0 0 0 0.001 0 0 +9.09546 0.001 0 0 0 0.001 0 0 +9.0979 0.001 0 0 0 0.001 0 0 +9.10034 0.001 0 0 0 0.001 0 0 +9.10278 0.001 0 0 0 0.001 0 0 +9.10522 0.001 0 0 0 0.001 0 0 +9.10767 0.001 0 0 0 0.001 0 0 +9.11011 0.001 0 0 0 0.001 0 0 +9.11255 0.001 0 0 0 0.001 0 0 +9.11499 0.001 0 0 0 0.001 0 0 +9.11743 0.001 0 0 0 0.001 0 0 +9.11987 0.001 0 0 0 0.001 0 0 +9.12231 0.001 0 0 0 0.001 0 0 +9.12476 0.001 0 0 0 0.001 0 0 +9.1272 0.001 0 0 0 0.001 0 0 +9.12964 0.001 0 0 0 0.001 0 0 +9.13208 0.001 0 0 0 0.001 0 0 +9.13452 0.001 0 0 0 0.001 0 0 +9.13696 0.001 0 0 0 0.001 0 0 +9.1394 0.001 0 0 0 0.001 0 0 +9.14185 0.001 0 0 0 0.001 0 0 +9.14429 0.001 0 0 0 0.001 0 0 +9.14673 0.001 0 0 0 0.001 0 0 +9.14917 0.001 0 0 0 0.001 0 0 +9.15161 0.001 0 0 0 0.001 0 0 +9.15405 0.001 0 0 0 0.001 0 0 +9.15649 0.001 0 0 0 0.001 0 0 +9.15894 0.001 0 0 0 0.001 0 0 +9.16138 0.001 0 0 0 0.001 0 0 +9.16382 0.001 0 0 0 0.001 0 0 +9.16626 0.001 0 0 0 0.001 0 0 +9.1687 0.001 0 0 0 0.001 0 0 +9.17114 0.001 0 0 0 0.001 0 0 +9.17358 0.001 0 0 0 0.001 0 0 +9.17603 0.001 0 0 0 0.001 0 0 +9.17847 0.001 0 0 0 0.001 0 0 +9.18091 0.001 0 0 0 0.001 0 0 +9.18335 0.001 0 0 0 0.001 0 0 +9.18579 0.001 0 0 0 0.001 0 0 +9.18823 0.001 0 0 0 0.001 0 0 +9.19067 0.001 0 0 0 0.001 0 0 +9.19312 0.001 0 0 0 0.001 0 0 +9.19556 0.001 0 0 0 0.001 0 0 +9.198 0.001 0 0 0 0.001 0 0 +9.20044 0.001 0 0 0 0.001 0 0 +9.20288 0.001 0 0 0 0.001 0 0 +9.20532 0.001 0 0 0 0.001 0 0 +9.20776 0.001 0 0 0 0.001 0 0 +9.21021 0.001 0 0 0 0.001 0 0 +9.21265 0.001 0 0 0 0.001 0 0 +9.21509 0.001 0 0 0 0.001 0 0 +9.21753 0.001 0 0 0 0.001 0 0 +9.21997 0.001 0 0 0 0.001 0 0 +9.22241 0.001 0 0 0 0.001 0 0 +9.22485 0.001 0 0 0 0.001 0 0 +9.22729 0.001 0 0 0 0.001 0 0 +9.22974 0.001 0 0 0 0.001 0 0 +9.23218 0.001 0 0 0 0.001 0 0 +9.23462 0.001 0 0 0 0.001 0 0 +9.23706 0.001 0 0 0 0.001 0 0 +9.2395 0.001 0 0 0 0.001 0 0 +9.24194 0.001 0 0 0 0.001 0 0 +9.24438 0.001 0 0 0 0.001 0 0 +9.24683 0.001 0 0 0 0.001 0 0 +9.24927 0.001 0 0 0 0.001 0 0 +9.25171 0.001 0 0 0 0.001 0 0 +9.25415 0.001 0 0 0 0.001 0 0 +9.25659 0.001 0 0 0 0.001 0 0 +9.25903 0.001 0 0 0 0.001 0 0 +9.26147 0.001 0 0 0 0.001 0 0 +9.26392 0.001 0 0 0 0.001 0 0 +9.26636 0.001 0 0 0 0.001 0 0 +9.2688 0.001 0 0 0 0.001 0 0 +9.27124 0.001 0 0 0 0.001 0 0 +9.27368 0.001 0 0 0 0.001 0 0 +9.27612 0.001 0 0 0 0.001 0 0 +9.27856 0.001 0 0 0 0.001 0 0 +9.28101 0.001 0 0 0 0.001 0 0 +9.28345 0.001 0 0 0 0.001 0 0 +9.28589 0.001 0 0 0 0.001 0 0 +9.28833 0.001 0 0 0 0.001 0 0 +9.29077 0.001 0 0 0 0.001 0 0 +9.29321 0.001 0 0 0 0.001 0 0 +9.29565 0.001 0 0 0 0.001 0 0 +9.2981 0.001 0 0 0 0.001 0 0 +9.30054 0.001 0 0 0 0.001 0 0 +9.30298 0.001 0 0 0 0.001 0 0 +9.30542 0.001 0 0 0 0.001 0 0 +9.30786 0.001 0 0 0 0.001 0 0 +9.3103 0.001 0 0 0 0.001 0 0 +9.31274 0.001 0 0 0 0.001 0 0 +9.31519 0.001 0 0 0 0.001 0 0 +9.31763 0.001 0 0 0 0.001 0 0 +9.32007 0.001 0 0 0 0.001 0 0 +9.32251 0.001 0 0 0 0.001 0 0 +9.32495 0.001 0 0 0 0.001 0 0 +9.32739 0.001 0 0 0 0.001 0 0 +9.32983 0.001 0 0 0 0.001 0 0 +9.33228 0.001 0 0 0 0.001 0 0 +9.33472 0.001 0 0 0 0.001 0 0 +9.33716 0.001 0 0 0 0.001 0 0 +9.3396 0.001 0 0 0 0.001 0 0 +9.34204 0.001 0 0 0 0.001 0 0 +9.34448 0.001 0 0 0 0.001 0 0 +9.34692 0.001 0 0 0 0.001 0 0 +9.34937 0.001 0 0 0 0.001 0 0 +9.35181 0.001 0 0 0 0.001 0 0 +9.35425 0.001 0 0 0 0.001 0 0 +9.35669 0.001 0 0 0 0.001 0 0 +9.35913 0.001 0 0 0 0.001 0 0 +9.36157 0.001 0 0 0 0.001 0 0 +9.36401 0.001 0 0 0 0.001 0 0 +9.36646 0.001 0 0 0 0.001 0 0 +9.3689 0.001 0 0 0 0.001 0 0 +9.37134 0.001 0 0 0 0.001 0 0 +9.37378 0.001 0 0 0 0.001 0 0 +9.37622 0.001 0 0 0 0.001 0 0 +9.37866 0.001 0 0 0 0.001 0 0 +9.3811 0.001 0 0 0 0.001 0 0 +9.38354 0.001 0 0 0 0.001 0 0 +9.38599 0.001 0 0 0 0.001 0 0 +9.38843 0.001 0 0 0 0.001 0 0 +9.39087 0.001 0 0 0 0.001 0 0 +9.39331 0.001 0 0 0 0.001 0 0 +9.39575 0.001 0 0 0 0.001 0 0 +9.39819 0.001 0 0 0 0.001 0 0 +9.40063 0.001 0 0 0 0.001 0 0 +9.40308 0.001 0 0 0 0.001 0 0 +9.40552 0.001 0 0 0 0.001 0 0 +9.40796 0.001 0 0 0 0.001 0 0 +9.4104 0.001 0 0 0 0.001 0 0 +9.41284 0.001 0 0 0 0.001 0 0 +9.41528 0.001 0 0 0 0.001 0 0 +9.41772 0.001 0 0 0 0.001 0 0 +9.42017 0.001 0 0 0 0.001 0 0 +9.42261 0.001 0 0 0 0.001 0 0 +9.42505 0.001 0 0 0 0.001 0 0 +9.42749 0.001 0 0 0 0.001 0 0 +9.42993 0.001 0 0 0 0.001 0 0 +9.43237 0.001 0 0 0 0.001 0 0 +9.43481 0.001 0 0 0 0.001 0 0 +9.43726 0.001 0 0 0 0.001 0 0 +9.4397 0.001 0 0 0 0.001 0 0 +9.44214 0.001 0 0 0 0.001 0 0 +9.44458 0.001 0 0 0 0.001 0 0 +9.44702 0.001 0 0 0 0.001 0 0 +9.44946 0.001 0 0 0 0.001 0 0 +9.4519 0.001 0 0 0 0.001 0 0 +9.45435 0.001 0 0 0 0.001 0 0 +9.45679 0.001 0 0 0 0.001 0 0 +9.45923 0.001 0 0 0 0.001 0 0 +9.46167 0.001 0 0 0 0.001 0 0 +9.46411 0.001 0 0 0 0.001 0 0 +9.46655 0.001 0 0 0 0.001 0 0 +9.46899 0.001 0 0 0 0.001 0 0 +9.47144 0.001 0 0 0 0.001 0 0 +9.47388 0.001 0 0 0 0.001 0 0 +9.47632 0.001 0 0 0 0.001 0 0 +9.47876 0.001 0 0 0 0.001 0 0 +9.4812 0.001 0 0 0 0.001 0 0 +9.48364 0.001 0 0 0 0.001 0 0 +9.48608 0.001 0 0 0 0.001 0 0 +9.48853 0.001 0 0 0 0.001 0 0 +9.49097 0.001 0 0 0 0.001 0 0 +9.49341 0.001 0 0 0 0.001 0 0 +9.49585 0.001 0 0 0 0.001 0 0 +9.49829 0.001 0 0 0 0.001 0 0 +9.50073 0.001 0 0 0 0.001 0 0 +9.50317 0.001 0 0 0 0.001 0 0 +9.50562 0.001 0 0 0 0.001 0 0 +9.50806 0.001 0 0 0 0.001 0 0 +9.5105 0.001 0 0 0 0.001 0 0 +9.51294 0.001 0 0 0 0.001 0 0 +9.51538 0.001 0 0 0 0.001 0 0 +9.51782 0.001 0 0 0 0.001 0 0 +9.52026 0.001 0 0 0 0.001 0 0 +9.52271 0.001 0 0 0 0.001 0 0 +9.52515 0.001 0 0 0 0.001 0 0 +9.52759 0.001 0 0 0 0.001 0 0 +9.53003 0.001 0 0 0 0.001 0 0 +9.53247 0.001 0 0 0 0.001 0 0 +9.53491 0.001 0 0 0 0.001 0 0 +9.53735 0.001 0 0 0 0.001 0 0 +9.53979 0.001 0 0 0 0.001 0 0 +9.54224 0.001 0 0 0 0.001 0 0 +9.54468 0.001 0 0 0 0.001 0 0 +9.54712 0.001 0 0 0 0.001 0 0 +9.54956 0.001 0 0 0 0.001 0 0 +9.552 0.001 0 0 0 0.001 0 0 +9.55444 0.001 0 0 0 0.001 0 0 +9.55688 0.001 0 0 0 0.001 0 0 +9.55933 0.001 0 0 0 0.001 0 0 +9.56177 0.001 0 0 0 0.001 0 0 +9.56421 0.001 0 0 0 0.001 0 0 +9.56665 0.001 0 0 0 0.001 0 0 +9.56909 0.001 0 0 0 0.001 0 0 +9.57153 0.001 0 0 0 0.001 0 0 +9.57397 0.001 0 0 0 0.001 0 0 +9.57642 0.001 0 0 0 0.001 0 0 +9.57886 0.001 0 0 0 0.001 0 0 +9.5813 0.001 0 0 0 0.001 0 0 +9.58374 0.001 0 0 0 0.001 0 0 +9.58618 0.001 0 0 0 0.001 0 0 +9.58862 0.001 0 0 0 0.001 0 0 +9.59106 0.001 0 0 0 0.001 0 0 +9.59351 0.001 0 0 0 0.001 0 0 +9.59595 0.001 0 0 0 0.001 0 0 +9.59839 0.001 0 0 0 0.001 0 0 +9.60083 0.001 0 0 0 0.001 0 0 +9.60327 0.001 0 0 0 0.001 0 0 +9.60571 0.001 0 0 0 0.001 0 0 +9.60815 0.001 0 0 0 0.001 0 0 +9.6106 0.001 0 0 0 0.001 0 0 +9.61304 0.001 0 0 0 0.001 0 0 +9.61548 0.001 0 0 0 0.001 0 0 +9.61792 0.001 0 0 0 0.001 0 0 +9.62036 0.001 0 0 0 0.001 0 0 +9.6228 0.001 0 0 0 0.001 0 0 +9.62524 0.001 0 0 0 0.001 0 0 +9.62769 0.001 0 0 0 0.001 0 0 +9.63013 0.001 0 0 0 0.001 0 0 +9.63257 0.001 0 0 0 0.001 0 0 +9.63501 0.001 0 0 0 0.001 0 0 +9.63745 0.001 0 0 0 0.001 0 0 +9.63989 0.001 0 0 0 0.001 0 0 +9.64233 0.001 0 0 0 0.001 0 0 +9.64478 0.001 0 0 0 0.001 0 0 +9.64722 0.001 0 0 0 0.001 0 0 +9.64966 0.001 0 0 0 0.001 0 0 +9.6521 0.001 0 0 0 0.001 0 0 +9.65454 0.001 0 0 0 0.001 0 0 +9.65698 0.001 0 0 0 0.001 0 0 +9.65942 0.001 0 0 0 0.001 0 0 +9.66187 0.001 0 0 0 0.001 0 0 +9.66431 0.001 0 0 0 0.001 0 0 +9.66675 0.001 0 0 0 0.001 0 0 +9.66919 0.001 0 0 0 0.001 0 0 +9.67163 0.001 0 0 0 0.001 0 0 +9.67407 0.001 0 0 0 0.001 0 0 +9.67651 0.001 0 0 0 0.001 0 0 +9.67896 0.001 0 0 0 0.001 0 0 +9.6814 0.001 0 0 0 0.001 0 0 +9.68384 0.001 0 0 0 0.001 0 0 +9.68628 0.001 0 0 0 0.001 0 0 +9.68872 0.001 0 0 0 0.001 0 0 +9.69116 0.001 0 0 0 0.001 0 0 +9.6936 0.001 0 0 0 0.001 0 0 +9.69604 0.001 0 0 0 0.001 0 0 +9.69849 0.001 0 0 0 0.001 0 0 +9.70093 0.001 0 0 0 0.001 0 0 +9.70337 0.001 0 0 0 0.001 0 0 +9.70581 0.001 0 0 0 0.001 0 0 +9.70825 0.001 0 0 0 0.001 0 0 +9.71069 0.001 0 0 0 0.001 0 0 +9.71313 0.001 0 0 0 0.001 0 0 +9.71558 0.001 0 0 0 0.001 0 0 +9.71802 0.001 0 0 0 0.001 0 0 +9.72046 0.001 0 0 0 0.001 0 0 +9.7229 0.001 0 0 0 0.001 0 0 +9.72534 0.001 0 0 0 0.001 0 0 +9.72778 0.001 0 0 0 0.001 0 0 +9.73022 0.001 0 0 0 0.001 0 0 +9.73267 0.001 0 0 0 0.001 0 0 +9.73511 0.001 0 0 0 0.001 0 0 +9.73755 0.001 0 0 0 0.001 0 0 +9.73999 0.001 0 0 0 0.001 0 0 +9.74243 0.001 0 0 0 0.001 0 0 +9.74487 0.001 0 0 0 0.001 0 0 +9.74731 0.001 0 0 0 0.001 0 0 +9.74976 0.001 0 0 0 0.001 0 0 +9.7522 0.001 0 0 0 0.001 0 0 +9.75464 0.001 0 0 0 0.001 0 0 +9.75708 0.001 0 0 0 0.001 0 0 +9.75952 0.001 0 0 0 0.001 0 0 +9.76196 0.001 0 0 0 0.001 0 0 +9.7644 0.001 0 0 0 0.001 0 0 +9.76685 0.001 0 0 0 0.001 0 0 +9.76929 0.001 0 0 0 0.001 0 0 +9.77173 0.001 0 0 0 0.001 0 0 +9.77417 0.001 0 0 0 0.001 0 0 +9.77661 0.001 0 0 0 0.001 0 0 +9.77905 0.001 0 0 0 0.001 0 0 +9.78149 0.001 0 0 0 0.001 0 0 +9.78394 0.001 0 0 0 0.001 0 0 +9.78638 0.001 0 0 0 0.001 0 0 +9.78882 0.001 0 0 0 0.001 0 0 +9.79126 0.001 0 0 0 0.001 0 0 +9.7937 0.001 0 0 0 0.001 0 0 +9.79614 0.001 0 0 0 0.001 0 0 +9.79858 0.001 0 0 0 0.001 0 0 +9.80103 0.001 0 0 0 0.001 0 0 +9.80347 0.001 0 0 0 0.001 0 0 +9.80591 0.001 0 0 0 0.001 0 0 +9.80835 0.001 0 0 0 0.001 0 0 +9.81079 0.001 0 0 0 0.001 0 0 +9.81323 0.001 0 0 0 0.001 0 0 +9.81567 0.001 0 0 0 0.001 0 0 +9.81812 0.001 0 0 0 0.001 0 0 +9.82056 0.001 0 0 0 0.001 0 0 +9.823 0.001 0 0 0 0.001 0 0 +9.82544 0.001 0 0 0 0.001 0 0 +9.82788 0.001 0 0 0 0.001 0 0 +9.83032 0.001 0 0 0 0.001 0 0 +9.83276 0.001 0 0 0 0.001 0 0 +9.83521 0.001 0 0 0 0.001 0 0 +9.83765 0.001 0 0 0 0.001 0 0 +9.84009 0.001 0 0 0 0.001 0 0 +9.84253 0.001 0 0 0 0.001 0 0 +9.84497 0.001 0 0 0 0.001 0 0 +9.84741 0.001 0 0 0 0.001 0 0 +9.84985 0.001 0 0 0 0.001 0 0 +9.85229 0.001 0 0 0 0.001 0 0 +9.85474 0.001 0 0 0 0.001 0 0 +9.85718 0.001 0 0 0 0.001 0 0 +9.85962 0.001 0 0 0 0.001 0 0 +9.86206 0.001 0 0 0 0.001 0 0 +9.8645 0.001 0 0 0 0.001 0 0 +9.86694 0.001 0 0 0 0.001 0 0 +9.86938 0.001 0 0 0 0.001 0 0 +9.87183 0.001 0 0 0 0.001 0 0 +9.87427 0.001 0 0 0 0.001 0 0 +9.87671 0.001 0 0 0 0.001 0 0 +9.87915 0.001 0 0 0 0.001 0 0 +9.88159 0.001 0 0 0 0.001 0 0 +9.88403 0.001 0 0 0 0.001 0 0 +9.88647 0.001 0 0 0 0.001 0 0 +9.88892 0.001 0 0 0 0.001 0 0 +9.89136 0.001 0 0 0 0.001 0 0 +9.8938 0.001 0 0 0 0.001 0 0 +9.89624 0.001 0 0 0 0.001 0 0 +9.89868 0.001 0 0 0 0.001 0 0 +9.90112 0.001 0 0 0 0.001 0 0 +9.90356 0.001 0 0 0 0.001 0 0 +9.90601 0.001 0 0 0 0.001 0 0 +9.90845 0.001 0 0 0 0.001 0 0 +9.91089 0.001 0 0 0 0.001 0 0 +9.91333 0.001 0 0 0 0.001 0 0 +9.91577 0.001 0 0 0 0.001 0 0 +9.91821 0.001 0 0 0 0.001 0 0 +9.92065 0.001 0 0 0 0.001 0 0 +9.9231 0.001 0 0 0 0.001 0 0 +9.92554 0.001 0 0 0 0.001 0 0 +9.92798 0.001 0 0 0 0.001 0 0 +9.93042 0.001 0 0 0 0.001 0 0 +9.93286 0.001 0 0 0 0.001 0 0 +9.9353 0.001 0 0 0 0.001 0 0 +9.93774 0.001 0 0 0 0.001 0 0 +9.94019 0.001 0 0 0 0.001 0 0 +9.94263 0.001 0 0 0 0.001 0 0 +9.94507 0.001 0 0 0 0.001 0 0 +9.94751 0.001 0 0 0 0.001 0 0 +9.94995 0.001 0 0 0 0.001 0 0 +9.95239 0.001 0 0 0 0.001 0 0 +9.95483 0.001 0 0 0 0.001 0 0 +9.95728 0.001 0 0 0 0.001 0 0 +9.95972 0.001 0 0 0 0.001 0 0 +9.96216 0.001 0 0 0 0.001 0 0 +9.9646 0.001 0 0 0 0.001 0 0 +9.96704 0.001 0 0 0 0.001 0 0 +9.96948 0.001 0 0 0 0.001 0 0 +9.97192 0.001 0 0 0 0.001 0 0 +9.97437 0.001 0 0 0 0.001 0 0 +9.97681 0.001 0 0 0 0.001 0 0 +9.97925 0.001 0 0 0 0.001 0 0 +9.98169 0.001 0 0 0 0.001 0 0 +9.98413 0.001 0 0 0 0.001 0 0 +9.98657 0.001 0 0 0 0.001 0 0 +9.98901 0.001 0 0 0 0.001 0 0 +9.99146 0.001 0 0 0 0.001 0 0 +9.9939 0.001 0 0 0 0.001 0 0 +9.99634 0.001 0 0 0 0.001 0 0 +9.99878 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=512.csv b/reference/swashes_1_nx=512.csv new file mode 100644 index 0000000..0a2916f --- /dev/null +++ b/reference/swashes_1_nx=512.csv @@ -0,0 +1,530 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.0195312 meters +# Number of cells: 512 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00976562 0.005 0 0 0 0.005 0 0 +0.0292969 0.005 0 0 0 0.005 0 0 +0.0488281 0.005 0 0 0 0.005 0 0 +0.0683594 0.005 0 0 0 0.005 0 0 +0.0878906 0.005 0 0 0 0.005 0 0 +0.107422 0.005 0 0 0 0.005 0 0 +0.126953 0.005 0 0 0 0.005 0 0 +0.146484 0.005 0 0 0 0.005 0 0 +0.166016 0.005 0 0 0 0.005 0 0 +0.185547 0.005 0 0 0 0.005 0 0 +0.205078 0.005 0 0 0 0.005 0 0 +0.224609 0.005 0 0 0 0.005 0 0 +0.244141 0.005 0 0 0 0.005 0 0 +0.263672 0.005 0 0 0 0.005 0 0 +0.283203 0.005 0 0 0 0.005 0 0 +0.302734 0.005 0 0 0 0.005 0 0 +0.322266 0.005 0 0 0 0.005 0 0 +0.341797 0.005 0 0 0 0.005 0 0 +0.361328 0.005 0 0 0 0.005 0 0 +0.380859 0.005 0 0 0 0.005 0 0 +0.400391 0.005 0 0 0 0.005 0 0 +0.419922 0.005 0 0 0 0.005 0 0 +0.439453 0.005 0 0 0 0.005 0 0 +0.458984 0.005 0 0 0 0.005 0 0 +0.478516 0.005 0 0 0 0.005 0 0 +0.498047 0.005 0 0 0 0.005 0 0 +0.517578 0.005 0 0 0 0.005 0 0 +0.537109 0.005 0 0 0 0.005 0 0 +0.556641 0.005 0 0 0 0.005 0 0 +0.576172 0.005 0 0 0 0.005 0 0 +0.595703 0.005 0 0 0 0.005 0 0 +0.615234 0.005 0 0 0 0.005 0 0 +0.634766 0.005 0 0 0 0.005 0 0 +0.654297 0.005 0 0 0 0.005 0 0 +0.673828 0.005 0 0 0 0.005 0 0 +0.693359 0.005 0 0 0 0.005 0 0 +0.712891 0.005 0 0 0 0.005 0 0 +0.732422 0.005 0 0 0 0.005 0 0 +0.751953 0.005 0 0 0 0.005 0 0 +0.771484 0.005 0 0 0 0.005 0 0 +0.791016 0.005 0 0 0 0.005 0 0 +0.810547 0.005 0 0 0 0.005 0 0 +0.830078 0.005 0 0 0 0.005 0 0 +0.849609 0.005 0 0 0 0.005 0 0 +0.869141 0.005 0 0 0 0.005 0 0 +0.888672 0.005 0 0 0 0.005 0 0 +0.908203 0.005 0 0 0 0.005 0 0 +0.927734 0.005 0 0 0 0.005 0 0 +0.947266 0.005 0 0 0 0.005 0 0 +0.966797 0.005 0 0 0 0.005 0 0 +0.986328 0.005 0 0 0 0.005 0 0 +1.00586 0.005 0 0 0 0.005 0 0 +1.02539 0.005 0 0 0 0.005 0 0 +1.04492 0.005 0 0 0 0.005 0 0 +1.06445 0.005 0 0 0 0.005 0 0 +1.08398 0.005 0 0 0 0.005 0 0 +1.10352 0.005 0 0 0 0.005 0 0 +1.12305 0.005 0 0 0 0.005 0 0 +1.14258 0.005 0 0 0 0.005 0 0 +1.16211 0.005 0 0 0 0.005 0 0 +1.18164 0.005 0 0 0 0.005 0 0 +1.20117 0.005 0 0 0 0.005 0 0 +1.2207 0.005 0 0 0 0.005 0 0 +1.24023 0.005 0 0 0 0.005 0 0 +1.25977 0.005 0 0 0 0.005 0 0 +1.2793 0.005 0 0 0 0.005 0 0 +1.29883 0.005 0 0 0 0.005 0 0 +1.31836 0.005 0 0 0 0.005 0 0 +1.33789 0.005 0 0 0 0.005 0 0 +1.35742 0.005 0 0 0 0.005 0 0 +1.37695 0.005 0 0 0 0.005 0 0 +1.39648 0.005 0 0 0 0.005 0 0 +1.41602 0.005 0 0 0 0.005 0 0 +1.43555 0.005 0 0 0 0.005 0 0 +1.45508 0.005 0 0 0 0.005 0 0 +1.47461 0.005 0 0 0 0.005 0 0 +1.49414 0.005 0 0 0 0.005 0 0 +1.51367 0.005 0 0 0 0.005 0 0 +1.5332 0.005 0 0 0 0.005 0 0 +1.55273 0.005 0 0 0 0.005 0 0 +1.57227 0.005 0 0 0 0.005 0 0 +1.5918 0.005 0 0 0 0.005 0 0 +1.61133 0.005 0 0 0 0.005 0 0 +1.63086 0.005 0 0 0 0.005 0 0 +1.65039 0.005 0 0 0 0.005 0 0 +1.66992 0.005 0 0 0 0.005 0 0 +1.68945 0.005 0 0 0 0.005 0 0 +1.70898 0.005 0 0 0 0.005 0 0 +1.72852 0.005 0 0 0 0.005 0 0 +1.74805 0.005 0 0 0 0.005 0 0 +1.76758 0.005 0 0 0 0.005 0 0 +1.78711 0.005 0 0 0 0.005 0 0 +1.80664 0.005 0 0 0 0.005 0 0 +1.82617 0.005 0 0 0 0.005 0 0 +1.8457 0.005 0 0 0 0.005 0 0 +1.86523 0.005 0 0 0 0.005 0 0 +1.88477 0.005 0 0 0 0.005 0 0 +1.9043 0.005 0 0 0 0.005 0 0 +1.92383 0.005 0 0 0 0.005 0 0 +1.94336 0.005 0 0 0 0.005 0 0 +1.96289 0.005 0 0 0 0.005 0 0 +1.98242 0.005 0 0 0 0.005 0 0 +2.00195 0.005 0 0 0 0.005 0 0 +2.02148 0.005 0 0 0 0.005 0 0 +2.04102 0.005 0 0 0 0.005 0 0 +2.06055 0.005 0 0 0 0.005 0 0 +2.08008 0.005 0 0 0 0.005 0 0 +2.09961 0.005 0 0 0 0.005 0 0 +2.11914 0.005 0 0 0 0.005 0 0 +2.13867 0.005 0 0 0 0.005 0 0 +2.1582 0.005 0 0 0 0.005 0 0 +2.17773 0.005 0 0 0 0.005 0 0 +2.19727 0.005 0 0 0 0.005 0 0 +2.2168 0.005 0 0 0 0.005 0 0 +2.23633 0.005 0 0 0 0.005 0 0 +2.25586 0.005 0 0 0 0.005 0 0 +2.27539 0.005 0 0 0 0.005 0 0 +2.29492 0.005 0 0 0 0.005 0 0 +2.31445 0.005 0 0 0 0.005 0 0 +2.33398 0.005 0 0 0 0.005 0 0 +2.35352 0.005 0 0 0 0.005 0 0 +2.37305 0.005 0 0 0 0.005 0 0 +2.39258 0.005 0 0 0 0.005 0 0 +2.41211 0.005 0 0 0 0.005 0 0 +2.43164 0.005 0 0 0 0.005 0 0 +2.45117 0.005 0 0 0 0.005 0 0 +2.4707 0.005 0 0 0 0.005 0 0 +2.49023 0.005 0 0 0 0.005 0 0 +2.50977 0.005 0 0 0 0.005 0 0 +2.5293 0.005 0 0 0 0.005 0 0 +2.54883 0.005 0 0 0 0.005 0 0 +2.56836 0.005 0 0 0 0.005 0 0 +2.58789 0.005 0 0 0 0.005 0 0 +2.60742 0.005 0 0 0 0.005 0 0 +2.62695 0.005 0 0 0 0.005 0 0 +2.64648 0.005 0 0 0 0.005 0 0 +2.66602 0.005 0 0 0 0.005 0 0 +2.68555 0.005 0 0 0 0.005 0 0 +2.70508 0.005 0 0 0 0.005 0 0 +2.72461 0.005 0 0 0 0.005 0 0 +2.74414 0.005 0 0 0 0.005 0 0 +2.76367 0.005 0 0 0 0.005 0 0 +2.7832 0.005 0 0 0 0.005 0 0 +2.80273 0.005 0 0 0 0.005 0 0 +2.82227 0.005 0 0 0 0.005 0 0 +2.8418 0.005 0 0 0 0.005 0 0 +2.86133 0.005 0 0 0 0.005 0 0 +2.88086 0.005 0 0 0 0.005 0 0 +2.90039 0.005 0 0 0 0.005 0 0 +2.91992 0.005 0 0 0 0.005 0 0 +2.93945 0.005 0 0 0 0.005 0 0 +2.95898 0.005 0 0 0 0.005 0 0 +2.97852 0.005 0 0 0 0.005 0 0 +2.99805 0.005 0 0 0 0.005 0 0 +3.01758 0.005 0 0 0 0.005 0 0 +3.03711 0.005 0 0 0 0.005 0 0 +3.05664 0.005 0 0 0 0.005 0 0 +3.07617 0.005 0 0 0 0.005 0 0 +3.0957 0.005 0 0 0 0.005 0 0 +3.11523 0.005 0 0 0 0.005 0 0 +3.13477 0.005 0 0 0 0.005 0 0 +3.1543 0.005 0 0 0 0.005 0 0 +3.17383 0.005 0 0 0 0.005 0 0 +3.19336 0.005 0 0 0 0.005 0 0 +3.21289 0.005 0 0 0 0.005 0 0 +3.23242 0.005 0 0 0 0.005 0 0 +3.25195 0.005 0 0 0 0.005 0 0 +3.27148 0.005 0 0 0 0.005 0 0 +3.29102 0.005 0 0 0 0.005 0 0 +3.31055 0.005 0 0 0 0.005 0 0 +3.33008 0.005 0 0 0 0.005 0 0 +3.34961 0.005 0 0 0 0.005 0 0 +3.36914 0.005 0 0 0 0.005 0 0 +3.38867 0.005 0 0 0 0.005 0 0 +3.4082 0.005 0 0 0 0.005 0 0 +3.42773 0.005 0 0 0 0.005 0 0 +3.44727 0.005 0 0 0 0.005 0 0 +3.4668 0.005 0 0 0 0.005 0 0 +3.48633 0.005 0 0 0 0.005 0 0 +3.50586 0.005 0 0 0 0.005 0 0 +3.52539 0.005 0 0 0 0.005 0 0 +3.54492 0.005 0 0 0 0.005 0 0 +3.56445 0.005 0 0 0 0.005 0 0 +3.58398 0.005 0 0 0 0.005 0 0 +3.60352 0.005 0 0 0 0.005 0 0 +3.62305 0.005 0 0 0 0.005 0 0 +3.64258 0.005 0 0 0 0.005 0 0 +3.66211 0.005 0 0 0 0.005 0 0 +3.68164 0.00497376 0.00116386 0 5.78874e-006 0.00497376 0.00526893 0.000150603 +3.70117 0.00492501 0.00333399 0 1.642e-005 0.00492501 0.0151679 0.000301781 +3.7207 0.00487651 0.00550413 0 2.6841e-005 0.00487651 0.0251652 0.00041877 +3.74023 0.00482825 0.00767427 0 3.70533e-005 0.00482825 0.0352621 0.000519192 +3.75977 0.00478022 0.00984441 0 4.70585e-005 0.00478022 0.0454602 0.000608885 +3.7793 0.00473244 0.0120146 0 5.68581e-005 0.00473244 0.055761 0.000690725 +3.79883 0.00468489 0.0141847 0 6.64537e-005 0.00468489 0.0661661 0.000766402 +3.81836 0.00463759 0.0163548 0 7.58469e-005 0.00463759 0.0766771 0.00083702 +3.83789 0.00459052 0.018525 0 8.50393e-005 0.00459052 0.0872955 0.000903351 +3.85742 0.0045437 0.0206951 0 9.40323e-005 0.0045437 0.0980231 0.000965966 +3.87695 0.00449711 0.0228652 0 0.000102828 0.00449711 0.108862 0.0010253 +3.89648 0.00445077 0.0250354 0 0.000111427 0.00445077 0.119813 0.00108169 +3.91602 0.00440467 0.0272055 0 0.000119831 0.00440467 0.130878 0.00113542 +3.93555 0.0043588 0.0293757 0 0.000128043 0.0043588 0.142059 0.00118672 +3.95508 0.00431318 0.0315458 0 0.000136063 0.00431318 0.153359 0.00123577 +3.97461 0.00426779 0.0337159 0 0.000143893 0.00426779 0.164778 0.00128273 +3.99414 0.00422265 0.0358861 0 0.000151534 0.00422265 0.176319 0.00132775 +4.01367 0.00417774 0.0380562 0 0.000158989 0.00417774 0.187984 0.00137095 +4.0332 0.00413308 0.0402264 0 0.000166259 0.00413308 0.199774 0.00141243 +4.05273 0.00408866 0.0423965 0 0.000173345 0.00408866 0.211692 0.00145228 +4.07227 0.00404447 0.0445666 0 0.000180248 0.00404447 0.22374 0.00149059 +4.0918 0.00400053 0.0467368 0 0.000186972 0.00400053 0.23592 0.00152743 +4.11133 0.00395682 0.0489069 0 0.000193516 0.00395682 0.248235 0.00156287 +4.13086 0.00391336 0.0510771 0 0.000199883 0.00391336 0.260685 0.00159696 +4.15039 0.00387014 0.0532472 0 0.000206074 0.00387014 0.273274 0.00162977 +4.16992 0.00382715 0.0554173 0 0.000212091 0.00382715 0.286005 0.00166134 +4.18945 0.00378441 0.0575875 0 0.000217935 0.00378441 0.298878 0.00169172 +4.20898 0.0037419 0.0597576 0 0.000223607 0.0037419 0.311898 0.00172095 +4.22852 0.00369964 0.0619277 0 0.00022911 0.00369964 0.325066 0.00174907 +4.24805 0.00365762 0.0640979 0 0.000234446 0.00365762 0.338384 0.00177612 +4.26758 0.00361583 0.066268 0 0.000239614 0.00361583 0.351856 0.00180213 +4.28711 0.00357429 0.0684382 0 0.000244618 0.00357429 0.365484 0.00182713 +4.30664 0.00353299 0.0706083 0 0.000249458 0.00353299 0.379272 0.00185115 +4.32617 0.00349192 0.0727784 0 0.000254137 0.00349192 0.39322 0.00187423 +4.3457 0.0034511 0.0749486 0 0.000258655 0.0034511 0.407334 0.00189638 +4.36523 0.00341052 0.0771187 0 0.000263015 0.00341052 0.421614 0.00191762 +4.38477 0.00337017 0.0792889 0 0.000267217 0.00337017 0.436065 0.001938 +4.4043 0.00333007 0.081459 0 0.000271264 0.00333007 0.45069 0.00195752 +4.42383 0.00329021 0.0836291 0 0.000275157 0.00329021 0.465491 0.0019762 +4.44336 0.00325058 0.0857993 0 0.000278898 0.00325058 0.480472 0.00199407 +4.46289 0.0032112 0.0879694 0 0.000282487 0.0032112 0.495637 0.00201114 +4.48242 0.00317206 0.0901396 0 0.000285928 0.00317206 0.510988 0.00202744 +4.50195 0.00313315 0.0923097 0 0.00028922 0.00313315 0.526529 0.00204297 +4.52148 0.00309449 0.0944798 0 0.000292367 0.00309449 0.542263 0.00205776 +4.54102 0.00305607 0.09665 0 0.000295369 0.00305607 0.558195 0.00207183 +4.56055 0.00301788 0.0988201 0 0.000298228 0.00301788 0.574327 0.00208517 +4.58008 0.00297994 0.10099 0 0.000300945 0.00297994 0.590665 0.00209782 +4.59961 0.00294224 0.10316 0 0.000303522 0.00294224 0.607211 0.00210978 +4.61914 0.00290477 0.105331 0 0.000305961 0.00290477 0.62397 0.00212107 +4.63867 0.00286755 0.107501 0 0.000308264 0.00286755 0.640945 0.0021317 +4.6582 0.00283057 0.109671 0 0.000310431 0.00283057 0.658142 0.00214167 +4.67773 0.00279383 0.111841 0 0.000312464 0.00279383 0.675564 0.00215102 +4.69727 0.00275732 0.114011 0 0.000314365 0.00275732 0.693216 0.00215973 +4.7168 0.00272106 0.116181 0 0.000316136 0.00272106 0.711103 0.00216784 +4.73633 0.00268504 0.118351 0 0.000317778 0.00268504 0.729228 0.00217533 +4.75586 0.00264925 0.120521 0 0.000319292 0.00264925 0.747598 0.00218224 +4.77539 0.00261371 0.122692 0 0.00032068 0.00261371 0.766217 0.00218856 +4.79492 0.00257841 0.124862 0 0.000321945 0.00257841 0.785089 0.00219431 +4.81445 0.00254335 0.127032 0 0.000323086 0.00254335 0.804221 0.00219949 +4.83398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04883 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06836 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08789 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10742 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12695 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14648 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16602 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18555 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20508 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22461 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24414 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26367 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30273 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32227 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36133 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38086 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40039 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41992 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43945 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45898 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47852 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49805 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51758 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53711 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55664 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57617 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61523 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63477 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67383 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69336 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71289 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73242 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75195 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77148 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79102 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81055 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83008 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84961 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86914 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88867 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92773 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94727 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98633 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00586 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02539 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04492 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06445 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2793 0.001 0 0 0 0.001 0 0 +6.29883 0.001 0 0 0 0.001 0 0 +6.31836 0.001 0 0 0 0.001 0 0 +6.33789 0.001 0 0 0 0.001 0 0 +6.35742 0.001 0 0 0 0.001 0 0 +6.37695 0.001 0 0 0 0.001 0 0 +6.39648 0.001 0 0 0 0.001 0 0 +6.41602 0.001 0 0 0 0.001 0 0 +6.43555 0.001 0 0 0 0.001 0 0 +6.45508 0.001 0 0 0 0.001 0 0 +6.47461 0.001 0 0 0 0.001 0 0 +6.49414 0.001 0 0 0 0.001 0 0 +6.51367 0.001 0 0 0 0.001 0 0 +6.5332 0.001 0 0 0 0.001 0 0 +6.55273 0.001 0 0 0 0.001 0 0 +6.57227 0.001 0 0 0 0.001 0 0 +6.5918 0.001 0 0 0 0.001 0 0 +6.61133 0.001 0 0 0 0.001 0 0 +6.63086 0.001 0 0 0 0.001 0 0 +6.65039 0.001 0 0 0 0.001 0 0 +6.66992 0.001 0 0 0 0.001 0 0 +6.68945 0.001 0 0 0 0.001 0 0 +6.70898 0.001 0 0 0 0.001 0 0 +6.72852 0.001 0 0 0 0.001 0 0 +6.74805 0.001 0 0 0 0.001 0 0 +6.76758 0.001 0 0 0 0.001 0 0 +6.78711 0.001 0 0 0 0.001 0 0 +6.80664 0.001 0 0 0 0.001 0 0 +6.82617 0.001 0 0 0 0.001 0 0 +6.8457 0.001 0 0 0 0.001 0 0 +6.86523 0.001 0 0 0 0.001 0 0 +6.88477 0.001 0 0 0 0.001 0 0 +6.9043 0.001 0 0 0 0.001 0 0 +6.92383 0.001 0 0 0 0.001 0 0 +6.94336 0.001 0 0 0 0.001 0 0 +6.96289 0.001 0 0 0 0.001 0 0 +6.98242 0.001 0 0 0 0.001 0 0 +7.00195 0.001 0 0 0 0.001 0 0 +7.02148 0.001 0 0 0 0.001 0 0 +7.04102 0.001 0 0 0 0.001 0 0 +7.06055 0.001 0 0 0 0.001 0 0 +7.08008 0.001 0 0 0 0.001 0 0 +7.09961 0.001 0 0 0 0.001 0 0 +7.11914 0.001 0 0 0 0.001 0 0 +7.13867 0.001 0 0 0 0.001 0 0 +7.1582 0.001 0 0 0 0.001 0 0 +7.17773 0.001 0 0 0 0.001 0 0 +7.19727 0.001 0 0 0 0.001 0 0 +7.2168 0.001 0 0 0 0.001 0 0 +7.23633 0.001 0 0 0 0.001 0 0 +7.25586 0.001 0 0 0 0.001 0 0 +7.27539 0.001 0 0 0 0.001 0 0 +7.29492 0.001 0 0 0 0.001 0 0 +7.31445 0.001 0 0 0 0.001 0 0 +7.33398 0.001 0 0 0 0.001 0 0 +7.35352 0.001 0 0 0 0.001 0 0 +7.37305 0.001 0 0 0 0.001 0 0 +7.39258 0.001 0 0 0 0.001 0 0 +7.41211 0.001 0 0 0 0.001 0 0 +7.43164 0.001 0 0 0 0.001 0 0 +7.45117 0.001 0 0 0 0.001 0 0 +7.4707 0.001 0 0 0 0.001 0 0 +7.49023 0.001 0 0 0 0.001 0 0 +7.50977 0.001 0 0 0 0.001 0 0 +7.5293 0.001 0 0 0 0.001 0 0 +7.54883 0.001 0 0 0 0.001 0 0 +7.56836 0.001 0 0 0 0.001 0 0 +7.58789 0.001 0 0 0 0.001 0 0 +7.60742 0.001 0 0 0 0.001 0 0 +7.62695 0.001 0 0 0 0.001 0 0 +7.64648 0.001 0 0 0 0.001 0 0 +7.66602 0.001 0 0 0 0.001 0 0 +7.68555 0.001 0 0 0 0.001 0 0 +7.70508 0.001 0 0 0 0.001 0 0 +7.72461 0.001 0 0 0 0.001 0 0 +7.74414 0.001 0 0 0 0.001 0 0 +7.76367 0.001 0 0 0 0.001 0 0 +7.7832 0.001 0 0 0 0.001 0 0 +7.80273 0.001 0 0 0 0.001 0 0 +7.82227 0.001 0 0 0 0.001 0 0 +7.8418 0.001 0 0 0 0.001 0 0 +7.86133 0.001 0 0 0 0.001 0 0 +7.88086 0.001 0 0 0 0.001 0 0 +7.90039 0.001 0 0 0 0.001 0 0 +7.91992 0.001 0 0 0 0.001 0 0 +7.93945 0.001 0 0 0 0.001 0 0 +7.95898 0.001 0 0 0 0.001 0 0 +7.97852 0.001 0 0 0 0.001 0 0 +7.99805 0.001 0 0 0 0.001 0 0 +8.01758 0.001 0 0 0 0.001 0 0 +8.03711 0.001 0 0 0 0.001 0 0 +8.05664 0.001 0 0 0 0.001 0 0 +8.07617 0.001 0 0 0 0.001 0 0 +8.0957 0.001 0 0 0 0.001 0 0 +8.11523 0.001 0 0 0 0.001 0 0 +8.13477 0.001 0 0 0 0.001 0 0 +8.1543 0.001 0 0 0 0.001 0 0 +8.17383 0.001 0 0 0 0.001 0 0 +8.19336 0.001 0 0 0 0.001 0 0 +8.21289 0.001 0 0 0 0.001 0 0 +8.23242 0.001 0 0 0 0.001 0 0 +8.25195 0.001 0 0 0 0.001 0 0 +8.27148 0.001 0 0 0 0.001 0 0 +8.29102 0.001 0 0 0 0.001 0 0 +8.31055 0.001 0 0 0 0.001 0 0 +8.33008 0.001 0 0 0 0.001 0 0 +8.34961 0.001 0 0 0 0.001 0 0 +8.36914 0.001 0 0 0 0.001 0 0 +8.38867 0.001 0 0 0 0.001 0 0 +8.4082 0.001 0 0 0 0.001 0 0 +8.42773 0.001 0 0 0 0.001 0 0 +8.44727 0.001 0 0 0 0.001 0 0 +8.4668 0.001 0 0 0 0.001 0 0 +8.48633 0.001 0 0 0 0.001 0 0 +8.50586 0.001 0 0 0 0.001 0 0 +8.52539 0.001 0 0 0 0.001 0 0 +8.54492 0.001 0 0 0 0.001 0 0 +8.56445 0.001 0 0 0 0.001 0 0 +8.58398 0.001 0 0 0 0.001 0 0 +8.60352 0.001 0 0 0 0.001 0 0 +8.62305 0.001 0 0 0 0.001 0 0 +8.64258 0.001 0 0 0 0.001 0 0 +8.66211 0.001 0 0 0 0.001 0 0 +8.68164 0.001 0 0 0 0.001 0 0 +8.70117 0.001 0 0 0 0.001 0 0 +8.7207 0.001 0 0 0 0.001 0 0 +8.74023 0.001 0 0 0 0.001 0 0 +8.75977 0.001 0 0 0 0.001 0 0 +8.7793 0.001 0 0 0 0.001 0 0 +8.79883 0.001 0 0 0 0.001 0 0 +8.81836 0.001 0 0 0 0.001 0 0 +8.83789 0.001 0 0 0 0.001 0 0 +8.85742 0.001 0 0 0 0.001 0 0 +8.87695 0.001 0 0 0 0.001 0 0 +8.89648 0.001 0 0 0 0.001 0 0 +8.91602 0.001 0 0 0 0.001 0 0 +8.93555 0.001 0 0 0 0.001 0 0 +8.95508 0.001 0 0 0 0.001 0 0 +8.97461 0.001 0 0 0 0.001 0 0 +8.99414 0.001 0 0 0 0.001 0 0 +9.01367 0.001 0 0 0 0.001 0 0 +9.0332 0.001 0 0 0 0.001 0 0 +9.05273 0.001 0 0 0 0.001 0 0 +9.07227 0.001 0 0 0 0.001 0 0 +9.0918 0.001 0 0 0 0.001 0 0 +9.11133 0.001 0 0 0 0.001 0 0 +9.13086 0.001 0 0 0 0.001 0 0 +9.15039 0.001 0 0 0 0.001 0 0 +9.16992 0.001 0 0 0 0.001 0 0 +9.18945 0.001 0 0 0 0.001 0 0 +9.20898 0.001 0 0 0 0.001 0 0 +9.22852 0.001 0 0 0 0.001 0 0 +9.24805 0.001 0 0 0 0.001 0 0 +9.26758 0.001 0 0 0 0.001 0 0 +9.28711 0.001 0 0 0 0.001 0 0 +9.30664 0.001 0 0 0 0.001 0 0 +9.32617 0.001 0 0 0 0.001 0 0 +9.3457 0.001 0 0 0 0.001 0 0 +9.36523 0.001 0 0 0 0.001 0 0 +9.38477 0.001 0 0 0 0.001 0 0 +9.4043 0.001 0 0 0 0.001 0 0 +9.42383 0.001 0 0 0 0.001 0 0 +9.44336 0.001 0 0 0 0.001 0 0 +9.46289 0.001 0 0 0 0.001 0 0 +9.48242 0.001 0 0 0 0.001 0 0 +9.50195 0.001 0 0 0 0.001 0 0 +9.52148 0.001 0 0 0 0.001 0 0 +9.54102 0.001 0 0 0 0.001 0 0 +9.56055 0.001 0 0 0 0.001 0 0 +9.58008 0.001 0 0 0 0.001 0 0 +9.59961 0.001 0 0 0 0.001 0 0 +9.61914 0.001 0 0 0 0.001 0 0 +9.63867 0.001 0 0 0 0.001 0 0 +9.6582 0.001 0 0 0 0.001 0 0 +9.67773 0.001 0 0 0 0.001 0 0 +9.69727 0.001 0 0 0 0.001 0 0 +9.7168 0.001 0 0 0 0.001 0 0 +9.73633 0.001 0 0 0 0.001 0 0 +9.75586 0.001 0 0 0 0.001 0 0 +9.77539 0.001 0 0 0 0.001 0 0 +9.79492 0.001 0 0 0 0.001 0 0 +9.81445 0.001 0 0 0 0.001 0 0 +9.83398 0.001 0 0 0 0.001 0 0 +9.85352 0.001 0 0 0 0.001 0 0 +9.87305 0.001 0 0 0 0.001 0 0 +9.89258 0.001 0 0 0 0.001 0 0 +9.91211 0.001 0 0 0 0.001 0 0 +9.93164 0.001 0 0 0 0.001 0 0 +9.95117 0.001 0 0 0 0.001 0 0 +9.9707 0.001 0 0 0 0.001 0 0 +9.99023 0.001 0 0 0 0.001 0 0 From a166fc5116f36a83ca1a4e9ab5193f1cc38eb1eb Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Fri, 29 Dec 2023 15:02:11 +0100 Subject: [PATCH 07/55] add a job script --- Jobs/job_lumi.slrum | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 Jobs/job_lumi.slrum diff --git a/Jobs/job_lumi.slrum b/Jobs/job_lumi.slrum new file mode 100644 index 0000000..b946bbf --- /dev/null +++ b/Jobs/job_lumi.slrum @@ -0,0 +1,34 @@ +#!/bin/bash -e +#SBATCH --job-name=lumi +#SBATCH --account=project_4650000xx +#SBATCH --time=00:10:00 +#SBATCH --partition=dev-g +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=2 +#SBATCH --gpus=2 +#SBATCH --gpus-per-node=2 +#SBATCH -o %x-%j.out +# + +N=$SLURM_JOB_NUM_NODES +echo "--nbr of nodes:", $N +echo "--total nbr of gpus:", $SLURM_NTASKS + +Mydir=/project/project_4650000xx +Myapplication=${Mydir}/FiniteVolumeGPU_hip/mpiTesting.py + +#modules +ml LUMI/23.03 partition/G +ml lumi-container-wrapper +ml cray-python/3.9.13.1 +ml rocm/5.2.3 + +ml craype-accel-amd-gfx90a +ml cray-mpich/8.1.27 + +#Enable GPU-aware MPI +export MPICH_GPU_SUPPORT_ENABLED=1 + +export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH" + +srun python ${Myapplication} -nx 1024 -ny 1024 --profile From 74b24c33d7793bcb5930e13e101a995f2389780f Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:05:41 +0100 Subject: [PATCH 08/55] Update README.md --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f9bb082..5f68f36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FiniteVolumeGPU -This is a HIP version of the [FiniteVolume code](#https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. +This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. ## Setup A good place to start exploring this codebase is the notebooks. Complete the following steps to run the notebooks: @@ -22,19 +22,24 @@ Have a look at the conda documentation and https://towardsdatascience.com/how-to Here is a step-by-step guide on installing packages on LUMI-G ### Step 0: load modules +``` ml LUMI/23.03 ml lumi-container-wrapper ml cray-python/3.9.13.1 +``` ### Step 1: run conda-container Installation via conda can be done as: - +``` conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml - +``` where the file `conda_environment_lumi.yml` contains packages to be installed. ### Step 2: Set the env. variable to search for binaries +``` export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" - +``` ### An alternative: Convert to a singularity container with cotainr +``` cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml +``` From 28ab54a86e7f78a425d3ab183d1c2fe940ed5800 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Wed, 17 Jan 2024 17:17:48 +0100 Subject: [PATCH 09/55] add hip-based compilation --- GPUSimulators/CudaContext.py | 51 +++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py index b4a2490..e77ef06 100644 --- a/GPUSimulators/CudaContext.py +++ b/GPUSimulators/CudaContext.py @@ -41,13 +41,7 @@ from hip import rccl from GPUSimulators import Autotuner, Common - -""" -Class which keeps track of the CUDA context and some helper functions -""" -class CudaContext(object): - - def hip_check(call_result): +def hip_check(call_result): err = call_result[0] result = call_result[1:] if len(result) == 1: @@ -61,6 +55,11 @@ class CudaContext(object): raise RuntimeError(str(err)) return result +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): """ Create a new CUDA context @@ -245,6 +244,7 @@ class CudaContext(object): with io.open(cached_kernel_filename, "rb") as file: file_str = file.read() + #No hip counterpart of module_from_buffer module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) self.modules[kernel_hash] = module @@ -271,8 +271,41 @@ class CudaContext(object): import warnings with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) - cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) - module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + + #cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + #module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + + #HIP version of compilation: but "name_of_fct" needs to be defined. e.g. + #source = b"""\ + #extern "C" __global__ void name_of_fct(float factor, int n, short unused1, int unused2, float unused3, float *x) { + #int tid = threadIdx.x + blockIdx.x * blockDim.x; + #if (tid < n) { + #x[tid] *= factor; + # } + #} + #""" + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_string, b"name_of_fct", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + #kernel = hip_check(hip.hipModuleGetFunction(module, b"name_of_fct")) + if (self.use_cache): with io.open(cached_kernel_filename, "wb") as file: file.write(cubin) From b90035c902159dfe402ebf8fe7138a406b7c46b7 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Wed, 17 Jan 2024 17:19:07 +0100 Subject: [PATCH 10/55] create time Event with hip --- GPUSimulators/Autotuner.py | 45 +++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/GPUSimulators/Autotuner.py b/GPUSimulators/Autotuner.py index 8dc38e4..84aedc2 100644 --- a/GPUSimulators/Autotuner.py +++ b/GPUSimulators/Autotuner.py @@ -27,11 +27,25 @@ import logging from socket import gethostname #import pycuda.driver as cuda -import pyhip as cuda +from hip import hip,hiprtc from GPUSimulators import Common, Simulator, CudaContext class Autotuner: + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, nx=2048, ny=2048, block_widths=range(8, 32, 1), @@ -184,24 +198,35 @@ class Autotuner: return np.nan #Create timer events - start = cuda.Event() - end = cuda.Event() - + #start = cuda.Event() + #end = cuda.Event() + stream = hip_check(hip.hipStreamCreate()) + + start = hip_check(hip.hipEventCreate()) + end = hip_check(hip.hipEventCreate()) + #Warmup for i in range(warmup_timesteps): sim.stepEuler(sim.dt) #Run simulation with timer - start.record(sim.stream) + #start.record(sim.stream) + #start recording + hip_check(hip.hipEventRecord(start, stream)) for i in range(timesteps): sim.stepEuler(sim.dt) - end.record(sim.stream) - + #end.record(sim.stream) + #stop recording and synchronize + hip_check(hip.hipEventRecord(end, stream)) + #Synchronize end event - end.synchronize() - + #end.synchronize() + hip_check(hip.hipEventSynchronize(end)) + #Compute megacells - gpu_elapsed = end.time_since(start)*1.0e-3 + #gpu_elapsed = end.time_since(start)*1.0e-3 + gpu_elapsed = hip_check(hip.hipEventElapsedTime(start, end)) + megacells = (sim.nx*sim.ny*timesteps / (1000*1000)) / gpu_elapsed #Sanity check solution From 41e496634ee24f2481d195cd5a7aba9775f25938 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Mon, 5 Feb 2024 16:35:02 +0100 Subject: [PATCH 11/55] Re-write Common.py with hip-python --- GPUSimulators/Common.py | 229 +++++++++++++++++++++++++++------------- 1 file changed, 157 insertions(+), 72 deletions(-) diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py index 76902c5..5c18579 100644 --- a/GPUSimulators/Common.py +++ b/GPUSimulators/Common.py @@ -35,14 +35,12 @@ import gc import netCDF4 import json -import pycuda.compiler as cuda_compiler -import pycuda.gpuarray -import pycuda.driver as cuda -from pycuda.tools import PageLockedMemoryPool - - - +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda +#from pycuda.tools import PageLockedMemoryPool +from hip import hip, hiprtc def safeCall(cmd): @@ -511,10 +509,20 @@ class CudaArray2D: #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny) #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here - self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) - - #For returning to download - self.memorypool = PageLockedMemoryPool() + #Initialize an array on GPU with zeros + #self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) + self.data_h = np.zeros((ny_halo, nx_halo), dtype="float32") + num_bytes = self.data_h.size * self.data_h.itemsize + + # init device array and upload host data + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(ny_halo, nx_halo)) + + # copy data from host to device + hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + + #For returning to download (No counterpart in hip-python) + #self.memorypool = PageLockedMemoryPool() #If we don't have any data, just allocate and return if cpu_data is None: @@ -553,7 +561,14 @@ class CudaArray2D: #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) #Allocate host memory #The following fails, don't know why (crashes python) - cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) + #cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) + #see here type of memory: https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemoryType + cpu_data = np.empty((ny, nx), dtype=np.float32) + num_bytes = cpu_data.size * cpu_data.itemsize + #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can #be accessed directly by the GPU device + #hipHostMallocDefault:Memory is mapped and portable (default allocation) + #hipHostMallocPortable: memory is explicitely portable across different devices + cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32) #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32) @@ -563,20 +578,45 @@ class CudaArray2D: assert y+ny <= self.ny + 2*self.y_halo #Create copy object from device to host - copy = cuda.Memcpy2D() - copy.set_src_device(self.data.gpudata) - copy.set_dst_host(cpu_data) + #copy = cuda.Memcpy2D() + #copy.set_src_device(self.data.gpudata) + #copy.set_dst_host(cpu_data) #Set offsets and pitch of source - copy.src_x_in_bytes = int(x)*self.data.strides[1] - copy.src_y = int(y) - copy.src_pitch = self.data.strides[0] + #copy.src_x_in_bytes = int(x)*self.data.strides[1] + #copy.src_y = int(y) + #copy.src_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - copy.width_in_bytes = int(nx)*cpu_data.itemsize - copy.height = int(ny) + #copy.width_in_bytes = int(nx)*cpu_data.itemsize + #copy.height = int(ny) + #The equivalent of cuda.Memcpy2D in hip-python would be: but it fails with an error pointing to cpu_data + #and a message: "RuntimeError: hipError_t.hipErrorInvalidValue" + #shape = (nx,ny) + #num_bytes = cpu_data.size * cpu_data.itemsize + #dst_pitch_bytes = cpu_data.strides[0] + #src_pitch_bytes = num_bytes // shape[0] + #src_pitch_bytes = data.strides[0] + #width_bytes = int(nx)*cpu_data.itemsize + #height_Nrows = int(ny) + #hipMemcpy2D(dst, unsigned long dpitch, src, unsigned long spitch, unsigned long width, unsigned long height, kind) + #copy = hip_check(hip.hipMemcpy2D(cpu_data, #pointer to destination + # dst_pitch_bytes, #pitch of destination array + # data, #pointer to source + # src_pitch_bytes, #pitch of source array + # width_bytes, #number of bytes in each row + # height_Nrows, #number of rows to copy + # hip.hipMemcpyKind.hipMemcpyDeviceToHost)) #kind + + #this is an alternative: + #copy from device to host + cpu_data = np.empty((ny, nx), dtype=np.float32) + num_bytes = cpu_data.size * cpu_data.itemsize + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + copy(stream) if asynch==False: stream.synchronize() @@ -599,29 +639,33 @@ class CudaArray2D: assert(y+ny <= self.ny + 2*self.y_halo) #Create copy object from device to host - copy = cuda.Memcpy2D() - copy.set_dst_device(self.data.gpudata) - copy.set_src_host(cpu_data) + #Well this copy from src:host to dst:device AND NOT from device to host + #copy = cuda.Memcpy2D() + #copy.set_dst_device(self.data.gpudata) + #copy.set_src_host(cpu_data) #Set offsets and pitch of source - copy.dst_x_in_bytes = int(x)*self.data.strides[1] - copy.dst_y = int(y) - copy.dst_pitch = self.data.strides[0] + #copy.dst_x_in_bytes = int(x)*self.data.strides[1] + #copy.dst_y = int(y) + #copy.dst_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - copy.width_in_bytes = int(nx)*cpu_data.itemsize - copy.height = int(ny) + #copy.width_in_bytes = int(nx)*cpu_data.itemsize + #copy.height = int(ny) + #copy from host de device + num_bytes = cpu_data.size * cpu_data.itemsize + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=cpu_data.shape) + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + copy(stream) - - - - """ Class that holds 2D data """ @@ -644,10 +688,20 @@ class CudaArray3D: #self.logger.debug("Allocating [%dx%dx%d] buffer", self.nx, self.ny, self.nz) #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here - self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) + #self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) + self.data_h = np.zeros((nz_halo, ny_halo, nx_halo), dtype="float32") + num_bytes = self.data_h.size * self.data_h.itemsize + + # init device array and upload host data + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(nz_halo, ny_halo, nx_halo)) + + # copy data from host to device + hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + #For returning to download - self.memorypool = PageLockedMemoryPool() + #self.memorypool = PageLockedMemoryPool() #If we don't have any data, just allocate and return if cpu_data is None: @@ -659,30 +713,37 @@ class CudaArray3D: assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" #Create copy object from host to device - copy = cuda.Memcpy3D() - copy.set_src_host(cpu_data) - copy.set_dst_device(self.data.gpudata) + #copy = cuda.Memcpy3D() + #copy.set_src_host(cpu_data) + #copy.set_dst_device(self.data.gpudata) #Set offsets of destination - x_offset = (nx_halo - cpu_data.shape[2]) // 2 - y_offset = (ny_halo - cpu_data.shape[1]) // 2 - z_offset = (nz_halo - cpu_data.shape[0]) // 2 - copy.dst_x_in_bytes = x_offset*self.data.strides[1] - copy.dst_y = y_offset - copy.dst_z = z_offset + #x_offset = (nx_halo - cpu_data.shape[2]) // 2 + #y_offset = (ny_halo - cpu_data.shape[1]) // 2 + #z_offset = (nz_halo - cpu_data.shape[0]) // 2 + #copy.dst_x_in_bytes = x_offset*self.data.strides[1] + #copy.dst_y = y_offset + #copy.dst_z = z_offset #Set pitch of destination - copy.dst_pitch = self.data.strides[0] + #copy.dst_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - width = max(self.nx, cpu_data.shape[2]) - height = max(self.ny, cpu_data.shape[1]) - depth = max(self.nz, cpu-data.shape[0]) - copy.width_in_bytes = width*cpu_data.itemsize - copy.height = height - copy.depth = depth + #width = max(self.nx, cpu_data.shape[2]) + #height = max(self.ny, cpu_data.shape[1]) + #depth = max(self.nz, cpu-data.shape[0]) + #copy.width_in_bytes = width*cpu_data.itemsize + #copy.height = height + #copy.depth = depth + #copy from host to device + num_bytes = cpu_data.size * cpu_data.itemsize + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=cpu_data.shape) + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + #Perform the copy copy(stream) @@ -701,26 +762,31 @@ class CudaArray3D: #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) #Allocate host memory #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32) - #cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) - cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) + cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) + #cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) #Create copy object from device to host - copy = cuda.Memcpy2D() - copy.set_src_device(self.data.gpudata) - copy.set_dst_host(cpu_data) + #copy = cuda.Memcpy2D() + #copy.set_src_device(self.data.gpudata) + #copy.set_dst_host(cpu_data) #Set offsets and pitch of source - copy.src_x_in_bytes = self.x_halo*self.data.strides[1] - copy.src_y = self.y_halo - copy.src_z = self.z_halo - copy.src_pitch = self.data.strides[0] + #copy.src_x_in_bytes = self.x_halo*self.data.strides[1] + #copy.src_y = self.y_halo + #copy.src_z = self.z_halo + #copy.src_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - copy.width_in_bytes = self.nx*cpu_data.itemsize - copy.height = self.ny - copy.depth = self.nz + #copy.width_in_bytes = self.nx*cpu_data.itemsize + #copy.height = self.ny + #copy.depth = self.nz + #copy from device to host + num_bytes = cpu_data.size * cpu_data.itemsize + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + copy(stream) if asynch==False: stream.synchronize() @@ -728,13 +794,6 @@ class CudaArray3D: return cpu_data - - - - - - - """ A class representing an Arakawa A type (unstaggered, logically Cartesian) grid """ @@ -768,13 +827,39 @@ class ArakawaA2D: #stream.synchronize() return cpu_variables - + + #hipblas + def sum_hipblas(self, num_elements, data): + num_bytes_r = np.dtype(np.float32).itemsize + result_d = hip_check(hip.hipMalloc(num_bytes_r)) + result_h = np.zeros(1, dtype=np.float32) + print("--bytes:", num_bytes_r) + + # call hipblasSaxpy + initialization + handle = hip_check(hipblas.hipblasCreate()) + #hip_check(hipblas.hipblasSaxpy(handle, num_elements, ctypes.addressof(alpha), x_d, 1, y_d, 1)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasSasum(handle, num_elements, data, 1, result_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in result_d) back to host (store in result_h) + hip_check(hip.hipMemcpy(result_h,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + + # clean up + hip_check(hip.hipFree(data)) + return result_h + def check(self): """ Checks that data is still sane """ for i, gpu_variable in enumerate(self.gpu_variables): - var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() + #compute sum with hipblas + #var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() + var_sum = self.sum_hipblas(gpu_variable.ny,gpu_variable.data) + self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny)) assert np.isnan(var_sum) == False, "Data contains NaN values!" - \ No newline at end of file + From 7591307b0e1ae4251607c7a22a1a010b18fe3dbd Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:45:19 +0100 Subject: [PATCH 12/55] add "import hipblas" --- GPUSimulators/Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py index 5c18579..49dcaf1 100644 --- a/GPUSimulators/Common.py +++ b/GPUSimulators/Common.py @@ -41,7 +41,7 @@ import json #from pycuda.tools import PageLockedMemoryPool from hip import hip, hiprtc - +from hip import hipblas def safeCall(cmd): logger = logging.getLogger(__name__) From 9bf115ddfd0d57339a249e72323ef41adbdd583b Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 12:45:48 +0100 Subject: [PATCH 13/55] re-write EE2D_KP07_dimsplit.py with hip-python --- GPUSimulators/EE2D_KP07_dimsplit.py | 523 ++++++++++++++++++++++------ 1 file changed, 408 insertions(+), 115 deletions(-) diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py index 2c3f810..0429b28 100644 --- a/GPUSimulators/EE2D_KP07_dimsplit.py +++ b/GPUSimulators/EE2D_KP07_dimsplit.py @@ -23,9 +23,10 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -54,6 +55,21 @@ class EE2D_KP07_dimsplit (BaseSimulator): gamma: Gas constant p: pressure """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, rho, rho_u, rho_v, E, @@ -79,20 +95,45 @@ class EE2D_KP07_dimsplit (BaseSimulator): self.theta = np.float32(theta) #Get kernels - module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("KP07DimsplitKernel") - self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") - - + #module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", + # defines={ + # 'BLOCK_WIDTH': self.block_size[0], + # 'BLOCK_HEIGHT': self.block_size[1] + # }, + # compile_args={ + # 'no_extern_c': True, + # 'options': ["--use_fast_math"], + # }, + # jit_compile_args={}) + #self.kernel = module.get_function("KP07DimsplitKernel") + #self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") + # + kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -102,7 +143,13 @@ class EE2D_KP07_dimsplit (BaseSimulator): nx, ny, 2, 2, [None, None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + # init device array cfl_data + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho))) dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho))) self.dt = min(dt_x, dt_y) @@ -116,25 +163,65 @@ class EE2D_KP07_dimsplit (BaseSimulator): if external and internal: #print("COMPLETE DOMAIN (dt=" + str(dt) + ")") - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - 0, 0, - self.nx, self.ny) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# self.nx, self.ny) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--External & Internal: Launching Kernel is ok") + return if external and not internal: @@ -147,96 +234,231 @@ class EE2D_KP07_dimsplit (BaseSimulator): # NORTH # (x0, y0) x (x1, y1) # (0, ny-y_halo) x (nx, ny) - self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - 0, self.ny - int(self.u0[0].y_halo), - self.nx, self.ny) +# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, self.ny - int(self.u0[0].y_halo), +# self.nx, self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *ns_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo), + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + # SOUTH # (x0, y0) x (x1, y1) # (0, 0) x (nx, y_halo) - self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - 0, 0, - self.nx, int(self.u0[0].y_halo)) - +# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# self.nx, int(self.u0[0].y_halo)) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *ns_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo) + ) + ) + ) + + we_grid_size = (1, self.grid_size[1]) # WEST # (x0, y0) x (x1, y1) # (0, 0) x (x_halo, ny) - self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - 0, 0, - int(self.u0[0].x_halo), self.ny) +# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# int(self.u0[0].x_halo), self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *we_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) + ) + ) + ) + # EAST # (x0, y0) x (x1, y1) # (nx-x_halo, 0) x (nx, ny) - self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - self.nx - int(self.u0[0].x_halo), 0, - self.nx, self.ny) +# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# self.nx - int(self.u0[0].x_halo), 0, +# self.nx, self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *we_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), 0, + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--External and not Internal: Launching Kernel is ok") + return if internal and not external: @@ -263,6 +485,45 @@ class EE2D_KP07_dimsplit (BaseSimulator): self.cfl_data.gpudata, int(self.u0[0].x_halo), int(self.u0[0].y_halo), self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo)) + + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.internal_stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.u0[0].y_halo), + ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Internal and not External: Launching Kernel is ok") return def swapBuffers(self): @@ -277,6 +538,38 @@ class EE2D_KP07_dimsplit (BaseSimulator): self.u1.check() return + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 From e2b56c6ab953a0043e168dc8fd768c99675313a3 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 14:20:12 +0100 Subject: [PATCH 14/55] Re-write FORCE.py with hip-python --- GPUSimulators/FORCE.py | 176 +++++++++++++++++++++++++++++++++-------- 1 file changed, 144 insertions(+), 32 deletions(-) diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index 2e60be6..22a1593 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -25,8 +25,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -53,6 +53,20 @@ class FORCE (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, @@ -74,19 +88,46 @@ class FORCE (Simulator.BaseSimulator): self.g = np.float32(g) #Get kernels - module = context.get_module("cuda/SWE2D_FORCE.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("FORCEKernel") - self.kernel.prepare("iiffffiPiPiPiPiPiPiP") - +# module = context.get_module("cuda/SWE2D_FORCE.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("FORCEKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_FORCE.cu')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .FORCEKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"FORCEKernel")) + + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -96,34 +137,105 @@ class FORCE (Simulator.BaseSimulator): nx, ny, 1, 1, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) self.cfl_data.fill(dt, stream=self.stream) def substep(self, dt, step_number): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) - self.u0, self.u1 = self.u1, self.u0 +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) +# self.u0, self.u1 = self.u1, self.u0 + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .FORCEKernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt From b53f5bf601449108353127ff5ca2a2432a5a703d Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:25:56 +0100 Subject: [PATCH 15/55] replace .cu with cu.hip --- GPUSimulators/EE2D_KP07_dimsplit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py index 0429b28..935eb90 100644 --- a/GPUSimulators/EE2D_KP07_dimsplit.py +++ b/GPUSimulators/EE2D_KP07_dimsplit.py @@ -108,7 +108,7 @@ class EE2D_KP07_dimsplit (BaseSimulator): #self.kernel = module.get_function("KP07DimsplitKernel") #self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") # - kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu')) + kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() From 56445c201ed4c3ba5baf661f7139d7523d2c4f9a Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:26:26 +0100 Subject: [PATCH 16/55] replace .cu with cu.hip --- GPUSimulators/FORCE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index 22a1593..ce42285 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -88,7 +88,7 @@ class FORCE (Simulator.BaseSimulator): self.g = np.float32(g) #Get kernels -# module = context.get_module("cuda/SWE2D_FORCE.cu", +# module = context.get_module("cuda/SWE2D_FORCE.cu.hip", # defines={ # 'BLOCK_WIDTH': self.block_size[0], # 'BLOCK_HEIGHT': self.block_size[1] From e1992ca416814e09fbdf0b4bff5691b23ed88cc1 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:36:19 +0100 Subject: [PATCH 17/55] missing 'tab' --- GPUSimulators/FORCE.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index ce42285..ebf4416 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -187,15 +187,14 @@ class FORCE (Simulator.BaseSimulator): ) ) ) - - self.u0, self.u1 = self.u1, self.u0 + self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .FORCEKernel. is ok") + print("--Launching Kernel .FORCEKernel. is ok") def getOutput(self): return self.u0 From 1afaa068e3a8aa660cade508298470ba20875b75 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 14:45:33 +0100 Subject: [PATCH 18/55] re-write HLL.py with hip-python --- GPUSimulators/HLL.py | 171 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 140 insertions(+), 31 deletions(-) diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py index 9911f9b..55d888b 100644 --- a/GPUSimulators/HLL.py +++ b/GPUSimulators/HLL.py @@ -24,8 +24,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -48,6 +48,20 @@ class HLL (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, @@ -69,19 +83,45 @@ class HLL (Simulator.BaseSimulator): self.g = np.float32(g) #Get kernels - module = context.get_module("cuda/SWE2D_HLL.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("HLLKernel") - self.kernel.prepare("iiffffiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_HLL.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("HLLKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .HLLKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"HLLKernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -91,34 +131,103 @@ class HLL (Simulator.BaseSimulator): nx, ny, 1, 1, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) self.cfl_data.fill(dt, stream=self.stream) def substep(self, dt, step_number): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + self.u0, self.u1 = self.u1, self.u0 - + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .HLLKernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 From 866e54c43ffb93428a10be4f3f9804f07ee8fcff Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 14:55:14 +0100 Subject: [PATCH 19/55] re-write HLL2.py with hip-python --- GPUSimulators/HLL2.py | 162 +++++++++++++++++++++++++++++++++--------- 1 file changed, 130 insertions(+), 32 deletions(-) diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py index 64fa765..b248eb2 100644 --- a/GPUSimulators/HLL2.py +++ b/GPUSimulators/HLL2.py @@ -24,8 +24,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -73,19 +73,45 @@ class HLL2 (Simulator.BaseSimulator): self.theta = np.float32(theta) #Get kernels - module = context.get_module("cuda/SWE2D_HLL2.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("HLL2Kernel") - self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_HLL2.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("HLL2Kernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL2.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .HLL2Kernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"HLL2Kernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -95,7 +121,12 @@ class HLL2 (Simulator.BaseSimulator): nx, ny, 2, 2, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) @@ -105,29 +136,96 @@ class HLL2 (Simulator.BaseSimulator): self.substepDimsplit(dt*0.5, step_number) def substepDimsplit(self, dt, substep): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) self.u0, self.u1 = self.u1, self.u0 + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .HLL2Kernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 + From b63412ff9c45a9b66cda0003fe66aa7e1b626361 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:07:05 +0100 Subject: [PATCH 20/55] add hipDeviceSynchronize --- GPUSimulators/FORCE.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index ebf4416..205c4d7 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -187,9 +187,11 @@ class FORCE (Simulator.BaseSimulator): ) ) ) + + hip_check(hip.hipDeviceSynchronize()) self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) hip_check(hip.hipFree(cfl_data)) From eeed7544255507c2a7785fe69a5dedea309a91cc Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:07:58 +0100 Subject: [PATCH 21/55] add hipDeviceSynchronize --- GPUSimulators/HLL.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py index 55d888b..885fbc3 100644 --- a/GPUSimulators/HLL.py +++ b/GPUSimulators/HLL.py @@ -180,9 +180,10 @@ class HLL (Simulator.BaseSimulator): ) ) + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipDeviceSynchronize()) hip_check(hip.hipModuleUnload(module)) hip_check(hip.hipFree(cfl_data)) From ffb0707c265b9a9fa9c6647f74268143c1e9ad70 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:08:34 +0100 Subject: [PATCH 22/55] add hipDeviceSynchronize --- GPUSimulators/HLL2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py index b248eb2..e9b1313 100644 --- a/GPUSimulators/HLL2.py +++ b/GPUSimulators/HLL2.py @@ -177,9 +177,10 @@ class HLL2 (Simulator.BaseSimulator): ) ) ) + + hip_check(hip.hipDeviceSynchronize()) self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipDeviceSynchronize()) hip_check(hip.hipModuleUnload(module)) hip_check(hip.hipFree(cfl_data)) From 419d720b856e9573955d0c37d0fb8a09eacaecae Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 15:14:57 +0100 Subject: [PATCH 23/55] re-write KP07.py with hip-python --- GPUSimulators/KP07.py | 163 +++++++++++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 32 deletions(-) diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py index 8e2af50..5ad5d8c 100644 --- a/GPUSimulators/KP07.py +++ b/GPUSimulators/KP07.py @@ -29,8 +29,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -76,19 +76,45 @@ class KP07 (Simulator.BaseSimulator): self.order = np.int32(order) #Get kernels - module = context.get_module("cuda/SWE2D_KP07.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("KP07Kernel") - self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_KP07.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("KP07Kernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .KP07Kernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07Kernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -98,7 +124,12 @@ class KP07 (Simulator.BaseSimulator): nx, ny, 2, 2, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) @@ -110,29 +141,97 @@ class KP07 (Simulator.BaseSimulator): def substepRK(self, dt, substep): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.theta, - Simulator.stepOrderToCodedInt(step=substep, order=self.order), - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# Simulator.stepOrderToCodedInt(step=substep, order=self.order), +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + Simulator.stepOrderToCodedInt(step=substep, order=self.order), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .KP07Kernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5**(self.order-1) \ No newline at end of file + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5**(self.order-1) From 916d03385f7f4920c65e26925d04f2fa93900d02 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 15:24:21 +0100 Subject: [PATCH 24/55] re-write KP07_dimsplit.py with hip-python --- GPUSimulators/KP07_dimsplit.py | 160 ++++++++++++++++++++++++++------- 1 file changed, 129 insertions(+), 31 deletions(-) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py index c90cf07..c9e2c80 100644 --- a/GPUSimulators/KP07_dimsplit.py +++ b/GPUSimulators/KP07_dimsplit.py @@ -29,8 +29,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -77,19 +77,45 @@ class KP07_dimsplit(Simulator.BaseSimulator): self.theta = np.float32(theta) #Get kernels - module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("KP07DimsplitKernel") - self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("KP07DimsplitKernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07_dimsplit.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .KP07DimsplitKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -99,7 +125,12 @@ class KP07_dimsplit(Simulator.BaseSimulator): nx, ny, self.gc_x, self.gc_y, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) @@ -109,21 +140,56 @@ class KP07_dimsplit(Simulator.BaseSimulator): self.substepDimsplit(dt*0.5, step_number) def substepDimsplit(self, dt, substep): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + ctypes.c_int(substep) + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .KP07DimsplitKernel. is ok") def getOutput(self): return self.u0 @@ -131,7 +197,39 @@ class KP07_dimsplit(Simulator.BaseSimulator): def check(self): self.u0.check() self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 From 3b2c571dbb08c3630f4600b78ab9f17f722a1abb Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 15:31:51 +0100 Subject: [PATCH 25/55] re-write LxF.py with hip-python --- GPUSimulators/LxF.py | 159 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 128 insertions(+), 31 deletions(-) diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py index 48d9127..3b989d3 100644 --- a/GPUSimulators/LxF.py +++ b/GPUSimulators/LxF.py @@ -25,8 +25,8 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -70,18 +70,44 @@ class LxF (Simulator.BaseSimulator): self.g = np.float32(g) # Get kernels - module = context.get_module("cuda/SWE2D_LxF.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("LxFKernel") - self.kernel.prepare("iiffffiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_LxF.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("LxFKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_LxF.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .LxFKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"LxFKernel")) #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -92,34 +118,105 @@ class LxF (Simulator.BaseSimulator): nx, ny, 1, 1, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) self.cfl_data.fill(dt, stream=self.stream) def substep(self, dt, step_number): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 - + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .LxFKernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 From 64d446cbbfa4f07acd291a37736c6ef927dd33cc Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:35:01 +0100 Subject: [PATCH 26/55] add hip_check --- GPUSimulators/HLL2.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py index e9b1313..f8282a9 100644 --- a/GPUSimulators/HLL2.py +++ b/GPUSimulators/HLL2.py @@ -50,6 +50,20 @@ class HLL2 (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, From bf04b4bd1b606f98d5ea936a653d59c7e0f0e8ae Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:36:33 +0100 Subject: [PATCH 27/55] add hip_check --- GPUSimulators/KP07.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py index 5ad5d8c..a37d25d 100644 --- a/GPUSimulators/KP07.py +++ b/GPUSimulators/KP07.py @@ -51,6 +51,20 @@ class KP07 (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, From 999ea7bb577869bc41f08ea2d5c15e23c7b542aa Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:37:25 +0100 Subject: [PATCH 28/55] add hip_check --- GPUSimulators/KP07_dimsplit.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py index c9e2c80..d6a9efd 100644 --- a/GPUSimulators/KP07_dimsplit.py +++ b/GPUSimulators/KP07_dimsplit.py @@ -52,6 +52,21 @@ class KP07_dimsplit(Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, From 28126ee4e11b334abc4340a3c2b9bc766a641215 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:38:12 +0100 Subject: [PATCH 29/55] add hip_check --- GPUSimulators/LxF.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py index 3b989d3..7751b30 100644 --- a/GPUSimulators/LxF.py +++ b/GPUSimulators/LxF.py @@ -49,6 +49,21 @@ class LxF (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, From 2981ac26f11b5c8858cb8a9e4b6ff9d4d8f56498 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:49:09 +0100 Subject: [PATCH 30/55] add "import ctypes" --- GPUSimulators/LxF.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py index 7751b30..98e54c6 100644 --- a/GPUSimulators/LxF.py +++ b/GPUSimulators/LxF.py @@ -24,6 +24,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From eb43e9b4b7a5941f996e59d008203c5d3989346a Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:49:34 +0100 Subject: [PATCH 31/55] add "import ctypes" --- GPUSimulators/FORCE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index 205c4d7..092711a 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -24,7 +24,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np - +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From 11e46c8aefd8141a557ab251fc14f7a106f13926 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:50:06 +0100 Subject: [PATCH 32/55] add "import ctypes" --- GPUSimulators/HLL.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py index 885fbc3..792d3c6 100644 --- a/GPUSimulators/HLL.py +++ b/GPUSimulators/HLL.py @@ -23,6 +23,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From a3f2e20eaba3042f2410a0acafebd1ba640cc99f Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:50:32 +0100 Subject: [PATCH 33/55] add "import ctypes" --- GPUSimulators/HLL2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py index f8282a9..b5c0dc0 100644 --- a/GPUSimulators/HLL2.py +++ b/GPUSimulators/HLL2.py @@ -23,6 +23,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From 5d45643b524916901d81b118d1d79b01e1ebdcaf Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:51:02 +0100 Subject: [PATCH 34/55] add "import ctypes" --- GPUSimulators/KP07.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py index a37d25d..93ce5e9 100644 --- a/GPUSimulators/KP07.py +++ b/GPUSimulators/KP07.py @@ -28,6 +28,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From 35b213e8d17f07a3f0c9df13aa187e684fead78d Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:51:28 +0100 Subject: [PATCH 35/55] add "import ctypes" --- GPUSimulators/KP07_dimsplit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py index d6a9efd..0a5cfc7 100644 --- a/GPUSimulators/KP07_dimsplit.py +++ b/GPUSimulators/KP07_dimsplit.py @@ -28,6 +28,7 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes #from pycuda import gpuarray from hip import hip,hiprtc From 41367e2ba6f7a22a6dfbc5a458762359aaaa30aa Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 20 Feb 2024 15:51:57 +0100 Subject: [PATCH 36/55] re-write WAF.py with hip-python --- GPUSimulators/WAF.py | 176 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 145 insertions(+), 31 deletions(-) diff --git a/GPUSimulators/WAF.py b/GPUSimulators/WAF.py index 22860fc..7e2763c 100644 --- a/GPUSimulators/WAF.py +++ b/GPUSimulators/WAF.py @@ -24,9 +24,10 @@ along with this program. If not, see . from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np +import ctypes -from pycuda import gpuarray - +#from pycuda import gpuarray +from hip import hip,hiprtc @@ -47,6 +48,21 @@ class WAF (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, context, h0, hu0, hv0, @@ -68,19 +84,45 @@ class WAF (Simulator.BaseSimulator): self.g = np.float32(g) #Get kernels - module = context.get_module("cuda/SWE2D_WAF.cu", - defines={ - 'BLOCK_WIDTH': self.block_size[0], - 'BLOCK_HEIGHT': self.block_size[1] - }, - compile_args={ - 'no_extern_c': True, - 'options': ["--use_fast_math"], - }, - jit_compile_args={}) - self.kernel = module.get_function("WAFKernel") - self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") +# module = context.get_module("cuda/SWE2D_WAF.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("WAFKernel") +# self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_WAF.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .WAFKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"WAFKernel")) + #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, @@ -90,7 +132,12 @@ class WAF (Simulator.BaseSimulator): nx, ny, 2, 2, [None, None, None]) - self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) @@ -100,28 +147,95 @@ class WAF (Simulator.BaseSimulator): self.substepDimsplit(dt*0.5, step_number) def substepDimsplit(self, dt, substep): - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.cfl_data.gpudata) +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .WAFKernel. is ok") + def getOutput(self): return self.u0 def check(self): self.u0.check() self.u1.check() - + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + def computeDt(self): - max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5 \ No newline at end of file + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 From 655913021cc63e7af05581ca3c379a0204195e05 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Mon, 26 Feb 2024 10:45:49 +0100 Subject: [PATCH 37/55] Re-write MPISimulator.py with hip-python --- GPUSimulators/MPISimulator.py | 57 +++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py index 31f62e8..ebb5787 100644 --- a/GPUSimulators/MPISimulator.py +++ b/GPUSimulators/MPISimulator.py @@ -26,9 +26,9 @@ import numpy as np from mpi4py import MPI import time -import pycuda.driver as cuda +#import pycuda.driver as cuda #import nvtx - +from hip import hip, hiprtc class MPIGrid(object): @@ -292,17 +292,54 @@ class MPISimulator(Simulator.BaseSimulator): #Note that east and west also transfer ghost cells #whilst north/south only transfer internal cells #Reuses the width/height defined in the read-extets above - self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) - self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) - self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) - self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) + ##self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) + + ##self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) + ##self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) + ##self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) + + self.in_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + num_bytes_e = self.in_e.size * self.in_e.itemsize + #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can be accessed directly by the GPU device + #hipHostMallocDefault:Memory is mapped and portable (default allocation) + #hipHostMallocPortable: memory is explicitely portable across different devices + self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + + self.in_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + num_bytes_w = self.in_w.size * self.in_w.itemsize + self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + + self.in_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + num_bytes_n = self.in_n.size * self.in_n.itemsize + self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + + self.in_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + num_bytes_s = self.in_s.size * self.in_s.itemsize + self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) #Allocate data for sending - self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) - self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) - self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) - self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) + #self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) + #self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) + #self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) + #self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) + self.out_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + num_bytes_e = self.out_e.size * self.out_e.itemsize + self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + + self.out_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + num_bytes_w = self.out_w.size * self.out_w.itemsize + self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + + self.out_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + num_bytes_n = self.out_n.size * self.out_n.itemsize + self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + + self.out_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + num_bytes_s = self.out_s.size * self.out_s.itemsize + self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + + self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) self.full_exchange() From 7573668e53f0d48ce86536941a004692601a2928 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:55:14 +0100 Subject: [PATCH 38/55] add "import hip, hiprtc" --- GPUSimulators/SHMEMSimulator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py index 4385919..1a608ed 100644 --- a/GPUSimulators/SHMEMSimulator.py +++ b/GPUSimulators/SHMEMSimulator.py @@ -24,7 +24,8 @@ import logging from GPUSimulators import Simulator, CudaContext import numpy as np -import pycuda.driver as cuda +#import pycuda.driver as cuda +from hip import hip, hiprtc import time From 7a8214e3f0c239efa80dad03777ae87c83dca30b Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:01:45 +0100 Subject: [PATCH 39/55] add hip_check() --- GPUSimulators/Common.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py index 49dcaf1..6681450 100644 --- a/GPUSimulators/Common.py +++ b/GPUSimulators/Common.py @@ -43,6 +43,20 @@ import json from hip import hip, hiprtc from hip import hipblas +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def safeCall(cmd): logger = logging.getLogger(__name__) try: From 9d997430ee18ac91c89ab7efd7cfe4510f016ee2 Mon Sep 17 00:00:00 2001 From: Hicham Agueny <95568317+HichamAgueny@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:04:58 +0100 Subject: [PATCH 40/55] add hip_check() --- GPUSimulators/MPISimulator.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py index ebb5787..29706d0 100644 --- a/GPUSimulators/MPISimulator.py +++ b/GPUSimulators/MPISimulator.py @@ -206,6 +206,20 @@ class MPISimulator(Simulator.BaseSimulator): """ Class which handles communication between simulators on different MPI nodes """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, sim, grid): self.profiling_data_mpi = { 'start': {}, 'end': {} } self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0 From ca9171a5bf64d09a250e590604a2a2110d65ca04 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Mon, 26 Feb 2024 12:39:56 +0100 Subject: [PATCH 41/55] Simulator.py with hip-python --- GPUSimulators/Simulator.py | 50 ++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/GPUSimulators/Simulator.py b/GPUSimulators/Simulator.py index 178183d..b804d79 100644 --- a/GPUSimulators/Simulator.py +++ b/GPUSimulators/Simulator.py @@ -25,16 +25,15 @@ import numpy as np import logging from enum import IntEnum -import pycuda.compiler as cuda_compiler -import pycuda.gpuarray -import pycuda.driver as cuda +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda + +from hip import hip, hiprtc from GPUSimulators import Common - - - class BoundaryCondition(object): """ Class for holding boundary conditions for global boundaries @@ -101,13 +100,17 @@ class BoundaryCondition(object): - - - - - class BaseSimulator(object): - + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + return result + def __init__(self, context, nx, ny, @@ -161,9 +164,11 @@ class BaseSimulator(object): ) #Create a CUDA stream - self.stream = cuda.Stream() - self.internal_stream = cuda.Stream() - + #self.stream = cuda.Stream() + #self.internal_stream = cuda.Stream() + self.stream = hip_check(hip.hipStreamCreate()) + self.internal_stream = hip_check(hip.hipStreamCreate()) + #Keep track of simulation time and number of timesteps self.t = 0.0 self.nt = 0 @@ -231,7 +236,9 @@ class BaseSimulator(object): return self.getOutput().download(self.stream, variables) def synchronize(self): - self.stream.synchronize() + #self.stream.synchronize() + #Synchronize the stream to ensure operations in the stream is complete + hip_check(hip.hipStreamSynchronize(self.stream)) def simTime(self): return self.t @@ -268,15 +275,6 @@ class BaseSimulator(object): - - - - - - - - - def stepOrderToCodedInt(step, order): """ Helper function which packs the step and order into a single integer @@ -285,4 +283,4 @@ def stepOrderToCodedInt(step, order): #print("Step: {0:032b}".format(step)) #print("Order: {0:032b}".format(order)) #print("Mix: {0:032b}".format(step_order)) - return np.int32(step_order) \ No newline at end of file + return np.int32(step_order) From e2b1281f5bbc47572d35b934a040193eca6e1eb2 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Mon, 26 Feb 2024 12:55:34 +0100 Subject: [PATCH 42/55] update streams with hip-python --- GPUSimulators/MPISimulator.py | 14 +++++++----- GPUSimulators/SHMEMSimulatorGroup.py | 33 ++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py index 29706d0..f13de52 100644 --- a/GPUSimulators/MPISimulator.py +++ b/GPUSimulators/MPISimulator.py @@ -219,7 +219,7 @@ class MPISimulator(Simulator.BaseSimulator): ): raise RuntimeError(str(err)) return result - + def __init__(self, sim, grid): self.profiling_data_mpi = { 'start': {}, 'end': {} } self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0 @@ -382,8 +382,10 @@ class MPISimulator(Simulator.BaseSimulator): self.full_exchange() #nvtx.mark("sync start", color="blue") - self.sim.stream.synchronize() - self.sim.internal_stream.synchronize() + #self.sim.stream.synchronize() + #self.sim.internal_stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) + hip_check(hip.hipStreamSynchronize(self.sim.internal_stream)) #nvtx.mark("sync end", color="blue") self.profiling_data_mpi["n_time_steps"] += 1 @@ -433,7 +435,8 @@ class MPISimulator(Simulator.BaseSimulator): if self.south is not None: for k in range(self.nvars): self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_s[k,:,:], asynch=True, extent=self.read_s) - self.sim.stream.synchronize() + #self.sim.stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() @@ -488,7 +491,8 @@ class MPISimulator(Simulator.BaseSimulator): if self.west is not None: for k in range(self.nvars): self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_w[k,:,:], asynch=True, extent=self.read_w) - self.sim.stream.synchronize() + #self.sim.stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() diff --git a/GPUSimulators/SHMEMSimulatorGroup.py b/GPUSimulators/SHMEMSimulatorGroup.py index fc11d50..c9dc30f 100644 --- a/GPUSimulators/SHMEMSimulatorGroup.py +++ b/GPUSimulators/SHMEMSimulatorGroup.py @@ -24,7 +24,8 @@ import logging from GPUSimulators import Simulator, CudaContext import numpy as np -import pycuda.driver as cuda +#import pycuda.driver as cuda +from hip import hip, hiprtc import time @@ -33,13 +34,28 @@ class SHMEMGrid(object): Class which represents an SHMEM grid of GPUs. Facilitates easy communication between neighboring subdomains in the grid. Contains one CUDA context per subdomain. """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, ngpus=None, ndims=2): self.logger = logging.getLogger(__name__) - cuda.init(flags=0) - self.logger.info("Initializing CUDA") - num_cuda_devices = cuda.Device.count() - + #cuda.init(flags=0) + self.logger.info("Initializing HIP") + #num_cuda_devices = cuda.Device.count() + num_cuda_devices = hip_check(hip.hipGetDeviceCount()) + if ngpus is None: ngpus = num_cuda_devices @@ -357,7 +373,9 @@ class SHMEMSimulatorGroup(object): for k in range(self.nvars[i]): # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) - self.sims[i].stream.synchronize() + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) + def ns_upload(self, i): #Upload to the GPU @@ -378,7 +396,8 @@ class SHMEMSimulatorGroup(object): for k in range(self.nvars[i]): # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) - self.sims[i].stream.synchronize() + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) def ew_upload(self, i): #Upload to the GPU From 80ffaf9b44a09184e0ed4c7204d83b0337e739d3 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Mon, 26 Feb 2024 12:58:55 +0100 Subject: [PATCH 43/55] update streams with hip-python --- GPUSimulators/Autotuner.py | 304 -- GPUSimulators/Common.py | 879 ---- GPUSimulators/CudaContext.py | 328 -- GPUSimulators/CudaContext_cu.py | 272 -- GPUSimulators/EE2D_KP07_dimsplit.py | 575 --- GPUSimulators/FORCE.py | 242 - GPUSimulators/HLL.py | 235 - GPUSimulators/HLL2.py | 247 - GPUSimulators/IPythonMagic.py | 193 - GPUSimulators/KP07.py | 252 - GPUSimulators/KP07_dimsplit.py | 251 - GPUSimulators/LxF.py | 238 - GPUSimulators/MPISimulator.py | 535 --- GPUSimulators/SHMEMSimulator.py | 264 -- GPUSimulators/SHMEMSimulatorGroup.py | 413 -- GPUSimulators/Simulator.py | 286 -- GPUSimulators/WAF.py | 241 - GPUSimulators/__init__.py | 5 - .../__pycache__/MPISimulator.cpython-39.pyc | Bin 11058 -> 0 bytes .../__pycache__/Simulator.cpython-39.pyc | Bin 8433 -> 0 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 182 -> 0 bytes GPUSimulators/cuda/EE2D_KP07_dimsplit.cu | 250 - GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip | 251 - GPUSimulators/cuda/EulerCommon.h | 187 - GPUSimulators/cuda/SWE2D_FORCE.cu | 143 - GPUSimulators/cuda/SWE2D_FORCE.cu.hip | 144 - GPUSimulators/cuda/SWE2D_HLL.cu | 161 - GPUSimulators/cuda/SWE2D_HLL.cu.hip | 162 - GPUSimulators/cuda/SWE2D_HLL2.cu | 216 - GPUSimulators/cuda/SWE2D_HLL2.cu.hip | 217 - GPUSimulators/cuda/SWE2D_KP07.cu | 233 - GPUSimulators/cuda/SWE2D_KP07.cu.hip | 234 - GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu | 216 - GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip | 217 - GPUSimulators/cuda/SWE2D_LxF.cu | 168 - GPUSimulators/cuda/SWE2D_LxF.cu.hip | 169 - GPUSimulators/cuda/SWE2D_WAF.cu | 178 - GPUSimulators/cuda/SWE2D_WAF.cu.hip | 179 - GPUSimulators/cuda/SWECommon.h | 533 --- GPUSimulators/cuda/common.h | 557 --- GPUSimulators/cuda/limiters.h | 118 - GPUSimulators/helpers/InitialConditions.py | 355 -- GPUSimulators/helpers/Visualization.py | 61 - Jobs/job_lumi.slrum | 34 - LICENSE | 674 --- README.md | 45 - SYSTEMS.md | 62 - conda_environment_lumi.yml | 36 - mpiTesting.py | 230 - reference/swashes_1_nx=1024.csv | 1042 ----- reference/swashes_1_nx=128.csv | 146 - reference/swashes_1_nx=2048.csv | 2066 --------- reference/swashes_1_nx=256.csv | 274 -- reference/swashes_1_nx=4096.csv | 4114 ----------------- reference/swashes_1_nx=512.csv | 530 --- 55 files changed, 19962 deletions(-) delete mode 100644 GPUSimulators/Autotuner.py delete mode 100644 GPUSimulators/Common.py delete mode 100644 GPUSimulators/CudaContext.py delete mode 100644 GPUSimulators/CudaContext_cu.py delete mode 100644 GPUSimulators/EE2D_KP07_dimsplit.py delete mode 100644 GPUSimulators/FORCE.py delete mode 100644 GPUSimulators/HLL.py delete mode 100644 GPUSimulators/HLL2.py delete mode 100644 GPUSimulators/IPythonMagic.py delete mode 100644 GPUSimulators/KP07.py delete mode 100644 GPUSimulators/KP07_dimsplit.py delete mode 100644 GPUSimulators/LxF.py delete mode 100644 GPUSimulators/MPISimulator.py delete mode 100644 GPUSimulators/SHMEMSimulator.py delete mode 100644 GPUSimulators/SHMEMSimulatorGroup.py delete mode 100644 GPUSimulators/Simulator.py delete mode 100644 GPUSimulators/WAF.py delete mode 100644 GPUSimulators/__init__.py delete mode 100644 GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc delete mode 100644 GPUSimulators/__pycache__/Simulator.cpython-39.pyc delete mode 100644 GPUSimulators/__pycache__/__init__.cpython-39.pyc delete mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu delete mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip delete mode 100644 GPUSimulators/cuda/EulerCommon.h delete mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu delete mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu delete mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu delete mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu delete mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu delete mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu delete mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu.hip delete mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu delete mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu.hip delete mode 100644 GPUSimulators/cuda/SWECommon.h delete mode 100644 GPUSimulators/cuda/common.h delete mode 100644 GPUSimulators/cuda/limiters.h delete mode 100644 GPUSimulators/helpers/InitialConditions.py delete mode 100644 GPUSimulators/helpers/Visualization.py delete mode 100644 Jobs/job_lumi.slrum delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 SYSTEMS.md delete mode 100644 conda_environment_lumi.yml delete mode 100644 mpiTesting.py delete mode 100644 reference/swashes_1_nx=1024.csv delete mode 100644 reference/swashes_1_nx=128.csv delete mode 100644 reference/swashes_1_nx=2048.csv delete mode 100644 reference/swashes_1_nx=256.csv delete mode 100644 reference/swashes_1_nx=4096.csv delete mode 100644 reference/swashes_1_nx=512.csv diff --git a/GPUSimulators/Autotuner.py b/GPUSimulators/Autotuner.py deleted file mode 100644 index 84aedc2..0000000 --- a/GPUSimulators/Autotuner.py +++ /dev/null @@ -1,304 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the different helper functions and -classes - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os -import gc -import numpy as np -import logging -from socket import gethostname - -#import pycuda.driver as cuda -from hip import hip,hiprtc - -from GPUSimulators import Common, Simulator, CudaContext - -class Autotuner: - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - nx=2048, ny=2048, - block_widths=range(8, 32, 1), - block_heights=range(8, 32, 1)): - logger = logging.getLogger(__name__) - self.filename = "autotuning_data_" + gethostname() + ".npz" - self.nx = nx - self.ny = ny - self.block_widths = block_widths - self.block_heights = block_heights - self.performance = {} - - - def benchmark(self, simulator, force=False): - logger = logging.getLogger(__name__) - - #Run through simulators and benchmark - key = str(simulator.__name__) - logger.info("Benchmarking %s to %s", key, self.filename) - - #If this simulator has been benchmarked already, skip it - if (force==False and os.path.isfile(self.filename)): - with np.load(self.filename) as data: - if key in data["simulators"]: - logger.info("%s already benchmarked - skipping", key) - return - - # Set arguments to send to the simulators during construction - context = CudaContext.CudaContext(autotuning=False) - g = 9.81 - h0, hu0, hv0, dx, dy, dt = Autotuner.gen_test_data(nx=self.nx, ny=self.ny, g=g) - arguments = { - 'context': context, - 'h0': h0, 'hu0': hu0, 'hv0': hv0, - 'nx': self.nx, 'ny': self.ny, - 'dx': dx, 'dy': dy, 'dt': 0.9*dt, - 'g': g - } - - # Load existing data into memory - benchmark_data = { - "simulators": [], - } - if (os.path.isfile(self.filename)): - with np.load(self.filename) as data: - for k, v in data.items(): - benchmark_data[k] = v - - # Run benchmark - benchmark_data[key + "_megacells"] = Autotuner.benchmark_single_simulator(simulator, arguments, self.block_widths, self.block_heights) - benchmark_data[key + "_block_widths"] = self.block_widths - benchmark_data[key + "_block_heights"] = self.block_heights - benchmark_data[key + "_arguments"] = str(arguments) - - existing_sims = benchmark_data["simulators"] - if (isinstance(existing_sims, np.ndarray)): - existing_sims = existing_sims.tolist() - if (key not in existing_sims): - benchmark_data["simulators"] = existing_sims + [key] - - # Save to file - np.savez_compressed(self.filename, **benchmark_data) - - - - """ - Function which reads a numpy file with autotuning data - and reports the maximum performance and block size - """ - def get_peak_performance(self, simulator): - logger = logging.getLogger(__name__) - - assert issubclass(simulator, Simulator.BaseSimulator) - key = simulator.__name__ - - if (key in self.performance): - return self.performance[key] - else: - #Run simulation if required - if (not os.path.isfile(self.filename)): - logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) - self.benchmark(simulator) - - with np.load(self.filename) as data: - if key not in data['simulators']: - logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) - data.close() - self.benchmark(simulator) - data = np.load(self.filename) - - def find_max_index(megacells): - max_index = np.nanargmax(megacells) - return np.unravel_index(max_index, megacells.shape) - - megacells = data[key + '_megacells'] - block_widths = data[key + '_block_widths'] - block_heights = data[key + '_block_heights'] - j, i = find_max_index(megacells) - - self.performance[key] = { "block_width": block_widths[i], - "block_height": block_heights[j], - "megacells": megacells[j, i] } - logger.debug("Returning %s as peak performance parameters", self.performance[key]) - return self.performance[key] - - #This should never happen - raise "Something wrong: Could not get autotuning data!" - return None - - - - """ - Runs a set of benchmarks for a single simulator - """ - def benchmark_single_simulator(simulator, arguments, block_widths, block_heights): - logger = logging.getLogger(__name__) - - megacells = np.empty((len(block_heights), len(block_widths))) - megacells.fill(np.nan) - - logger.debug("Running %d benchmarks with %s", len(block_heights)*len(block_widths), simulator.__name__) - - sim_arguments = arguments.copy() - - with Common.Timer(simulator.__name__) as t: - for j, block_height in enumerate(block_heights): - sim_arguments.update({'block_height': block_height}) - for i, block_width in enumerate(block_widths): - sim_arguments.update({'block_width': block_width}) - megacells[j, i] = Autotuner.run_benchmark(simulator, sim_arguments) - - - logger.debug("Completed %s in %f seconds", simulator.__name__, t.secs) - - return megacells - - - """ - Runs a benchmark, and returns the number of megacells achieved - """ - def run_benchmark(simulator, arguments, timesteps=10, warmup_timesteps=2): - logger = logging.getLogger(__name__) - - #Initialize simulator - try: - sim = simulator(**arguments) - except: - #An exception raised - not possible to continue - logger.debug("Failed creating %s with arguments %s", simulator.__name__, str(arguments)) - return np.nan - - #Create timer events - #start = cuda.Event() - #end = cuda.Event() - stream = hip_check(hip.hipStreamCreate()) - - start = hip_check(hip.hipEventCreate()) - end = hip_check(hip.hipEventCreate()) - - #Warmup - for i in range(warmup_timesteps): - sim.stepEuler(sim.dt) - - #Run simulation with timer - #start.record(sim.stream) - #start recording - hip_check(hip.hipEventRecord(start, stream)) - for i in range(timesteps): - sim.stepEuler(sim.dt) - #end.record(sim.stream) - #stop recording and synchronize - hip_check(hip.hipEventRecord(end, stream)) - - #Synchronize end event - #end.synchronize() - hip_check(hip.hipEventSynchronize(end)) - - #Compute megacells - #gpu_elapsed = end.time_since(start)*1.0e-3 - gpu_elapsed = hip_check(hip.hipEventElapsedTime(start, end)) - - megacells = (sim.nx*sim.ny*timesteps / (1000*1000)) / gpu_elapsed - - #Sanity check solution - h, hu, hv = sim.download() - sane = True - sane = sane and Autotuner.sanity_check(h, 0.3, 0.7) - sane = sane and Autotuner.sanity_check(hu, -0.2, 0.2) - sane = sane and Autotuner.sanity_check(hv, -0.2, 0.2) - - if (sane): - logger.debug("%s [%d x %d] succeeded: %f megacells, gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], megacells, gpu_elapsed) - return megacells - else: - logger.debug("%s [%d x %d] failed: gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], gpu_elapsed) - return np.nan - - - - """ - Generates test dataset - """ - def gen_test_data(nx, ny, g): - width = 100.0 - height = 100.0 - dx = width / float(nx) - dy = height / float(ny) - - x_center = dx*nx/2.0 - y_center = dy*ny/2.0 - - #Create a gaussian "dam break" that will not form shocks - size = width / 5.0 - dt = 10**10 - - h = np.zeros((ny, nx), dtype=np.float32); - hu = np.zeros((ny, nx), dtype=np.float32); - hv = np.zeros((ny, nx), dtype=np.float32); - - extent = 1.0/np.sqrt(2.0) - x = (dx*(np.arange(0, nx, dtype=np.float32)+0.5) - x_center) / size - y = (dy*(np.arange(0, ny, dtype=np.float32)+0.5) - y_center) / size - xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') - r = np.minimum(1.0, np.sqrt(xv**2 + yv**2)) - xv = None - yv = None - gc.collect() - - #Generate highres - cos = np.cos(np.pi*r) - h = 0.5 + 0.1*0.5*(1.0 + cos) - hu = 0.1*0.5*(1.0 + cos) - hv = hu.copy() - - scale = 0.7 - max_h_estimate = 0.6 - max_u_estimate = 0.1*np.sqrt(2.0) - dx = width/nx - dy = height/ny - dt = scale * min(dx, dy) / (max_u_estimate + np.sqrt(g*max_h_estimate)) - - return h, hu, hv, dx, dy, dt - - """ - Checks that a variable is "sane" - """ - def sanity_check(variable, bound_min, bound_max): - maxval = np.amax(variable) - minval = np.amin(variable) - if (np.isnan(maxval) - or np.isnan(minval) - or maxval > bound_max - or minval < bound_min): - return False - else: - return True diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py deleted file mode 100644 index 6681450..0000000 --- a/GPUSimulators/Common.py +++ /dev/null @@ -1,879 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the different helper functions and -classes - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import os - -import numpy as np -import time -import signal -import subprocess -import tempfile -import re -import io -import hashlib -import logging -import gc -import netCDF4 -import json - -#import pycuda.compiler as cuda_compiler -#import pycuda.gpuarray -#import pycuda.driver as cuda -#from pycuda.tools import PageLockedMemoryPool - -from hip import hip, hiprtc -from hip import hipblas - -def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - -def safeCall(cmd): - logger = logging.getLogger(__name__) - try: - #git rev-parse HEAD - current_dir = os.path.dirname(os.path.realpath(__file__)) - params = dict() - params['stderr'] = subprocess.STDOUT - params['cwd'] = current_dir - params['universal_newlines'] = True #text=True in more recent python - params['shell'] = False - if os.name == 'nt': - params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP - stdout = subprocess.check_output(cmd, **params) - except subprocess.CalledProcessError as e: - output = e.output - logger.error("Git failed, \nReturn code: " + str(e.returncode) + "\nOutput: " + output) - raise e - - return stdout - -def getGitHash(): - return safeCall(["git", "rev-parse", "HEAD"]) - -def getGitStatus(): - return safeCall(["git", "status", "--porcelain", "-uno"]) - -def toJson(in_dict, compressed=True): - """ - Creates JSON string from a dictionary - """ - logger = logging.getLogger(__name__) - out_dict = in_dict.copy() - for key in out_dict: - if isinstance(out_dict[key], np.ndarray): - out_dict[key] = out_dict[key].tolist() - else: - try: - json.dumps(out_dict[key]) - except: - value = str(out_dict[key]) - logger.warning("JSON: Converting {:s} to string ({:s})".format(key, value)) - out_dict[key] = value - return json.dumps(out_dict) - -def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names=[], dt=None): - """ - Runs a simulation, and stores output in netcdf file. Stores the times given in - save_times, and saves all of the variables in list save_var_names. Elements in - save_var_names can be set to None if you do not want to save them - """ - profiling_data_sim_runner = { 'start': {}, 'end': {} } - profiling_data_sim_runner["start"]["t_sim_init"] = 0 - profiling_data_sim_runner["end"]["t_sim_init"] = 0 - profiling_data_sim_runner["start"]["t_nc_write"] = 0 - profiling_data_sim_runner["end"]["t_nc_write"] = 0 - profiling_data_sim_runner["start"]["t_full_step"] = 0 - profiling_data_sim_runner["end"]["t_full_step"] = 0 - - profiling_data_sim_runner["start"]["t_sim_init"] = time.time() - - logger = logging.getLogger(__name__) - - assert len(save_times) > 0, "Need to specify which times to save" - - with Timer("construct") as t: - sim = simulator(**simulator_args) - logger.info("Constructed in " + str(t.secs) + " seconds") - - #Create netcdf file and simulate - with DataDumper(outfile, mode='w', clobber=False) as outdata: - - #Create attributes (metadata) - outdata.ncfile.created = time.ctime(time.time()) - outdata.ncfile.git_hash = getGitHash() - outdata.ncfile.git_status = getGitStatus() - outdata.ncfile.simulator = str(simulator) - - # do not write fields to attributes (they are to large) - simulator_args_for_ncfile = simulator_args.copy() - del simulator_args_for_ncfile["rho"] - del simulator_args_for_ncfile["rho_u"] - del simulator_args_for_ncfile["rho_v"] - del simulator_args_for_ncfile["E"] - outdata.ncfile.sim_args = toJson(simulator_args_for_ncfile) - - #Create dimensions - outdata.ncfile.createDimension('time', len(save_times)) - outdata.ncfile.createDimension('x', simulator_args['nx']) - outdata.ncfile.createDimension('y', simulator_args['ny']) - - #Create variables for dimensions - ncvars = {} - ncvars['time'] = outdata.ncfile.createVariable('time', np.dtype('float32').char, 'time') - ncvars['x'] = outdata.ncfile.createVariable( 'x', np.dtype('float32').char, 'x') - ncvars['y'] = outdata.ncfile.createVariable( 'y', np.dtype('float32').char, 'y') - - #Fill variables with proper values - ncvars['time'][:] = save_times - extent = sim.getExtent() - ncvars['x'][:] = np.linspace(extent[0], extent[1], simulator_args['nx']) - ncvars['y'][:] = np.linspace(extent[2], extent[3], simulator_args['ny']) - - #Choose which variables to download (prune None from list, but keep the index) - download_vars = [] - for i, var_name in enumerate(save_var_names): - if var_name is not None: - download_vars += [i] - save_var_names = list(save_var_names[i] for i in download_vars) - - #Create variables - for var_name in save_var_names: - ncvars[var_name] = outdata.ncfile.createVariable(var_name, np.dtype('float32').char, ('time', 'y', 'x'), zlib=True, least_significant_digit=3) - - #Create step sizes between each save - t_steps = np.empty_like(save_times) - t_steps[0] = save_times[0] - t_steps[1:] = save_times[1:] - save_times[0:-1] - - profiling_data_sim_runner["end"]["t_sim_init"] = time.time() - - #Start simulation loop - progress_printer = ProgressPrinter(save_times[-1], print_every=10) - for k in range(len(save_times)): - #Get target time and step size there - t_step = t_steps[k] - t_end = save_times[k] - - #Sanity check simulator - try: - sim.check() - except AssertionError as e: - logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e))) - return outdata.filename - - profiling_data_sim_runner["start"]["t_full_step"] += time.time() - - #Simulate - if (t_step > 0.0): - sim.simulate(t_step, dt) - - profiling_data_sim_runner["end"]["t_full_step"] += time.time() - - profiling_data_sim_runner["start"]["t_nc_write"] += time.time() - - #Download - save_vars = sim.download(download_vars) - - #Save to file - for i, var_name in enumerate(save_var_names): - ncvars[var_name][k, :] = save_vars[i] - - profiling_data_sim_runner["end"]["t_nc_write"] += time.time() - - #Write progress to screen - print_string = progress_printer.getPrintString(t_end) - if (print_string): - logger.debug(print_string) - - logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps())) - - return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi - - - - - - -class Timer(object): - """ - Class which keeps track of time spent for a section of code - """ - def __init__(self, tag, log_level=logging.DEBUG): - self.tag = tag - self.log_level = log_level - self.logger = logging.getLogger(__name__) - - def __enter__(self): - self.start = time.time() - return self - - def __exit__(self, *args): - self.end = time.time() - self.secs = self.end - self.start - self.msecs = self.secs * 1000 # millisecs - self.logger.log(self.log_level, "%s: %f ms", self.tag, self.msecs) - - def elapsed(self): - return time.time() - self.start - - - - - -class PopenFileBuffer(object): - """ - Simple class for holding a set of tempfiles - for communicating with a subprocess - """ - def __init__(self): - self.stdout = tempfile.TemporaryFile(mode='w+t') - self.stderr = tempfile.TemporaryFile(mode='w+t') - - def __del__(self): - self.stdout.close() - self.stderr.close() - - def read(self): - self.stdout.seek(0) - cout = self.stdout.read() - self.stdout.seek(0, 2) - - self.stderr.seek(0) - cerr = self.stderr.read() - self.stderr.seek(0, 2) - - return cout, cerr - -class IPEngine(object): - """ - Class for starting IPEngines for MPI processing in IPython - """ - def __init__(self, n_engines): - self.logger = logging.getLogger(__name__) - - #Start ipcontroller - self.logger.info("Starting IPController") - self.c_buff = PopenFileBuffer() - c_cmd = ["ipcontroller", "--ip='*'"] - c_params = dict() - c_params['stderr'] = self.c_buff.stderr - c_params['stdout'] = self.c_buff.stdout - c_params['shell'] = False - if os.name == 'nt': - c_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP - self.c = subprocess.Popen(c_cmd, **c_params) - - #Wait until controller is running - time.sleep(3) - - #Start engines - self.logger.info("Starting IPEngines") - self.e_buff = PopenFileBuffer() - e_cmd = ["mpiexec", "-n", str(n_engines), "ipengine", "--mpi"] - e_params = dict() - e_params['stderr'] = self.e_buff.stderr - e_params['stdout'] = self.e_buff.stdout - e_params['shell'] = False - if os.name == 'nt': - e_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP - self.e = subprocess.Popen(e_cmd, **e_params) - - # attach to a running cluster - import ipyparallel - self.cluster = ipyparallel.Client()#profile='mpi') - time.sleep(3) - while(len(self.cluster.ids) != n_engines): - time.sleep(0.5) - self.logger.info("Waiting for cluster...") - self.cluster = ipyparallel.Client()#profile='mpi') - - self.logger.info("Done") - - def __del__(self): - self.shutdown() - - def shutdown(self): - if (self.e is not None): - if (os.name == 'nt'): - self.logger.warn("Sending CTRL+C to IPEngine") - self.e.send_signal(signal.CTRL_C_EVENT) - - try: - self.e.communicate(timeout=3) - self.e.kill() - except subprocess.TimeoutExpired: - self.logger.warn("Killing IPEngine") - self.e.kill() - self.e.communicate() - self.e = None - - cout, cerr = self.e_buff.read() - self.logger.info("IPEngine cout: {:s}".format(cout)) - self.logger.info("IPEngine cerr: {:s}".format(cerr)) - self.e_buff = None - - gc.collect() - - if (self.c is not None): - if (os.name == 'nt'): - self.logger.warn("Sending CTRL+C to IPController") - self.c.send_signal(signal.CTRL_C_EVENT) - - try: - self.c.communicate(timeout=3) - self.c.kill() - except subprocess.TimeoutExpired: - self.logger.warn("Killing IPController") - self.c.kill() - self.c.communicate() - self.c = None - - cout, cerr = self.c_buff.read() - self.logger.info("IPController cout: {:s}".format(cout)) - self.logger.info("IPController cerr: {:s}".format(cerr)) - self.c_buff = None - - gc.collect() - - - - - - -class DataDumper(object): - """ - Simple class for holding a netCDF4 object - (handles opening and closing in a nice way) - Use as - with DataDumper("filename") as data: - ... - """ - def __init__(self, filename, *args, **kwargs): - self.logger = logging.getLogger(__name__) - - #Create directory if needed - filename = os.path.abspath(filename) - dirname = os.path.dirname(filename) - if dirname and not os.path.isdir(dirname): - self.logger.info("Creating directory " + dirname) - os.makedirs(dirname) - - #Get mode of file if we have that - mode = None - if (args): - mode = args[0] - elif (kwargs and 'mode' in kwargs.keys()): - mode = kwargs['mode'] - - #Create new unique file if writing - if (mode): - if (("w" in mode) or ("+" in mode) or ("a" in mode)): - i = 0 - stem, ext = os.path.splitext(filename) - while (os.path.isfile(filename)): - filename = "{:s}_{:04d}{:s}".format(stem, i, ext) - i = i+1 - self.filename = os.path.abspath(filename) - - #Save arguments - self.args = args - self.kwargs = kwargs - - #Log output - self.logger.info("Initialized " + self.filename) - - - def __enter__(self): - self.logger.info("Opening " + self.filename) - if (self.args): - self.logger.info("Arguments: " + str(self.args)) - if (self.kwargs): - self.logger.info("Keyword arguments: " + str(self.kwargs)) - self.ncfile = netCDF4.Dataset(self.filename, *self.args, **self.kwargs) - return self - - def __exit__(self, *args): - self.logger.info("Closing " + self.filename) - self.ncfile.close() - - - def toJson(in_dict): - out_dict = in_dict.copy() - - for key in out_dict: - if isinstance(out_dict[key], np.ndarray): - out_dict[key] = out_dict[key].tolist() - else: - try: - json.dumps(out_dict[key]) - except: - out_dict[key] = str(out_dict[key]) - - return json.dumps(out_dict) - - - - - -class ProgressPrinter(object): - """ - Small helper class for - """ - def __init__(self, total_steps, print_every=5): - self.logger = logging.getLogger(__name__) - self.start = time.time() - self.total_steps = total_steps - self.print_every = print_every - self.next_print_time = self.print_every - self.last_step = 0 - self.secs_per_iter = None - - def getPrintString(self, step): - elapsed = time.time() - self.start - if (elapsed > self.next_print_time): - dt = elapsed - (self.next_print_time - self.print_every) - dsteps = step - self.last_step - steps_remaining = self.total_steps - step - - if (dsteps == 0): - return - - self.last_step = step - self.next_print_time = elapsed + self.print_every - - if not self.secs_per_iter: - self.secs_per_iter = dt / dsteps - self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps) - - remaining_time = steps_remaining * self.secs_per_iter - - return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format( - ProgressPrinter.progressBar(step, self.total_steps), - ProgressPrinter.timeString(elapsed + remaining_time), - ProgressPrinter.timeString(elapsed), - ProgressPrinter.timeString(remaining_time)) - - def timeString(seconds): - seconds = int(max(seconds, 1)) - minutes, seconds = divmod(seconds, 60) - hours, minutes = divmod(minutes, 60) - periods = [('h', hours), ('m', minutes), ('s', seconds)] - time_string = ' '.join('{}{}'.format(value, name) - for name, value in periods - if value) - return time_string - - def progressBar(step, total_steps, width=30): - progress = np.round(width * step / total_steps).astype(np.int32) - progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%" - return progressbar - - - - - - - -""" -Class that holds 2D data -""" -class CudaArray2D: - """ - Uploads initial data to the CUDA device - """ - def __init__(self, stream, nx, ny, x_halo, y_halo, cpu_data=None, dtype=np.float32): - self.logger = logging.getLogger(__name__) - self.nx = nx - self.ny = ny - self.x_halo = x_halo - self.y_halo = y_halo - - nx_halo = nx + 2*x_halo - ny_halo = ny + 2*y_halo - - #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny) - #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here - #Initialize an array on GPU with zeros - #self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) - self.data_h = np.zeros((ny_halo, nx_halo), dtype="float32") - num_bytes = self.data_h.size * self.data_h.itemsize - - # init device array and upload host data - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=(ny_halo, nx_halo)) - - # copy data from host to device - hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - - #For returning to download (No counterpart in hip-python) - #self.memorypool = PageLockedMemoryPool() - - #If we don't have any data, just allocate and return - if cpu_data is None: - return - - #Make sure data is in proper format - assert cpu_data.shape == (ny_halo, nx_halo) or cpu_data.shape == (self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.ny, self.nx)), str((ny_halo, nx_halo))) - assert cpu_data.itemsize == 4, "Wrong size of data type" - assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" - - #Create copy object from host to device - x = (nx_halo - cpu_data.shape[1]) // 2 - y = (ny_halo - cpu_data.shape[0]) // 2 - self.upload(stream, cpu_data, extent=[x, y, cpu_data.shape[1], cpu_data.shape[0]]) - #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) - - - def __del__(self, *args): - #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) - self.data.gpudata.free() - self.data = None - - """ - Enables downloading data from GPU to Python - """ - def download(self, stream, cpu_data=None, asynch=False, extent=None): - if (extent is None): - x = self.x_halo - y = self.y_halo - nx = self.nx - ny = self.ny - else: - x, y, nx, ny = extent - - if (cpu_data is None): - #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) - #Allocate host memory - #The following fails, don't know why (crashes python) - #cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) - #see here type of memory: https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemoryType - cpu_data = np.empty((ny, nx), dtype=np.float32) - num_bytes = cpu_data.size * cpu_data.itemsize - #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can #be accessed directly by the GPU device - #hipHostMallocDefault:Memory is mapped and portable (default allocation) - #hipHostMallocPortable: memory is explicitely portable across different devices - cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) - #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32) - #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32) - - assert nx == cpu_data.shape[1] - assert ny == cpu_data.shape[0] - assert x+nx <= self.nx + 2*self.x_halo - assert y+ny <= self.ny + 2*self.y_halo - - #Create copy object from device to host - #copy = cuda.Memcpy2D() - #copy.set_src_device(self.data.gpudata) - #copy.set_dst_host(cpu_data) - - #Set offsets and pitch of source - #copy.src_x_in_bytes = int(x)*self.data.strides[1] - #copy.src_y = int(y) - #copy.src_pitch = self.data.strides[0] - - #Set width in bytes to copy for each row and - #number of rows to copy - #copy.width_in_bytes = int(nx)*cpu_data.itemsize - #copy.height = int(ny) - - #The equivalent of cuda.Memcpy2D in hip-python would be: but it fails with an error pointing to cpu_data - #and a message: "RuntimeError: hipError_t.hipErrorInvalidValue" - #shape = (nx,ny) - #num_bytes = cpu_data.size * cpu_data.itemsize - #dst_pitch_bytes = cpu_data.strides[0] - #src_pitch_bytes = num_bytes // shape[0] - #src_pitch_bytes = data.strides[0] - #width_bytes = int(nx)*cpu_data.itemsize - #height_Nrows = int(ny) - #hipMemcpy2D(dst, unsigned long dpitch, src, unsigned long spitch, unsigned long width, unsigned long height, kind) - #copy = hip_check(hip.hipMemcpy2D(cpu_data, #pointer to destination - # dst_pitch_bytes, #pitch of destination array - # data, #pointer to source - # src_pitch_bytes, #pitch of source array - # width_bytes, #number of bytes in each row - # height_Nrows, #number of rows to copy - # hip.hipMemcpyKind.hipMemcpyDeviceToHost)) #kind - - #this is an alternative: - #copy from device to host - cpu_data = np.empty((ny, nx), dtype=np.float32) - num_bytes = cpu_data.size * cpu_data.itemsize - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) - - copy(stream) - if asynch==False: - stream.synchronize() - - return cpu_data - - - def upload(self, stream, cpu_data, extent=None): - if (extent is None): - x = self.x_halo - y = self.y_halo - nx = self.nx - ny = self.ny - else: - x, y, nx, ny = extent - - assert(nx == cpu_data.shape[1]) - assert(ny == cpu_data.shape[0]) - assert(x+nx <= self.nx + 2*self.x_halo) - assert(y+ny <= self.ny + 2*self.y_halo) - - #Create copy object from device to host - #Well this copy from src:host to dst:device AND NOT from device to host - #copy = cuda.Memcpy2D() - #copy.set_dst_device(self.data.gpudata) - #copy.set_src_host(cpu_data) - - #Set offsets and pitch of source - #copy.dst_x_in_bytes = int(x)*self.data.strides[1] - #copy.dst_y = int(y) - #copy.dst_pitch = self.data.strides[0] - - #Set width in bytes to copy for each row and - #number of rows to copy - #copy.width_in_bytes = int(nx)*cpu_data.itemsize - #copy.height = int(ny) - - #copy from host de device - num_bytes = cpu_data.size * cpu_data.itemsize - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=cpu_data.shape) - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - - copy(stream) - - - - -""" -Class that holds 2D data -""" -class CudaArray3D: - """ - Uploads initial data to the CL device - """ - def __init__(self, stream, nx, ny, nz, x_halo, y_halo, z_halo, cpu_data=None, dtype=np.float32): - self.logger = logging.getLogger(__name__) - self.nx = nx - self.ny = ny - self.nz = nz - self.x_halo = x_halo - self.y_halo = y_halo - self.z_halo = z_halo - - nx_halo = nx + 2*x_halo - ny_halo = ny + 2*y_halo - nz_halo = nz + 2*z_halo - - #self.logger.debug("Allocating [%dx%dx%d] buffer", self.nx, self.ny, self.nz) - #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here - #self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) - - self.data_h = np.zeros((nz_halo, ny_halo, nx_halo), dtype="float32") - num_bytes = self.data_h.size * self.data_h.itemsize - - # init device array and upload host data - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=(nz_halo, ny_halo, nx_halo)) - - # copy data from host to device - hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - - #For returning to download - #self.memorypool = PageLockedMemoryPool() - - #If we don't have any data, just allocate and return - if cpu_data is None: - return - - #Make sure data is in proper format - assert cpu_data.shape == (nz_halo, ny_halo, nx_halo) or cpu_data.shape == (self.nz, self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.nz, self.ny, self.nx)), str((nz_halo, ny_halo, nx_halo))) - assert cpu_data.itemsize == 4, "Wrong size of data type" - assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" - - #Create copy object from host to device - #copy = cuda.Memcpy3D() - #copy.set_src_host(cpu_data) - #copy.set_dst_device(self.data.gpudata) - - #Set offsets of destination - #x_offset = (nx_halo - cpu_data.shape[2]) // 2 - #y_offset = (ny_halo - cpu_data.shape[1]) // 2 - #z_offset = (nz_halo - cpu_data.shape[0]) // 2 - #copy.dst_x_in_bytes = x_offset*self.data.strides[1] - #copy.dst_y = y_offset - #copy.dst_z = z_offset - - #Set pitch of destination - #copy.dst_pitch = self.data.strides[0] - - #Set width in bytes to copy for each row and - #number of rows to copy - #width = max(self.nx, cpu_data.shape[2]) - #height = max(self.ny, cpu_data.shape[1]) - #depth = max(self.nz, cpu-data.shape[0]) - #copy.width_in_bytes = width*cpu_data.itemsize - #copy.height = height - #copy.depth = depth - - #copy from host to device - num_bytes = cpu_data.size * cpu_data.itemsize - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=cpu_data.shape) - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - - #Perform the copy - copy(stream) - - #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) - - - def __del__(self, *args): - #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) - self.data.gpudata.free() - self.data = None - - """ - Enables downloading data from GPU to Python - """ - def download(self, stream, asynch=False): - #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) - #Allocate host memory - #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32) - cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) - #cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) - - #Create copy object from device to host - #copy = cuda.Memcpy2D() - #copy.set_src_device(self.data.gpudata) - #copy.set_dst_host(cpu_data) - - #Set offsets and pitch of source - #copy.src_x_in_bytes = self.x_halo*self.data.strides[1] - #copy.src_y = self.y_halo - #copy.src_z = self.z_halo - #copy.src_pitch = self.data.strides[0] - - #Set width in bytes to copy for each row and - #number of rows to copy - #copy.width_in_bytes = self.nx*cpu_data.itemsize - #copy.height = self.ny - #copy.depth = self.nz - - #copy from device to host - num_bytes = cpu_data.size * cpu_data.itemsize - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) - - copy(stream) - if asynch==False: - stream.synchronize() - - return cpu_data - - -""" -A class representing an Arakawa A type (unstaggered, logically Cartesian) grid -""" -class ArakawaA2D: - def __init__(self, stream, nx, ny, halo_x, halo_y, cpu_variables): - """ - Uploads initial data to the GPU device - """ - self.logger = logging.getLogger(__name__) - self.gpu_variables = [] - for cpu_variable in cpu_variables: - self.gpu_variables += [CudaArray2D(stream, nx, ny, halo_x, halo_y, cpu_variable)] - - def __getitem__(self, key): - assert type(key) == int, "Indexing is int based" - if (key > len(self.gpu_variables) or key < 0): - raise IndexError("Out of bounds") - return self.gpu_variables[key] - - def download(self, stream, variables=None): - """ - Enables downloading data from the GPU device to Python - """ - if variables is None: - variables=range(len(self.gpu_variables)) - - cpu_variables = [] - for i in variables: - assert i < len(self.gpu_variables), "Variable {:d} is out of range".format(i) - cpu_variables += [self.gpu_variables[i].download(stream, asynch=True)] - - #stream.synchronize() - return cpu_variables - - #hipblas - def sum_hipblas(self, num_elements, data): - num_bytes_r = np.dtype(np.float32).itemsize - result_d = hip_check(hip.hipMalloc(num_bytes_r)) - result_h = np.zeros(1, dtype=np.float32) - print("--bytes:", num_bytes_r) - - # call hipblasSaxpy + initialization - handle = hip_check(hipblas.hipblasCreate()) - #hip_check(hipblas.hipblasSaxpy(handle, num_elements, ctypes.addressof(alpha), x_d, 1, y_d, 1)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasSasum(handle, num_elements, data, 1, result_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in result_d) back to host (store in result_h) - hip_check(hip.hipMemcpy(result_h,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) - - # clean up - hip_check(hip.hipFree(data)) - return result_h - - def check(self): - """ - Checks that data is still sane - """ - for i, gpu_variable in enumerate(self.gpu_variables): - #compute sum with hipblas - #var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() - var_sum = self.sum_hipblas(gpu_variable.ny,gpu_variable.data) - - self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny)) - assert np.isnan(var_sum) == False, "Data contains NaN values!" - diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py deleted file mode 100644 index e77ef06..0000000 --- a/GPUSimulators/CudaContext.py +++ /dev/null @@ -1,328 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements Cuda context handling - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - - -import os - -import numpy as np -import time -import re -import io -import hashlib -import logging -import gc - -#import pycuda.compiler as cuda_compiler -#import pycuda.gpuarray -#import pycuda.driver as cuda - -from hip import hip,hiprtc -from hip import rccl - -from GPUSimulators import Autotuner, Common - - -def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - -""" -Class which keeps track of the CUDA context and some helper functions -""" -class CudaContext(object): - - def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): - """ - Create a new CUDA context - Set device to an id or pci_bus_id to select a specific GPU - Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context - """ - self.use_cache = use_cache - self.logger = logging.getLogger(__name__) - self.modules = {} - - self.module_path = os.path.dirname(os.path.realpath(__file__)) - - #Initialize cuda (must be first call to PyCUDA) - ##cuda.init(flags=0) - - ##self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) - - #Print some info about CUDA - ##self.logger.info("CUDA version %s", str(cuda.get_version())) - #self.logger.info("Driver version %s", str(cuda.get_driver_version())) - self.logger.info("Driver version %s", str(hip_check(hip.hipDriverGetVersion()))) - - if device is None: - device = 0 - - self.cuda_device = hip.Device(device) - #self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) - self.logger.info("Using device %d/%d '%s' (%s) GPU", device, hip_check(hip.hipGetDeviceCount())) - #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) - self.logger.debug(" => compute capability: %s", str(self.hip.hipDeviceComputeCapability(device))) - - # Create the CUDA context - #In HIP there is no need to specify a scheduling policy (it is abstracted). Here the HIP runtime system manages the workload to fit a specifc target architecture - #if context_flags is None: - # context_flags=cuda.ctx_flags.SCHED_AUTO - - #self.cuda_context = self.cuda_device.make_context(flags=context_flags) - - #free, total = cuda.mem_get_info() - total = hip_check(hip.hipDeviceTotalMem(device)) - #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) - self.logger.debug(" => memory: %d / %d MB available", int(total/(1024*1024))) - - #self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) - - #Create cache dir for cubin files - self.cache_path = os.path.join(self.module_path, "cuda_cache") - if (self.use_cache): - if not os.path.isdir(self.cache_path): - os.mkdir(self.cache_path) - self.logger.info("Using CUDA cache dir %s", self.cache_path) - - self.autotuner = None - if (autotuning): - self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") - self.autotuner = Autotuner.Autotuner() - - -# def __del__(self, *args): -# self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) - - # Loop over all contexts in stack, and remove "this" -# other_contexts = [] -# while (cuda.Context.get_current() != None): -# context = cuda.Context.get_current() -# if (context.handle != self.cuda_context.handle): -# self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) -# other_contexts = [context] + other_contexts -# cuda.Context.pop() -# else: -# self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) -# cuda.Context.pop() - - # Add all the contexts we popped that were not our own -# for context in other_contexts: -# self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) -# cuda.Context.push(context) - -# self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) -# self.cuda_context.detach() - - -# def __str__(self): -# return "CudaContext id " + str(self.cuda_context.handle) - - - def hash_kernel(kernel_filename, include_dirs): - # Generate a kernel ID for our caches - num_includes = 0 - max_includes = 100 - kernel_hasher = hashlib.md5() - logger = logging.getLogger(__name__) - - # Loop over file and includes, and check if something has changed - files = [kernel_filename] - while len(files): - - if (num_includes > max_includes): - raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) - - filename = files.pop() - - #logger.debug("Hashing %s", filename) - - modified = os.path.getmtime(filename) - - # Open the file - with io.open(filename, "r") as file: - - # Search for #inclue and also hash the file - file_str = file.read() - kernel_hasher.update(file_str.encode('utf-8')) - kernel_hasher.update(str(modified).encode('utf-8')) - - #Find all includes - includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) - - # Loop over everything that looks like an include - for include_file in includes: - - #Search through include directories for the file - file_path = os.path.dirname(filename) - for include_path in [file_path] + include_dirs: - - # If we find it, add it to list of files to check - temp_path = os.path.join(include_path, include_file) - if (os.path.isfile(temp_path)): - files = files + [temp_path] - num_includes = num_includes + 1 #For circular includes... - break - - return kernel_hasher.hexdigest() - - - """ - Reads a text file and creates an OpenCL kernel from that - """ - def get_module(self, kernel_filename, - include_dirs=[], \ - defines={}, \ - compile_args={'no_extern_c', True}, jit_compile_args={}): - """ - Helper function to print compilation output - """ - def cuda_compile_message_handler(compile_success_bool, info_str, error_str): - self.logger.debug("Compilation returned %s", str(compile_success_bool)) - if info_str: - self.logger.debug("Info: %s", info_str) - if error_str: - self.logger.debug("Error: %s", error_str) - - kernel_filename = os.path.normpath(kernel_filename) - kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) - #self.logger.debug("Getting %s", kernel_filename) - - # Create a hash of the kernel options - options_hasher = hashlib.md5() - options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); - options_hash = options_hasher.hexdigest() - - # Create hash of kernel souce - source_hash = CudaContext.hash_kernel( \ - kernel_path, \ - include_dirs=[self.module_path] + include_dirs) - - # Create final hash - root, ext = os.path.splitext(kernel_filename) - kernel_hash = root \ - + "_" + source_hash \ - + "_" + options_hash \ - + ext - cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) - - # If we have the kernel in our hashmap, return it - if (kernel_hash in self.modules.keys()): - self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) - return self.modules[kernel_hash] - - # If we have it on disk, return it - elif (self.use_cache and os.path.isfile(cached_kernel_filename)): - self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) - - with io.open(cached_kernel_filename, "rb") as file: - file_str = file.read() - #No hip counterpart of module_from_buffer - module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) - - self.modules[kernel_hash] = module - return module - - # Otherwise, compile it from source - else: - self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) - - #Create kernel string - kernel_string = "" - for key, value in defines.items(): - kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) - kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) - if (self.use_cache): - cached_kernel_dir = os.path.dirname(cached_kernel_filename) - if not os.path.isdir(cached_kernel_dir): - os.mkdir(cached_kernel_dir) - with io.open(cached_kernel_filename + ".txt", "w") as file: - file.write(kernel_string) - - - with Common.Timer("compiler") as timer: - import warnings - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) - - #cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) - #module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) - - #HIP version of compilation: but "name_of_fct" needs to be defined. e.g. - #source = b"""\ - #extern "C" __global__ void name_of_fct(float factor, int n, short unused1, int unused2, float unused3, float *x) { - #int tid = threadIdx.x + blockIdx.x * blockDim.x; - #if (tid < n) { - #x[tid] *= factor; - # } - #} - #""" - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_string, b"name_of_fct", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - #kernel = hip_check(hip.hipModuleGetFunction(module, b"name_of_fct")) - - if (self.use_cache): - with io.open(cached_kernel_filename, "wb") as file: - file.write(cubin) - - self.modules[kernel_hash] = module - return module - - """ - Clears the kernel cache (useful for debugging & development) - """ - def clear_kernel_cache(self): - self.logger.debug("Clearing cache") - self.modules = {} - gc.collect() - - """ - Synchronizes all streams etc - """ - def synchronize(self): - self.cuda_context.synchronize() diff --git a/GPUSimulators/CudaContext_cu.py b/GPUSimulators/CudaContext_cu.py deleted file mode 100644 index 6c90636..0000000 --- a/GPUSimulators/CudaContext_cu.py +++ /dev/null @@ -1,272 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements Cuda context handling - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - - -import os - -import numpy as np -import time -import re -import io -import hashlib -import logging -import gc - -import pycuda.compiler as cuda_compiler -import pycuda.gpuarray -import pycuda.driver as cuda - -from GPUSimulators import Autotuner, Common - - - -""" -Class which keeps track of the CUDA context and some helper functions -""" -class CudaContext(object): - - def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): - """ - Create a new CUDA context - Set device to an id or pci_bus_id to select a specific GPU - Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context - """ - self.use_cache = use_cache - self.logger = logging.getLogger(__name__) - self.modules = {} - - self.module_path = os.path.dirname(os.path.realpath(__file__)) - - #Initialize cuda (must be first call to PyCUDA) - cuda.init(flags=0) - - self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) - - #Print some info about CUDA - self.logger.info("CUDA version %s", str(cuda.get_version())) - self.logger.info("Driver version %s", str(cuda.get_driver_version())) - - if device is None: - device = 0 - - self.cuda_device = cuda.Device(device) - self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) - self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) - - # Create the CUDA context - if context_flags is None: - context_flags=cuda.ctx_flags.SCHED_AUTO - - self.cuda_context = self.cuda_device.make_context(flags=context_flags) - - free, total = cuda.mem_get_info() - self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) - - self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) - - #Create cache dir for cubin files - self.cache_path = os.path.join(self.module_path, "cuda_cache") - if (self.use_cache): - if not os.path.isdir(self.cache_path): - os.mkdir(self.cache_path) - self.logger.info("Using CUDA cache dir %s", self.cache_path) - - self.autotuner = None - if (autotuning): - self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") - self.autotuner = Autotuner.Autotuner() - - - def __del__(self, *args): - self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) - - # Loop over all contexts in stack, and remove "this" - other_contexts = [] - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() - if (context.handle != self.cuda_context.handle): - self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) - other_contexts = [context] + other_contexts - cuda.Context.pop() - else: - self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) - cuda.Context.pop() - - # Add all the contexts we popped that were not our own - for context in other_contexts: - self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) - cuda.Context.push(context) - - self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) - self.cuda_context.detach() - - - def __str__(self): - return "CudaContext id " + str(self.cuda_context.handle) - - - def hash_kernel(kernel_filename, include_dirs): - # Generate a kernel ID for our caches - num_includes = 0 - max_includes = 100 - kernel_hasher = hashlib.md5() - logger = logging.getLogger(__name__) - - # Loop over file and includes, and check if something has changed - files = [kernel_filename] - while len(files): - - if (num_includes > max_includes): - raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) - - filename = files.pop() - - #logger.debug("Hashing %s", filename) - - modified = os.path.getmtime(filename) - - # Open the file - with io.open(filename, "r") as file: - - # Search for #inclue and also hash the file - file_str = file.read() - kernel_hasher.update(file_str.encode('utf-8')) - kernel_hasher.update(str(modified).encode('utf-8')) - - #Find all includes - includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) - - # Loop over everything that looks like an include - for include_file in includes: - - #Search through include directories for the file - file_path = os.path.dirname(filename) - for include_path in [file_path] + include_dirs: - - # If we find it, add it to list of files to check - temp_path = os.path.join(include_path, include_file) - if (os.path.isfile(temp_path)): - files = files + [temp_path] - num_includes = num_includes + 1 #For circular includes... - break - - return kernel_hasher.hexdigest() - - - """ - Reads a text file and creates an OpenCL kernel from that - """ - def get_module(self, kernel_filename, - include_dirs=[], \ - defines={}, \ - compile_args={'no_extern_c', True}, jit_compile_args={}): - """ - Helper function to print compilation output - """ - def cuda_compile_message_handler(compile_success_bool, info_str, error_str): - self.logger.debug("Compilation returned %s", str(compile_success_bool)) - if info_str: - self.logger.debug("Info: %s", info_str) - if error_str: - self.logger.debug("Error: %s", error_str) - - kernel_filename = os.path.normpath(kernel_filename) - kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) - #self.logger.debug("Getting %s", kernel_filename) - - # Create a hash of the kernel options - options_hasher = hashlib.md5() - options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); - options_hash = options_hasher.hexdigest() - - # Create hash of kernel souce - source_hash = CudaContext.hash_kernel( \ - kernel_path, \ - include_dirs=[self.module_path] + include_dirs) - - # Create final hash - root, ext = os.path.splitext(kernel_filename) - kernel_hash = root \ - + "_" + source_hash \ - + "_" + options_hash \ - + ext - cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) - - # If we have the kernel in our hashmap, return it - if (kernel_hash in self.modules.keys()): - self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) - return self.modules[kernel_hash] - - # If we have it on disk, return it - elif (self.use_cache and os.path.isfile(cached_kernel_filename)): - self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) - - with io.open(cached_kernel_filename, "rb") as file: - file_str = file.read() - module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) - - self.modules[kernel_hash] = module - return module - - # Otherwise, compile it from source - else: - self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) - - #Create kernel string - kernel_string = "" - for key, value in defines.items(): - kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) - kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) - if (self.use_cache): - cached_kernel_dir = os.path.dirname(cached_kernel_filename) - if not os.path.isdir(cached_kernel_dir): - os.mkdir(cached_kernel_dir) - with io.open(cached_kernel_filename + ".txt", "w") as file: - file.write(kernel_string) - - - with Common.Timer("compiler") as timer: - import warnings - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) - cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) - module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) - if (self.use_cache): - with io.open(cached_kernel_filename, "wb") as file: - file.write(cubin) - - self.modules[kernel_hash] = module - return module - - """ - Clears the kernel cache (useful for debugging & development) - """ - def clear_kernel_cache(self): - self.logger.debug("Clearing cache") - self.modules = {} - gc.collect() - - """ - Synchronizes all streams etc - """ - def synchronize(self): - self.cuda_context.synchronize() \ No newline at end of file diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py deleted file mode 100644 index 935eb90..0000000 --- a/GPUSimulators/EE2D_KP07_dimsplit.py +++ /dev/null @@ -1,575 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the 2nd order HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - - - - - -""" -Class that solves the SW equations using the Forward-Backward linear scheme -""" -class EE2D_KP07_dimsplit (BaseSimulator): - - """ - Initialization routine - rho: Density - rho_u: Momentum along x-axis - rho_v: Momentum along y-axis - E: energy - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis - dy: Grid cell spacing along y-axis - dt: Size of each timestep - g: Gravitational constant - gamma: Gas constant - p: pressure - """ - - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - rho, rho_u, rho_v, E, - nx, ny, - dx, dy, - g, - gamma, - theta=1.3, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=8): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 2, - block_width, block_height) - self.g = np.float32(g) - self.gamma = np.float32(gamma) - self.theta = np.float32(theta) - - #Get kernels - #module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", - # defines={ - # 'BLOCK_WIDTH': self.block_size[0], - # 'BLOCK_HEIGHT': self.block_size[1] - # }, - # compile_args={ - # 'no_extern_c': True, - # 'options': ["--use_fast_math"], - # }, - # jit_compile_args={}) - #self.kernel = module.get_function("KP07DimsplitKernel") - #self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") - # - kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [rho, rho_u, rho_v, E]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [None, None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - # init device array cfl_data - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho))) - dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho))) - self.dt = min(dt_x, dt_y) - self.cfl_data.fill(self.dt, stream=self.stream) - - - def substep(self, dt, step_number, external=True, internal=True): - self.substepDimsplit(0.5*dt, step_number, external, internal) - - def substepDimsplit(self, dt, substep, external, internal): - if external and internal: - #print("COMPLETE DOMAIN (dt=" + str(dt) + ")") - -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# self.nx, self.ny) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - 0, 0, - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--External & Internal: Launching Kernel is ok") - - return - - if external and not internal: - ################################### - # XXX: Corners are treated twice! # - ################################### - - ns_grid_size = (self.grid_size[0], 1) - - # NORTH - # (x0, y0) x (x1, y1) - # (0, ny-y_halo) x (nx, ny) -# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, self.ny - int(self.u0[0].y_halo), -# self.nx, self.ny) - - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *ns_grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - 0, ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo), - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) - ) - ) - - - # SOUTH - # (x0, y0) x (x1, y1) - # (0, 0) x (nx, y_halo) -# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# self.nx, int(self.u0[0].y_halo)) - - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *ns_grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - 0, 0, - ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo) - ) - ) - ) - - - we_grid_size = (1, self.grid_size[1]) - - # WEST - # (x0, y0) x (x1, y1) - # (0, 0) x (x_halo, ny) -# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# int(self.u0[0].x_halo), self.ny) - - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *we_grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - 0, 0, - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ) - ) - ) - - - # EAST - # (x0, y0) x (x1, y1) - # (nx-x_halo, 0) x (nx, ny) -# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# self.nx - int(self.u0[0].x_halo), 0, -# self.nx, self.ny) - - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *we_grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), 0, - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--External and not Internal: Launching Kernel is ok") - - return - - if internal and not external: - - # INTERNAL DOMAIN - # (x0, y0) x (x1, y1) - # (x_halo, y_halo) x (nx - x_halo, ny - y_halo) - self.kernel.prepared_async_call(self.grid_size, self.block_size, self.internal_stream, - self.nx, self.ny, - self.dx, self.dy, dt, - self.g, - self.gamma, - self.theta, - substep, - self.boundary_conditions, - self.u0[0].data.gpudata, self.u0[0].data.strides[0], - self.u0[1].data.gpudata, self.u0[1].data.strides[0], - self.u0[2].data.gpudata, self.u0[2].data.strides[0], - self.u0[3].data.gpudata, self.u0[3].data.strides[0], - self.u1[0].data.gpudata, self.u1[0].data.strides[0], - self.u1[1].data.gpudata, self.u1[1].data.strides[0], - self.u1[2].data.gpudata, self.u1[2].data.strides[0], - self.u1[3].data.gpudata, self.u1[3].data.strides[0], - self.cfl_data.gpudata, - int(self.u0[0].x_halo), int(self.u0[0].y_halo), - self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo)) - - - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.internal_stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.gamma), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), - self.cfl_data, - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.u0[0].y_halo), - ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo) - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Internal and not External: Launching Kernel is ok") - return - - def swapBuffers(self): - self.u0, self.u1 = self.u1, self.u0 - return - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - return - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py deleted file mode 100644 index 092711a..0000000 --- a/GPUSimulators/FORCE.py +++ /dev/null @@ -1,242 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the FORCE flux -for the shallow water equations - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - - - - - - -""" -Class that solves the SW equations -""" -class FORCE (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 1, - block_width, block_height) - self.g = np.float32(g) - - #Get kernels -# module = context.get_module("cuda/SWE2D_FORCE.cu.hip", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("FORCEKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_FORCE.cu')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .FORCEKernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"FORCEKernel")) - - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) -# self.u0, self.u1 = self.u1, self.u0 - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - self.u0, self.u1 = self.u1, self.u0 - - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .FORCEKernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py deleted file mode 100644 index 792d3c6..0000000 --- a/GPUSimulators/HLL.py +++ /dev/null @@ -1,235 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - - -""" -Class that solves the SW equations using the Harten-Lax -van Leer approximate Riemann solver -""" -class HLL (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 1, - block_width, block_height); - self.g = np.float32(g) - - #Get kernels -# module = context.get_module("cuda/SWE2D_HLL.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("HLLKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .HLLKernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"HLLKernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .HLLKernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py deleted file mode 100644 index b5c0dc0..0000000 --- a/GPUSimulators/HLL2.py +++ /dev/null @@ -1,247 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the 2nd order HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - - - - -""" -Class that solves the SW equations using the Forward-Backward linear scheme -""" -class HLL2 (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - theta=1.8, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 2, - block_width, block_height); - self.g = np.float32(g) - self.theta = np.float32(theta) - - #Get kernels -# module = context.get_module("cuda/SWE2D_HLL2.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("HLL2Kernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL2.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .HLL2Kernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"HLL2Kernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): - self.substepDimsplit(dt*0.5, step_number) - - def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.theta), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .HLL2Kernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 - diff --git a/GPUSimulators/IPythonMagic.py b/GPUSimulators/IPythonMagic.py deleted file mode 100644 index fa452df..0000000 --- a/GPUSimulators/IPythonMagic.py +++ /dev/null @@ -1,193 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements helpers for IPython / Jupyter and CUDA - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import logging -import gc - -from IPython.core import magic_arguments -from IPython.core.magic import line_magic, Magics, magics_class -import pycuda.driver as cuda - -from GPUSimulators import Common, CudaContext - - -@magics_class -class MagicCudaContext(Magics): - @line_magic - @magic_arguments.magic_arguments() - @magic_arguments.argument( - 'name', type=str, help='Name of context to create') - @magic_arguments.argument( - '--blocking', '-b', action="store_true", help='Enable blocking context') - @magic_arguments.argument( - '--no_cache', '-nc', action="store_true", help='Disable caching of kernels') - @magic_arguments.argument( - '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels') - def cuda_context_handler(self, line): - args = magic_arguments.parse_argstring(self.cuda_context_handler, line) - self.logger = logging.getLogger(__name__) - - self.logger.info("Registering %s in user workspace", args.name) - - context_flags = None - if (args.blocking): - context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC - - if args.name in self.shell.user_ns.keys(): - self.logger.debug("Context already registered! Ignoring") - return - else: - self.logger.debug("Creating context") - use_cache = False if args.no_cache else True - use_autotuning = False if args.no_autotuning else True - self.shell.user_ns[args.name] = CudaContext.CudaContext(context_flags=context_flags, use_cache=use_cache, autotuning=use_autotuning) - - # this function will be called on exceptions in any cell - def custom_exc(shell, etype, evalue, tb, tb_offset=None): - self.logger.exception("Exception caught: Resetting to CUDA context %s", args.name) - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() - self.logger.info("Popping <%s>", str(context.handle)) - cuda.Context.pop() - - if args.name in self.shell.user_ns.keys(): - self.logger.info("Pushing <%s>", str(self.shell.user_ns[args.name].cuda_context.handle)) - self.shell.user_ns[args.name].cuda_context.push() - else: - self.logger.error("No CUDA context called %s found (something is wrong)", args.name) - self.logger.error("CUDA will not work now") - - self.logger.debug("==================================================================") - - # still show the error within the notebook, don't just swallow it - shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset) - - # this registers a custom exception handler for the whole current notebook - get_ipython().set_custom_exc((Exception,), custom_exc) - - - # Handle CUDA context when exiting python - import atexit - def exitfunc(): - self.logger.info("Exitfunc: Resetting CUDA context stack") - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() - self.logger.info("`-> Popping <%s>", str(context.handle)) - cuda.Context.pop() - self.logger.debug("==================================================================") - atexit.register(exitfunc) - - - - - - - - -@magics_class -class MagicLogger(Magics): - logger_initialized = False - - @line_magic - @magic_arguments.magic_arguments() - @magic_arguments.argument( - 'name', type=str, help='Name of context to create') - @magic_arguments.argument( - '--out', '-o', type=str, default='output.log', help='The filename to store the log to') - @magic_arguments.argument( - '--level', '-l', type=int, default=20, help='The level of logging to screen [0, 50]') - @magic_arguments.argument( - '--file_level', '-f', type=int, default=10, help='The level of logging to file [0, 50]') - def setup_logging(self, line): - if (self.logger_initialized): - logging.getLogger('GPUSimulators').info("Global logger already initialized!") - return; - else: - self.logger_initialized = True - - args = magic_arguments.parse_argstring(self.setup_logging, line) - import sys - - #Get root logger - logger = logging.getLogger('GPUSimulators') - logger.setLevel(min(args.level, args.file_level)) - - #Add log to screen - ch = logging.StreamHandler() - ch.setLevel(args.level) - logger.addHandler(ch) - logger.log(args.level, "Console logger using level %s", logging.getLevelName(args.level)) - - #Get the outfilename (try to evaluate if Python expression...) - try: - outfile = eval(args.out, self.shell.user_global_ns, self.shell.user_ns) - except: - outfile = args.out - - #Add log to file - logger.log(args.level, "File logger using level %s to %s", logging.getLevelName(args.file_level), outfile) - - fh = logging.FileHandler(outfile) - formatter = logging.Formatter('%(asctime)s:%(name)s:%(levelname)s: %(message)s') - fh.setFormatter(formatter) - fh.setLevel(args.file_level) - logger.addHandler(fh) - - logger.info("Python version %s", sys.version) - self.shell.user_ns[args.name] = logger - - - - - - -@magics_class -class MagicMPI(Magics): - - @line_magic - @magic_arguments.magic_arguments() - @magic_arguments.argument( - 'name', type=str, help='Name of context to create') - @magic_arguments.argument( - '--num_engines', '-n', type=int, default=4, help='Number of engines to start') - def setup_mpi(self, line): - args = magic_arguments.parse_argstring(self.setup_mpi, line) - logger = logging.getLogger('GPUSimulators') - if args.name in self.shell.user_ns.keys(): - logger.warning("MPI alreay set up, resetting") - self.shell.user_ns[args.name].shutdown() - self.shell.user_ns[args.name] = None - gc.collect() - self.shell.user_ns[args.name] = Common.IPEngine(args.num_engines) - - - - - - - - -# Register -ip = get_ipython() -ip.register_magics(MagicCudaContext) -ip.register_magics(MagicLogger) -ip.register_magics(MagicMPI) - diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py deleted file mode 100644 index 93ce5e9..0000000 --- a/GPUSimulators/KP07.py +++ /dev/null @@ -1,252 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - -""" -Class that solves the SW equations using the Forward-Backward linear scheme -""" -class KP07 (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - theta=1.3, - cfl_scale=0.9, - order=2, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - order, - block_width, block_height); - self.g = np.float32(g) - self.theta = np.float32(theta) - self.order = np.int32(order) - - #Get kernels -# module = context.get_module("cuda/SWE2D_KP07.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("KP07Kernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .KP07Kernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07Kernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - - def substep(self, dt, step_number): - self.substepRK(dt, step_number) - - - def substepRK(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# Simulator.stepOrderToCodedInt(step=substep, order=self.order), -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.theta), - Simulator.stepOrderToCodedInt(step=substep, order=self.order), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .KP07Kernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - return max_dt*0.5**(self.order-1) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py deleted file mode 100644 index 0a5cfc7..0000000 --- a/GPUSimulators/KP07_dimsplit.py +++ /dev/null @@ -1,251 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - -""" -Class that solves the SW equations using the dimentionally split KP07 scheme -""" -class KP07_dimsplit(Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - theta=1.3, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 2, - block_width, block_height) - self.gc_x = 2 - self.gc_y = 2 - self.g = np.float32(g) - self.theta = np.float32(theta) - - #Get kernels -# module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("KP07DimsplitKernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07_dimsplit.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .KP07DimsplitKernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - self.gc_x, self.gc_y, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - self.gc_x, self.gc_y, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): - self.substepDimsplit(dt*0.5, step_number) - - def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_float(self.theta), - ctypes.c_int(substep) - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - - self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .KP07DimsplitKernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py deleted file mode 100644 index 98e54c6..0000000 --- a/GPUSimulators/LxF.py +++ /dev/null @@ -1,238 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the classical Lax-Friedrichs numerical -scheme for the shallow water equations - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - - - -""" -Class that solves the SW equations using the Lax Friedrichs scheme -""" -class LxF (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 1, - block_width, block_height); - self.g = np.float32(g) - - # Get kernels -# module = context.get_module("cuda/SWE2D_LxF.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("LxFKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_LxF.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .LxFKernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"LxFKernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 1, 1, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .LxFKernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py deleted file mode 100644 index f13de52..0000000 --- a/GPUSimulators/MPISimulator.py +++ /dev/null @@ -1,535 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements MPI simulator class - -Copyright (C) 2018 SINTEF Digital - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - -import logging -from GPUSimulators import Simulator -import numpy as np -from mpi4py import MPI -import time - -#import pycuda.driver as cuda -#import nvtx -from hip import hip, hiprtc - - -class MPIGrid(object): - """ - Class which represents an MPI grid of nodes. Facilitates easy communication between - neighboring nodes - """ - def __init__(self, comm, ndims=2): - self.logger = logging.getLogger(__name__) - - assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" - assert comm.size >= 1, "Must have at least one node" - - self.grid = MPIGrid.getGrid(comm.size, ndims) - self.comm = comm - - self.logger.debug("Created MPI grid: {:}. Rank {:d} has coordinate {:}".format( - self.grid, self.comm.rank, self.getCoordinate())) - - def getCoordinate(self, rank=None): - if (rank is None): - rank = self.comm.rank - i = (rank % self.grid[0]) - j = (rank // self.grid[0]) - return i, j - - def getRank(self, i, j): - return j*self.grid[0] + i - - def getEast(self): - i, j = self.getCoordinate(self.comm.rank) - i = (i+1) % self.grid[0] - return self.getRank(i, j) - - def getWest(self): - i, j = self.getCoordinate(self.comm.rank) - i = (i+self.grid[0]-1) % self.grid[0] - return self.getRank(i, j) - - def getNorth(self): - i, j = self.getCoordinate(self.comm.rank) - j = (j+1) % self.grid[1] - return self.getRank(i, j) - - def getSouth(self): - i, j = self.getCoordinate(self.comm.rank) - j = (j+self.grid[1]-1) % self.grid[1] - return self.getRank(i, j) - - def getGrid(num_nodes, num_dims): - assert(isinstance(num_nodes, int)) - assert(isinstance(num_dims, int)) - - # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors - # Original code by https://stackoverflow.com/users/3928385/ishamael - # Factorizes a number into n roughly equal factors - - #Dictionary to remember already computed permutations - memo = {} - def dp(n, left): # returns tuple (cost, [factors]) - """ - Recursively searches through all factorizations - """ - - #Already tried: return existing result - if (n, left) in memo: - return memo[(n, left)] - - #Spent all factors: return number itself - if left == 1: - return (n, [n]) - - #Find new factor - i = 2 - best = n - bestTuple = [n] - while i * i < n: - #If factor found - if n % i == 0: - #Factorize remainder - rem = dp(n // i, left - 1) - - #If new permutation better, save it - if rem[0] + i < best: - best = rem[0] + i - bestTuple = [i] + rem[1] - i += 1 - - #Store calculation - memo[(n, left)] = (best, bestTuple) - return memo[(n, left)] - - - grid = dp(num_nodes, num_dims)[1] - - if (len(grid) < num_dims): - #Split problematic 4 - if (4 in grid): - grid.remove(4) - grid.append(2) - grid.append(2) - - #Pad with ones to guarantee num_dims - grid = grid + [1]*(num_dims - len(grid)) - - #Sort in descending order - grid = np.sort(grid) - grid = grid[::-1] - - # XXX: We only use vertical (north-south) partitioning for now - grid[0] = 1 - grid[1] = num_nodes - - return grid - - - def gather(self, data, root=0): - out_data = None - if (self.comm.rank == root): - out_data = np.empty([self.comm.size] + list(data.shape), dtype=data.dtype) - self.comm.Gather(data, out_data, root) - return out_data - - def getLocalRank(self): - """ - Returns the local rank on this node for this MPI process - """ - - # This function has been adapted from - # https://github.com/SheffieldML/PyDeepGP/blob/master/deepgp/util/parallel.py - # by Zhenwen Dai released under BSD 3-Clause "New" or "Revised" License: - # - # Copyright (c) 2016, Zhenwen Dai - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are met: - # - # * Redistributions of source code must retain the above copyright notice, this - # list of conditions and the following disclaimer. - # - # * Redistributions in binary form must reproduce the above copyright notice, - # this list of conditions and the following disclaimer in the documentation - # and/or other materials provided with the distribution. - # - # * Neither the name of DGP nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - #Get this ranks unique (physical) node name - node_name = MPI.Get_processor_name() - - #Gather the list of all node names on all nodes - node_names = self.comm.allgather(node_name) - - #Loop over all node names up until our rank - #and count how many duplicates of our nodename we find - local_rank = len([0 for name in node_names[:self.comm.rank] if name==node_name]) - - return local_rank - - -class MPISimulator(Simulator.BaseSimulator): - """ - Class which handles communication between simulators on different MPI nodes - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, sim, grid): - self.profiling_data_mpi = { 'start': {}, 'end': {} } - self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0 - self.profiling_data_mpi["end"]["t_mpi_halo_exchange"] = 0 - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] = 0 - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] = 0 - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] = 0 - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] = 0 - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] = 0 - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] = 0 - self.profiling_data_mpi["start"]["t_mpi_step"] = 0 - self.profiling_data_mpi["end"]["t_mpi_step"] = 0 - self.profiling_data_mpi["n_time_steps"] = 0 - self.logger = logging.getLogger(__name__) - - autotuner = sim.context.autotuner - sim.context.autotuner = None; - boundary_conditions = sim.getBoundaryConditions() - super().__init__(sim.context, - sim.nx, sim.ny, - sim.dx, sim.dy, - boundary_conditions, - sim.cfl_scale, - sim.num_substeps, - sim.block_size[0], sim.block_size[1]) - sim.context.autotuner = autotuner - - self.sim = sim - self.grid = grid - - #Get neighbor node ids - self.east = grid.getEast() - self.west = grid.getWest() - self.north = grid.getNorth() - self.south = grid.getSouth() - - #Get coordinate of this node - #and handle global boundary conditions - new_boundary_conditions = Simulator.BoundaryCondition({ - 'north': Simulator.BoundaryCondition.Type.Dirichlet, - 'south': Simulator.BoundaryCondition.Type.Dirichlet, - 'east': Simulator.BoundaryCondition.Type.Dirichlet, - 'west': Simulator.BoundaryCondition.Type.Dirichlet - }) - gi, gj = grid.getCoordinate() - #print("gi: " + str(gi) + ", gj: " + str(gj)) - if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): - self.west = None - new_boundary_conditions.west = boundary_conditions.west; - if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): - self.south = None - new_boundary_conditions.south = boundary_conditions.south; - if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): - self.east = None - new_boundary_conditions.east = boundary_conditions.east; - if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): - self.north = None - new_boundary_conditions.north = boundary_conditions.north; - sim.setBoundaryConditions(new_boundary_conditions) - - #Get number of variables - self.nvars = len(self.getOutput().gpu_variables) - - #Shorthands for computing extents and sizes - gc_x = int(self.sim.getOutput()[0].x_halo) - gc_y = int(self.sim.getOutput()[0].y_halo) - nx = int(self.sim.nx) - ny = int(self.sim.ny) - - #Set regions for ghost cells to read from - #These have the format [x0, y0, width, height] - self.read_e = np.array([ nx, 0, gc_x, ny + 2*gc_y]) - self.read_w = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) - self.read_n = np.array([gc_x, ny, nx, gc_y]) - self.read_s = np.array([gc_x, gc_y, nx, gc_y]) - - #Set regions for ghost cells to write to - self.write_e = self.read_e + np.array([gc_x, 0, 0, 0]) - self.write_w = self.read_w - np.array([gc_x, 0, 0, 0]) - self.write_n = self.read_n + np.array([0, gc_y, 0, 0]) - self.write_s = self.read_s - np.array([0, gc_y, 0, 0]) - - #Allocate data for receiving - #Note that east and west also transfer ghost cells - #whilst north/south only transfer internal cells - #Reuses the width/height defined in the read-extets above - ##self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) - - ##self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) - ##self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) - ##self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) - - self.in_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) - num_bytes_e = self.in_e.size * self.in_e.itemsize - #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can be accessed directly by the GPU device - #hipHostMallocDefault:Memory is mapped and portable (default allocation) - #hipHostMallocPortable: memory is explicitely portable across different devices - self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) - - self.in_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) - num_bytes_w = self.in_w.size * self.in_w.itemsize - self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) - - self.in_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) - num_bytes_n = self.in_n.size * self.in_n.itemsize - self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) - - self.in_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) - num_bytes_s = self.in_s.size * self.in_s.itemsize - self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) - - #Allocate data for sending - #self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) - #self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) - #self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) - #self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) - - self.out_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) - num_bytes_e = self.out_e.size * self.out_e.itemsize - self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) - - self.out_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) - num_bytes_w = self.out_w.size * self.out_w.itemsize - self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) - - self.out_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) - num_bytes_n = self.out_n.size * self.out_n.itemsize - self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) - - self.out_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) - num_bytes_s = self.out_s.size * self.out_s.itemsize - self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) - - - self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) - - self.full_exchange() - sim.context.synchronize() - - def substep(self, dt, step_number): - - #nvtx.mark("substep start", color="yellow") - - self.profiling_data_mpi["start"]["t_mpi_step"] += time.time() - - #nvtx.mark("substep external", color="blue") - self.sim.substep(dt, step_number, external=True, internal=False) # only "internal ghost cells" - - #nvtx.mark("substep internal", color="red") - self.sim.substep(dt, step_number, internal=True, external=False) # "internal ghost cells" excluded - - #nvtx.mark("substep full", color="blue") - #self.sim.substep(dt, step_number, external=True, internal=True) - - self.sim.swapBuffers() - - self.profiling_data_mpi["end"]["t_mpi_step"] += time.time() - - #nvtx.mark("exchange", color="blue") - self.full_exchange() - - #nvtx.mark("sync start", color="blue") - #self.sim.stream.synchronize() - #self.sim.internal_stream.synchronize() - hip_check(hip.hipStreamSynchronize(self.sim.stream)) - hip_check(hip.hipStreamSynchronize(self.sim.internal_stream)) - #nvtx.mark("sync end", color="blue") - - self.profiling_data_mpi["n_time_steps"] += 1 - - def getOutput(self): - return self.sim.getOutput() - - def synchronize(self): - self.sim.synchronize() - - def check(self): - return self.sim.check() - - def computeDt(self): - local_dt = np.array([np.float32(self.sim.computeDt())]); - global_dt = np.empty(1, dtype=np.float32) - self.grid.comm.Allreduce(local_dt, global_dt, op=MPI.MIN) - self.logger.debug("Local dt: {:f}, global dt: {:f}".format(local_dt[0], global_dt[0])) - return global_dt[0] - - - def getExtent(self): - """ - Function which returns the extent of node with rank - rank in the grid - """ - width = self.sim.nx*self.sim.dx - height = self.sim.ny*self.sim.dy - i, j = self.grid.getCoordinate() - x0 = i * width - y0 = j * height - x1 = x0 + width - y1 = y0 + height - return [x0, x1, y0, y1] - - def full_exchange(self): - #### - # First transfer internal cells north-south - #### - - #Download from the GPU - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() - - if self.north is not None: - for k in range(self.nvars): - self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_n[k,:,:], asynch=True, extent=self.read_n) - if self.south is not None: - for k in range(self.nvars): - self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_s[k,:,:], asynch=True, extent=self.read_s) - #self.sim.stream.synchronize() - hip_check(hip.hipStreamSynchronize(self.sim.stream)) - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() - - #Send/receive to north/south neighbours - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - comm_send = [] - comm_recv = [] - if self.north is not None: - comm_send += [self.grid.comm.Isend(self.out_n, dest=self.north, tag=4*self.nt + 0)] - comm_recv += [self.grid.comm.Irecv(self.in_n, source=self.north, tag=4*self.nt + 1)] - if self.south is not None: - comm_send += [self.grid.comm.Isend(self.out_s, dest=self.south, tag=4*self.nt + 1)] - comm_recv += [self.grid.comm.Irecv(self.in_s, source=self.south, tag=4*self.nt + 0)] - - #Wait for incoming transfers to complete - for comm in comm_recv: - comm.wait() - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - #Upload to the GPU - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() - - if self.north is not None: - for k in range(self.nvars): - self.sim.u0[k].upload(self.sim.stream, self.in_n[k,:,:], extent=self.write_n) - if self.south is not None: - for k in range(self.nvars): - self.sim.u0[k].upload(self.sim.stream, self.in_s[k,:,:], extent=self.write_s) - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() - - #Wait for sending to complete - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - for comm in comm_send: - comm.wait() - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - #### - # Then transfer east-west including ghost cells that have been filled in by north-south transfer above - #### - - #Download from the GPU - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() - - if self.east is not None: - for k in range(self.nvars): - self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_e[k,:,:], asynch=True, extent=self.read_e) - if self.west is not None: - for k in range(self.nvars): - self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_w[k,:,:], asynch=True, extent=self.read_w) - #self.sim.stream.synchronize() - hip_check(hip.hipStreamSynchronize(self.sim.stream)) - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() - - #Send/receive to east/west neighbours - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - comm_send = [] - comm_recv = [] - if self.east is not None: - comm_send += [self.grid.comm.Isend(self.out_e, dest=self.east, tag=4*self.nt + 2)] - comm_recv += [self.grid.comm.Irecv(self.in_e, source=self.east, tag=4*self.nt + 3)] - if self.west is not None: - comm_send += [self.grid.comm.Isend(self.out_w, dest=self.west, tag=4*self.nt + 3)] - comm_recv += [self.grid.comm.Irecv(self.in_w, source=self.west, tag=4*self.nt + 2)] - - #Wait for incoming transfers to complete - for comm in comm_recv: - comm.wait() - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - #Upload to the GPU - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() - - if self.east is not None: - for k in range(self.nvars): - self.sim.u0[k].upload(self.sim.stream, self.in_e[k,:,:], extent=self.write_e) - if self.west is not None: - for k in range(self.nvars): - self.sim.u0[k].upload(self.sim.stream, self.in_w[k,:,:], extent=self.write_w) - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() - - #Wait for sending to complete - self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() - - for comm in comm_send: - comm.wait() - - self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py deleted file mode 100644 index 1a608ed..0000000 --- a/GPUSimulators/SHMEMSimulator.py +++ /dev/null @@ -1,264 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements SHMEM simulator group class - -Copyright (C) 2020 Norwegian Meteorological Institute - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - -import logging -from GPUSimulators import Simulator, CudaContext -import numpy as np - -#import pycuda.driver as cuda -from hip import hip, hiprtc - -import time - -class SHMEMSimulator(Simulator.BaseSimulator): - """ - Class which handles communication and synchronization between simulators in different - contexts (presumably on different GPUs) - """ - def __init__(self, sims, grid): - self.logger = logging.getLogger(__name__) - - assert(len(sims) > 1) - - self.sims = sims - - # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? - # See also getOutput() and check(). - # - # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of - # SHMEMSimulators that have BaseSimulator as a superclass. - # - # This would also eliminate the need for all the array bookkeeping in this class. - autotuner = sims[0].context.autotuner - sims[0].context.autotuner = None - boundary_conditions = sims[0].getBoundaryConditions() - super().__init__(sims[0].context, - sims[0].nx, sims[0].ny, - sims[0].dx, sims[0].dy, - boundary_conditions, - sims[0].cfl_scale, - sims[0].num_substeps, - sims[0].block_size[0], sims[0].block_size[1]) - sims[0].context.autotuner = autotuner - - self.sims = sims - self.grid = grid - - self.east = [None] * len(self.sims) - self.west = [None] * len(self.sims) - self.north = [None] * len(self.sims) - self.south = [None] * len(self.sims) - - self.nvars = [None] * len(self.sims) - - self.read_e = [None] * len(self.sims) - self.read_w = [None] * len(self.sims) - self.read_n = [None] * len(self.sims) - self.read_s = [None] * len(self.sims) - - self.write_e = [None] * len(self.sims) - self.write_w = [None] * len(self.sims) - self.write_n = [None] * len(self.sims) - self.write_s = [None] * len(self.sims) - - self.e = [None] * len(self.sims) - self.w = [None] * len(self.sims) - self.n = [None] * len(self.sims) - self.s = [None] * len(self.sims) - - for i, sim in enumerate(self.sims): - #Get neighbor subdomain ids - self.east[i] = grid.getEast(i) - self.west[i] = grid.getWest(i) - self.north[i] = grid.getNorth(i) - self.south[i] = grid.getSouth(i) - - #Get coordinate of this subdomain - #and handle global boundary conditions - new_boundary_conditions = Simulator.BoundaryCondition({ - 'north': Simulator.BoundaryCondition.Type.Dirichlet, - 'south': Simulator.BoundaryCondition.Type.Dirichlet, - 'east': Simulator.BoundaryCondition.Type.Dirichlet, - 'west': Simulator.BoundaryCondition.Type.Dirichlet - }) - gi, gj = grid.getCoordinate(i) - if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): - self.west = None - new_boundary_conditions.west = boundary_conditions.west; - if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): - self.south = None - new_boundary_conditions.south = boundary_conditions.south; - if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): - self.east = None - new_boundary_conditions.east = boundary_conditions.east; - if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): - self.north = None - new_boundary_conditions.north = boundary_conditions.north; - sim.setBoundaryConditions(new_boundary_conditions) - - #Get number of variables - self.nvars[i] = len(sim.getOutput().gpu_variables) - - #Shorthands for computing extents and sizes - gc_x = int(sim.getOutput()[0].x_halo) - gc_y = int(sim.getOutput()[0].y_halo) - nx = int(sim.nx) - ny = int(sim.ny) - - #Set regions for ghost cells to read from - #These have the format [x0, y0, width, height] - self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) - self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) - self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) - self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) - - #Set regions for ghost cells to write to - self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) - self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) - self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) - self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) - - #Allocate host data - #Note that east and west also transfer ghost cells - #whilst north/south only transfer internal cells - #Reuses the width/height defined in the read-extets above - self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) - self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) - self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) - self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) - - self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) - - - def substep(self, dt, step_number): - self.exchange() - - for i, sim in enumerate(self.sims): - sim.substep(dt, step_number) - - def getOutput(self): - # XXX: Does not return what we would expect. - # Returns first subdomain, but we want the whole domain. - return self.sims[0].getOutput() - - def synchronize(self): - for sim in self.sims: - sim.synchronize() - - def check(self): - # XXX: Does not return what we would expect. - # Checks only first subdomain, but we want to check the whole domain. - return self.sims[0].check() - - def computeDt(self): - global_dt = float("inf") - - for sim in self.sims: - sim.context.synchronize() - - for sim in self.sims: - local_dt = sim.computeDt() - if local_dt < global_dt: - global_dt = local_dt - self.logger.debug("Local dt: {:f}".format(local_dt)) - - self.logger.debug("Global dt: {:f}".format(global_dt)) - return global_dt - - def getExtent(self, index=0): - """ - Function which returns the extent of the subdomain with index - index in the grid - """ - width = self.sims[index].nx*self.sims[index].dx - height = self.sims[index].ny*self.sims[index].dy - i, j = self.grid.getCoordinate(index) - x0 = i * width - y0 = j * height - x1 = x0 + width - y1 = y0 + height - return [x0, x1, y0, y1] - - def exchange(self): - #### - # First transfer internal cells north-south - #### - for i in range(len(self.sims)): - self.ns_download(i) - - for i in range(len(self.sims)): - self.ns_upload(i) - - #### - # Then transfer east-west including ghost cells that have been filled in by north-south transfer above - #### - for i in range(len(self.sims)): - self.ew_download(i) - - for i in range(len(self.sims)): - self.ew_upload(i) - - def ns_download(self, i): - #Download from the GPU - if self.north[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) - if self.south[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) - self.sims[i].stream.synchronize() - - def ns_upload(self, i): - #Upload to the GPU - if self.north[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) - if self.south[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) - - def ew_download(self, i): - #Download from the GPU - if self.east[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) - if self.west[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) - self.sims[i].stream.synchronize() - - def ew_upload(self, i): - #Upload to the GPU - if self.east[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) - #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) - #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) - if self.west[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) - #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) - #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/SHMEMSimulatorGroup.py b/GPUSimulators/SHMEMSimulatorGroup.py deleted file mode 100644 index c9dc30f..0000000 --- a/GPUSimulators/SHMEMSimulatorGroup.py +++ /dev/null @@ -1,413 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements SHMEM simulator group class - -Copyright (C) 2020 Norwegian Meteorological Institute - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - -import logging -from GPUSimulators import Simulator, CudaContext -import numpy as np - -#import pycuda.driver as cuda -from hip import hip, hiprtc - -import time - -class SHMEMGrid(object): - """ - Class which represents an SHMEM grid of GPUs. Facilitates easy communication between - neighboring subdomains in the grid. Contains one CUDA context per subdomain. - """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, ngpus=None, ndims=2): - self.logger = logging.getLogger(__name__) - - #cuda.init(flags=0) - self.logger.info("Initializing HIP") - #num_cuda_devices = cuda.Device.count() - num_cuda_devices = hip_check(hip.hipGetDeviceCount()) - - if ngpus is None: - ngpus = num_cuda_devices - - # XXX: disabled for testing on single-GPU system - #assert ngpus <= num_cuda_devices, "Trying to allocate more GPUs than are available in the system." - #assert ngpus >= 2, "Must have at least two GPUs available to run multi-GPU simulations." - - assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" - - self.ngpus = ngpus - self.ndims = ndims - - self.grid = SHMEMGrid.getGrid(self.ngpus, self.ndims) - - self.logger.debug("Created {:}-dimensional SHMEM grid, using {:} GPUs".format( - self.ndims, self.ngpus)) - - # XXX: Is this a natural place to store the contexts? Consider moving contexts out of this - # class, into notebook / calling script (shmemTesting.py) - self.cuda_contexts = [] - - for i in range(self.ngpus): - # XXX: disabled for testing on single-GPU system - #self.cuda_contexts.append(CudaContext.CudaContext(device=i, autotuning=False)) - self.cuda_contexts.append(CudaContext.CudaContext(device=0, autotuning=False)) - - def getCoordinate(self, index): - i = (index % self.grid[0]) - j = (index // self.grid[0]) - return i, j - - def getIndex(self, i, j): - return j*self.grid[0] + i - - def getEast(self, index): - i, j = self.getCoordinate(index) - i = (i+1) % self.grid[0] - return self.getIndex(i, j) - - def getWest(self, index): - i, j = self.getCoordinate(index) - i = (i+self.grid[0]-1) % self.grid[0] - return self.getIndex(i, j) - - def getNorth(self, index): - i, j = self.getCoordinate(index) - j = (j+1) % self.grid[1] - return self.getIndex(i, j) - - def getSouth(self, index): - i, j = self.getCoordinate(index) - j = (j+self.grid[1]-1) % self.grid[1] - return self.getIndex(i, j) - - def getGrid(num_gpus, num_dims): - assert(isinstance(num_gpus, int)) - assert(isinstance(num_dims, int)) - - # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors - # Original code by https://stackoverflow.com/users/3928385/ishamael - # Factorizes a number into n roughly equal factors - - #Dictionary to remember already computed permutations - memo = {} - def dp(n, left): # returns tuple (cost, [factors]) - """ - Recursively searches through all factorizations - """ - - #Already tried: return existing result - if (n, left) in memo: - return memo[(n, left)] - - #Spent all factors: return number itself - if left == 1: - return (n, [n]) - - #Find new factor - i = 2 - best = n - bestTuple = [n] - while i * i < n: - #If factor found - if n % i == 0: - #Factorize remainder - rem = dp(n // i, left - 1) - - #If new permutation better, save it - if rem[0] + i < best: - best = rem[0] + i - bestTuple = [i] + rem[1] - i += 1 - - #Store calculation - memo[(n, left)] = (best, bestTuple) - return memo[(n, left)] - - - grid = dp(num_gpus, num_dims)[1] - - if (len(grid) < num_dims): - #Split problematic 4 - if (4 in grid): - grid.remove(4) - grid.append(2) - grid.append(2) - - #Pad with ones to guarantee num_dims - grid = grid + [1]*(num_dims - len(grid)) - - #Sort in descending order - grid = np.sort(grid) - grid = grid[::-1] - - return grid - -class SHMEMSimulatorGroup(object): - """ - Class which handles communication and synchronization between simulators in different - contexts (typically on different GPUs) - """ - def __init__(self, sims, grid): - self.logger = logging.getLogger(__name__) - - assert(len(sims) > 1) - - self.sims = sims - - # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? - # See also getOutput() and check(). - # - # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of - # SHMEMSimulators that have BaseSimulator as a superclass. - # - # This would also eliminate the need for all the array bookkeeping in this class. - # - CONT HERE! Model shmemTesting after mpiTesting and divide existing functionality between SHMEMSimulatorGroup and SHMEMSimulator - - autotuner = sims[0].context.autotuner - sims[0].context.autotuner = None - boundary_conditions = sims[0].getBoundaryConditions() - super().__init__(sims[0].context, - sims[0].nx, sims[0].ny, - sims[0].dx, sims[0].dy, - boundary_conditions, - sims[0].cfl_scale, - sims[0].num_substeps, - sims[0].block_size[0], sims[0].block_size[1]) - sims[0].context.autotuner = autotuner - - self.sims = sims - self.grid = grid - - self.east = [None] * len(self.sims) - self.west = [None] * len(self.sims) - self.north = [None] * len(self.sims) - self.south = [None] * len(self.sims) - - self.nvars = [None] * len(self.sims) - - self.read_e = [None] * len(self.sims) - self.read_w = [None] * len(self.sims) - self.read_n = [None] * len(self.sims) - self.read_s = [None] * len(self.sims) - - self.write_e = [None] * len(self.sims) - self.write_w = [None] * len(self.sims) - self.write_n = [None] * len(self.sims) - self.write_s = [None] * len(self.sims) - - self.e = [None] * len(self.sims) - self.w = [None] * len(self.sims) - self.n = [None] * len(self.sims) - self.s = [None] * len(self.sims) - - for i, sim in enumerate(self.sims): - #Get neighbor subdomain ids - self.east[i] = grid.getEast(i) - self.west[i] = grid.getWest(i) - self.north[i] = grid.getNorth(i) - self.south[i] = grid.getSouth(i) - - #Get coordinate of this subdomain - #and handle global boundary conditions - new_boundary_conditions = Simulator.BoundaryCondition({ - 'north': Simulator.BoundaryCondition.Type.Dirichlet, - 'south': Simulator.BoundaryCondition.Type.Dirichlet, - 'east': Simulator.BoundaryCondition.Type.Dirichlet, - 'west': Simulator.BoundaryCondition.Type.Dirichlet - }) - gi, gj = grid.getCoordinate(i) - if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): - self.west = None - new_boundary_conditions.west = boundary_conditions.west; - if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): - self.south = None - new_boundary_conditions.south = boundary_conditions.south; - if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): - self.east = None - new_boundary_conditions.east = boundary_conditions.east; - if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): - self.north = None - new_boundary_conditions.north = boundary_conditions.north; - sim.setBoundaryConditions(new_boundary_conditions) - - #Get number of variables - self.nvars[i] = len(sim.getOutput().gpu_variables) - - #Shorthands for computing extents and sizes - gc_x = int(sim.getOutput()[0].x_halo) - gc_y = int(sim.getOutput()[0].y_halo) - nx = int(sim.nx) - ny = int(sim.ny) - - #Set regions for ghost cells to read from - #These have the format [x0, y0, width, height] - self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) - self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) - self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) - self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) - - #Set regions for ghost cells to write to - self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) - self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) - self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) - self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) - - #Allocate host data - #Note that east and west also transfer ghost cells - #whilst north/south only transfer internal cells - #Reuses the width/height defined in the read-extets above - self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) - self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) - self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) - self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) - - self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) - - - def substep(self, dt, step_number): - self.exchange() - - for i, sim in enumerate(self.sims): - sim.substep(dt, step_number) - - def getOutput(self): - # XXX: Does not return what we would expect. - # Returns first subdomain, but we want the whole domain. - return self.sims[0].getOutput() - - def synchronize(self): - for sim in self.sims: - sim.synchronize() - - def check(self): - # XXX: Does not return what we would expect. - # Checks only first subdomain, but we want to check the whole domain. - return self.sims[0].check() - - def computeDt(self): - global_dt = float("inf") - - for sim in self.sims: - sim.context.synchronize() - - for sim in self.sims: - local_dt = sim.computeDt() - if local_dt < global_dt: - global_dt = local_dt - self.logger.debug("Local dt: {:f}".format(local_dt)) - - self.logger.debug("Global dt: {:f}".format(global_dt)) - return global_dt - - def getExtent(self, index=0): - """ - Function which returns the extent of the subdomain with index - index in the grid - """ - width = self.sims[index].nx*self.sims[index].dx - height = self.sims[index].ny*self.sims[index].dy - i, j = self.grid.getCoordinate(index) - x0 = i * width - y0 = j * height - x1 = x0 + width - y1 = y0 + height - return [x0, x1, y0, y1] - - def exchange(self): - #### - # First transfer internal cells north-south - #### - for i in range(len(self.sims)): - self.ns_download(i) - - for i in range(len(self.sims)): - self.ns_upload(i) - - #### - # Then transfer east-west including ghost cells that have been filled in by north-south transfer above - #### - for i in range(len(self.sims)): - self.ew_download(i) - - for i in range(len(self.sims)): - self.ew_upload(i) - - def ns_download(self, i): - #Download from the GPU - if self.north[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) - if self.south[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) - #self.sims[i].stream.synchronize() - hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) - - - def ns_upload(self, i): - #Upload to the GPU - if self.north[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) - if self.south[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) - - def ew_download(self, i): - #Download from the GPU - if self.east[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) - if self.west[i] is not None: - for k in range(self.nvars[i]): - # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) - self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) - #self.sims[i].stream.synchronize() - hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) - - def ew_upload(self, i): - #Upload to the GPU - if self.east[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) - #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) - #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) - if self.west[i] is not None: - for k in range(self.nvars[i]): - self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) - #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) - #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/Simulator.py b/GPUSimulators/Simulator.py deleted file mode 100644 index b804d79..0000000 --- a/GPUSimulators/Simulator.py +++ /dev/null @@ -1,286 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the classical Lax-Friedrichs numerical -scheme for the shallow water equations - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -import numpy as np -import logging -from enum import IntEnum - -#import pycuda.compiler as cuda_compiler -#import pycuda.gpuarray -#import pycuda.driver as cuda - -from hip import hip, hiprtc - -from GPUSimulators import Common - - -class BoundaryCondition(object): - """ - Class for holding boundary conditions for global boundaries - """ - - - class Type(IntEnum): - """ - Enum that describes the different types of boundary conditions - WARNING: MUST MATCH THAT OF common.h IN CUDA - """ - Dirichlet = 0, - Neumann = 1, - Periodic = 2, - Reflective = 3 - - def __init__(self, types={ - 'north': Type.Reflective, - 'south': Type.Reflective, - 'east': Type.Reflective, - 'west': Type.Reflective - }): - """ - Constructor - """ - self.north = types['north'] - self.south = types['south'] - self.east = types['east'] - self.west = types['west'] - - if (self.north == BoundaryCondition.Type.Neumann \ - or self.south == BoundaryCondition.Type.Neumann \ - or self.east == BoundaryCondition.Type.Neumann \ - or self.west == BoundaryCondition.Type.Neumann): - raise(NotImplementedError("Neumann boundary condition not supported")) - - def __str__(self): - return '[north={:s}, south={:s}, east={:s}, west={:s}]'.format(str(self.north), str(self.south), str(self.east), str(self.west)) - - - def asCodedInt(self): - """ - Helper function which packs four boundary conditions into one integer - """ - bc = 0 - bc = bc | (self.north & 0x0000000F) << 24 - bc = bc | (self.south & 0x0000000F) << 16 - bc = bc | (self.east & 0x0000000F) << 8 - bc = bc | (self.west & 0x0000000F) << 0 - - #for t in types: - # print("{0:s}, {1:d}, {1:032b}, {1:08b}".format(t, types[t])) - #print("bc: {0:032b}".format(bc)) - - return np.int32(bc) - - def getTypes(bc): - types = {} - types['north'] = BoundaryCondition.Type((bc >> 24) & 0x0000000F) - types['south'] = BoundaryCondition.Type((bc >> 16) & 0x0000000F) - types['east'] = BoundaryCondition.Type((bc >> 8) & 0x0000000F) - types['west'] = BoundaryCondition.Type((bc >> 0) & 0x0000000F) - return types - - - -class BaseSimulator(object): - - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - num_substeps, - block_width, block_height): - """ - Initialization routine - context: GPU context to use - kernel_wrapper: wrapper function of GPU kernel - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - cfl_scale: Courant number - num_substeps: Number of substeps to perform for a full step - """ - #Get logger - self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__) - - #Save input parameters - #Notice that we need to specify them in the correct dataformat for the - #GPU kernel - self.context = context - self.nx = np.int32(nx) - self.ny = np.int32(ny) - self.dx = np.float32(dx) - self.dy = np.float32(dy) - self.setBoundaryConditions(boundary_conditions) - self.cfl_scale = cfl_scale - self.num_substeps = num_substeps - - #Handle autotuning block size - if (self.context.autotuner): - peak_configuration = self.context.autotuner.get_peak_performance(self.__class__) - block_width = int(peak_configuration["block_width"]) - block_height = int(peak_configuration["block_height"]) - self.logger.debug("Used autotuning to get block size [%d x %d]", block_width, block_height) - - #Compute kernel launch parameters - self.block_size = (block_width, block_height, 1) - self.grid_size = ( - int(np.ceil(self.nx / float(self.block_size[0]))), - int(np.ceil(self.ny / float(self.block_size[1]))) - ) - - #Create a CUDA stream - #self.stream = cuda.Stream() - #self.internal_stream = cuda.Stream() - self.stream = hip_check(hip.hipStreamCreate()) - self.internal_stream = hip_check(hip.hipStreamCreate()) - - #Keep track of simulation time and number of timesteps - self.t = 0.0 - self.nt = 0 - - - def __str__(self): - return "{:s} [{:d}x{:d}]".format(self.__class__.__name__, self.nx, self.ny) - - - def simulate(self, t, dt=None): - """ - Function which simulates t_end seconds using the step function - Requires that the step() function is implemented in the subclasses - """ - - printer = Common.ProgressPrinter(t) - - t_start = self.simTime() - t_end = t_start + t - - update_dt = True - if (dt is not None): - update_dt = False - self.dt = dt - - while(self.simTime() < t_end): - # Update dt every 100 timesteps and cross your fingers it works - # for the next 100 - if (update_dt and (self.simSteps() % 100 == 0)): - self.dt = self.computeDt()*self.cfl_scale - - # Compute timestep for "this" iteration (i.e., shorten last timestep) - current_dt = np.float32(min(self.dt, t_end-self.simTime())) - - # Stop if end reached (should not happen) - if (current_dt <= 0.0): - self.logger.warning("Timestep size {:d} is less than or equal to zero!".format(self.simSteps())) - break - - # Step forward in time - self.step(current_dt) - - #Print info - print_string = printer.getPrintString(self.simTime() - t_start) - if (print_string): - self.logger.info("%s: %s", self, print_string) - try: - self.check() - except AssertionError as e: - e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),) - raise - - - def step(self, dt): - """ - Function which performs one single timestep of size dt - """ - for i in range(self.num_substeps): - self.substep(dt, i) - - self.t += dt - self.nt += 1 - - def download(self, variables=None): - return self.getOutput().download(self.stream, variables) - - def synchronize(self): - #self.stream.synchronize() - #Synchronize the stream to ensure operations in the stream is complete - hip_check(hip.hipStreamSynchronize(self.stream)) - - def simTime(self): - return self.t - - def simSteps(self): - return self.nt - - def getExtent(self): - return [0, 0, self.nx*self.dx, self.ny*self.dy] - - def setBoundaryConditions(self, boundary_conditions): - self.logger.debug("Boundary conditions set to {:s}".format(str(boundary_conditions))) - self.boundary_conditions = boundary_conditions.asCodedInt() - - def getBoundaryConditions(self): - return BoundaryCondition(BoundaryCondition.getTypes(self.boundary_conditions)) - - def substep(self, dt, step_number): - """ - Function which performs one single substep with stepsize dt - """ - raise(NotImplementedError("Needs to be implemented in subclass")) - - def getOutput(self): - raise(NotImplementedError("Needs to be implemented in subclass")) - - def check(self): - self.logger.warning("check() is not implemented - please implement") - #raise(NotImplementedError("Needs to be implemented in subclass")) - - def computeDt(self): - raise(NotImplementedError("Needs to be implemented in subclass")) - - - - -def stepOrderToCodedInt(step, order): - """ - Helper function which packs the step and order into a single integer - """ - step_order = (step << 16) | (order & 0x0000ffff) - #print("Step: {0:032b}".format(step)) - #print("Order: {0:032b}".format(order)) - #print("Mix: {0:032b}".format(step_order)) - return np.int32(step_order) diff --git a/GPUSimulators/WAF.py b/GPUSimulators/WAF.py deleted file mode 100644 index 7e2763c..0000000 --- a/GPUSimulators/WAF.py +++ /dev/null @@ -1,241 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements the Weighted average flux (WAF) described in -E. Toro, Shock-Capturing methods for free-surface shallow flows, 2001 - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -#Import packages we need -from GPUSimulators import Simulator, Common -from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition -import numpy as np -import ctypes - -#from pycuda import gpuarray -from hip import hip,hiprtc - - - -""" -Class that solves the SW equations using the Forward-Backward linear scheme -""" -class WAF (Simulator.BaseSimulator): - - """ - Initialization routine - h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells - hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells - hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells - nx: Number of cells along x-axis - ny: Number of cells along y-axis - dx: Grid cell spacing along x-axis (20 000 m) - dy: Grid cell spacing along y-axis (20 000 m) - dt: Size of each timestep (90 s) - g: Gravitational accelleration (9.81 m/s^2) - """ - - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - - def __init__(self, - context, - h0, hu0, hv0, - nx, ny, - dx, dy, - g, - cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), - block_width=16, block_height=16): - - # Call super constructor - super().__init__(context, - nx, ny, - dx, dy, - boundary_conditions, - cfl_scale, - 2, - block_width, block_height); - self.g = np.float32(g) - - #Get kernels -# module = context.get_module("cuda/SWE2D_WAF.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("WAFKernel") -# self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_WAF.cu.hip')) - with open(kernel_file_path, 'r') as file: - kernel_source = file.read() - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 0, [], [])) - - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel .WAFKernel. for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - - kernel = hip_check(hip.hipModuleGetFunction(module, b"WAFKernel")) - - #Create data by uploading to device - self.u0 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [h0, hu0, hv0]) - self.u1 = Common.ArakawaA2D(self.stream, - nx, ny, - 2, 2, - [None, None, None]) - #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - - dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) - dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) - dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - def substep(self, dt, step_number): - self.substepDimsplit(dt*0.5, step_number) - - def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) - - #launch kernel - hip_check( - hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, - stream=self.stream, - kernelParams=None, - extra=( # pass kernel's arguments - ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), - ctypes.c_float(self.g), - ctypes.c_int(substep), - ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) - ) - ) - - hip_check(hip.hipDeviceSynchronize()) - - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .WAFKernel. is ok") - - def getOutput(self): - return self.u0 - - def check(self): - self.u0.check() - self.u1.check() - - # computing min with hipblas: the output is an index - def min_hipblas(self, num_elements, cfl_data, stream): - num_bytes = num_elements * np.dtype(np.float32).itemsize - num_bytes_i = np.dtype(np.int32).itemsize - indx_d = hip_check(hip.hipMalloc(num_bytes_i)) - indx_h = np.zeros(1, dtype=np.int32) - x_temp = np.zeros(num_elements, dtype=np.float32) - - #print("--size.data:", cfl_data.size) - handle = hip_check(hipblas.hipblasCreate()) - - #hip_check(hipblas.hipblasGetStream(handle, stream)) - #"incx" [int] specifies the increment for the elements of x. incx must be > 0. - hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) - - # destruction of handle - hip_check(hipblas.hipblasDestroy(handle)) - - # copy result (stored in indx_d) back to the host (store in indx_h) - hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) - hip_check(hip.hipStreamSynchronize(stream)) - - min_value = x_temp.flatten()[indx_h[0]-1] - - # clean up - hip_check(hip.hipStreamDestroy(stream)) - hip_check(hip.hipFree(cfl_data)) - return min_value - - def computeDt(self): - #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 diff --git a/GPUSimulators/__init__.py b/GPUSimulators/__init__.py deleted file mode 100644 index 4e04e36..0000000 --- a/GPUSimulators/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/env python -# -*- coding: utf-8 -*- - - -# Nothing general to do diff --git a/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc b/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc deleted file mode 100644 index da4daacc64c69c69b772807cf18375b7c8cf127b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11058 zcmb_iOK=-UdY%`6!4RYS*uv09;PGHib$-8;*6=r=nJH)=_{+;GFt zvX)xyE$LTlk$rZlXuop)!W;JL^2*wcoA!0T>PK$FN@ta=s&tz+7N|(iv%^*;+H|FN z$=+&p?6Mo!(i46dNq@Z)VcE!b1F?v80Eb`MvQW_pgeUE&=Gl=ao1xvRaN*X$jf zMA^RBUm%_BoZXVvSvMkprQK@Npdz5$vVoAjaSI1Gytgl5`vC)8YqhbB8fL@BHvLA! zUia)y=v6w6a~3dQzrVcp_Ivl%?5isu*zaGxd-v+f+6QlTqmMeD9iq6>~AiG?PqV}c5#m&vlg=)}QXvyj#PMS_RT)eVi zeFa`oR*GtzU+q&U3OY!(OfADd9fh~>GJkmr_(~~hLM`dyw9tj|SS=Z1PGm(6xr{JH z{;^WZ3QLUPn-c{wj;|>u#4&vHVp1H(*Ai3W1ioY9j5sM~FtQ*{iCNT)iwyW{KF$G@ zThbTXp=IN@MBHO<*8Fk}IBQEUWEKM(5SLZakx&=3gcmN@H{G(|053&eXnStB1=Kd1 zoxm?M;ev9aP0tH>-oOJduD7HQobqU{{R+&jlxgGrALUKFOL)T?l1S;Pk+!RL6-Vy^ z9rv|x{vAbVkJOLUu39&Qem@f~pvIt@DAUVQZCBgXFe2Bbab{Pk>%F|l^ep6Zc+L7) zSLqdYm0f*Tg@96kitLKaV7=`(?*(C}-EPSU)Dm=>>tG>lS@;k(#DCcLyPYs1zKAwk zHaLa2x7i|D^p#38oZ)sRJG3@{naFMh9`Aem;*#`$FfqXACHohbb{6crZg3wtu>*Py zfykB=et?eDS{#pajaIb^1$`PT?rQ( z*gZ(&!Xh}L?zYdG)m)pU3YfuPqY&EZ+13Kk|OoYIK_3tvmOen%`c;cfbhY zB4}z*T4-<4j%bYI{1D59lSmY`piQb%c(a)KObfGZ@fz6%)EP~&_L;8x$VjLsyX@-%p4})@^+RspO2)H-EDnCVANV6#F z6rK@~sr_h>X`_dM%-{lxryw?F6X28MKL9*oBiX?-lsgK{_q{N>h7JQT=SRV$_M^c( z0<)mW6tjKW9KtM7BxCkSxJ(e%3WQee)-Yhe-+*EI9gc?V2m}%t>O_V>pgMv;wCtU} zvU`6{> z1u(Opf+VSQ8q7|-c3+b(p+>2qZWZK<1xZe!yaVvN8U%xa&HM^|v>64uP?MkYmbEz^ zKdU^_b~4=zZNIC5zI62de6LJ058H#ePNz$A3iWktaN~`x-ZcoS1fI)EKSz)ijqPRD z-()e+@2*$wz&L&2HMU?Cxw2e?MG)0wt5dDnE{w>ETZV1!Z?j>LSQ^Pl+fL}3j+n&; zi-lq~R)g4Rc$G+!e2a~Buv95e=~@Sdf2>Qd`BaNF(Thu0oC92qK*uR$UU10#wJ?|W?tef&@tQOmayvp z_{e=IdvHwqWIqBWY3G4VARS)?d>>AdweqQg6EcMI7utpa7}Oyq0r@YSj}5W`pQ<1) ztjPx99xue!WkNn|zxI`X#P`^(bNmZ z3)=SdVZZ_nbf`rYo>jt-6EcsA*m9gCJ^-V*;5Zu{w~?C2ImZ#Lvg628=$BX~Y>GTf z#pft_o{|?RAqJFhQbOD^M2T#&Fptt{QQweCrfL{g!OEGsnMt1|E5~+t8OXQkfW*e9T(QUPQ#6&HZF zfo;8gkLhmMxvL?J(SVdb)MeWXry!aU!1t+CJPH4XofZ~hJl}|vA*7LwOdAZRaXdaA zp?(RC4p!iKG9>e(G#klf2S@FIJ;U1-F}e5Ca9REO8XJBGJ_cf3nuCh#7^J4yC1f5%QM zyB5hvlz1M@ew<8sX9@sqZFXo3YIw{n z)nm*o@cn$3jj!khSe4(~RqEe|sirtTh+YKk9H$zBqW(k3V{>2KS=8l|y1IqDmFCBg zAB&2;m%0{c$Lx-Y@ot_>xGHdmxqrlM=AmuplD6(x*Xrie{x7F>`JuW6KsT62XhNgD zWXU^=Xh~jy@>JT&6vLq@!DV8p`Eh=oH&{N%uhZ`;SE z10%j|A8#HQ@ooFKd|<@4?c=97DWi8qO!j^cxvBbF^{dgf-V$O;a12zOa_{=Cf-j^% z?*`>K%5;3CsKFM3eFKXT7Gk2#mmxDtCnBfW_MI96kB;}S3;~Uh&O2y{=MQOtHrosu zEmy>Ihqb~;rnct}YePs+r18+$N;-s**NzK;6CujwP$6X7w`d>&L{23@Cf6-CD3}!+ zo3O(ZlbEEvyohA`MQEEO0?(EZhq5S`;;&3BZVI$-kptoi2M!2-H`6aWCa4eN z+@?fK4jt0WWqbq zyO(@6lTVO*LV1jEHc1H?j!8(3iRUVc@i^`m8Uw12$HJ|k4EHg>(G&|woa>Ki7#SGy zeI%t5>nRmDppujfN~Z#E(>bg`tNJCaS}z&Zvh$GBE%^b>RYC$kChs_WZv^{r&&DH5 z@n7+Vze0jQBuDZnnm7Dda49LuN1hp_8r)1no&4I!nOauO!cXJJFt`;x{>#i3w0XFq zrkbQLaz07nlIqjyG}bB@^Jqt2pHunCzM}G@r(f+^tjlr5ftfdmiH;a%gk=gkEa45m zL;`WeQAMF;l+$Dc!6AAbcH3oYAy*J?A>BBL%oBqI*+^ZsU)M8OywO}aDEdR@>SfX% zT0Kj~R63VA$ir|W2t+bIy@M}9@vA~JE9br5%&*+6%r?W zLL~a=gy>WdiiV4$@jj%BhNo)645P7AZdxM{Xck4?o*6d&DYB6Wm7^CWp6oB0Xe#+j zRP~o8{K!9}WZ%Nn@cz)kvm*-+aq-bhkM4JP**JqZQu+R&3x9-#OG67!jV#RLMwTQd zARWUfUPp%Sm6RUrB-{;wly}vt%CP{Yj%Zi}q8A!sAJm?$=js&w(4u_Z68aOJ33;qn z5Jqnt@dEfGU>KyS=I1<|Fx)6{%$C$6F8#Dtd;2-|bZrsQtxsj=oLz0S)^R`Fujo6o zB+_M>k|rfg$9Zz&aR+@pl0T+OmW6b1asFzffm`ZM*-PAY{oUmiR-t=Rx9^mjkmzp1 z0ZHeRrIFo#sE;w{hlE=?8bzH&jAaJ0XdhJ#pc%n4dEw+6GoDGf5>+A2L@A<`h-gqQ zjW(f7_(d?4Js2JFM+hI(X5d>uXv&ylMZD$Zpx!DQIWC{$f zF89JSBtPJ~rq`CqjV_xggTY)%FTm-xY|v25C9KD)f?|rSFXYcD$LJ@PhsdFLN1WO8 z1x%!DjjlSAxFz4q#M;C2C2i|Gz87L`>%u;Cv(mr8FnWY^(veDL5=fuutBn1q2SX=?5wf5d^#xEKp_&4_hskD;x6)B%Euy2q)#-KR!#DVF-~@LuhObF8AbX__=a*vK zA9JMpW356?HI1@_LAqdr9H23t?bqL7Hn`4*&NiOT*2QdCw>L4+F68bXqFvIxW%ui_ zDu@a%2k{6hIIJadzw5{9ll9~H=J6s*k92Ap8VHvf`6r0+Q`j5im-IV<@AODN3;o8@ ze$R1k0Ui$QmRsnI z3;Xea7Cz%IG`Vu*xtv0;e$n*BoGAcy{$bjII& zhZ&zeV#enNGk$~_|G)Sk*{dPfM`-m7R+t;`!NifYYLE0gHbSfYT^sT@BXd8?_~`Qi zjX9hTo*(H)FdWSX#>PLP_y2|ulKp4o&xTHBRNC$Ffqn2~u<8pb-DWoC8Df$v8TmKg zVZP@No$q^t`5J>g{a^SXv+?hPc@FRa+trh)UH$ve5No)NGRyEU>7u2S#Xoo8ZN=F{ zQ6>iUyXYogq69h#b|5Zk;Pc0N}+u0!mtx)F|Wc|7857sr#3%MIGdO-gQ2a*7giZP>X;a8E|Jq&>;c zDPb3CJoTK*9vt}8duAu$A$4H~>22ix1ur{DxGT{M74sR~mC$3F6O!7ClgYAVqsQYT z{}M_fEg3jp_(vxh#&H%AqgPSTQDEXKjkmWbXt5w9E}DFvsTQM!QWO%hgX7W#!bN>yAR3TRY diff --git a/GPUSimulators/__pycache__/Simulator.cpython-39.pyc b/GPUSimulators/__pycache__/Simulator.cpython-39.pyc deleted file mode 100644 index dc16706a34d26a4641268277e3f0d257f351d864..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8433 zcmbVR+ix3JdY>DI_|HP)7M^Ga1=ngC*SX>AHAB74nu)jRTkTver>A3f zJ6|hg^NKbI- zHCho1x^01)>pM}Sg$1k`O0^rMuY3^%3QW`L_HgJxTX1VRFrN8JtIYq0xXLj;jvE{H%qQA=>P z(Q|0+3Z1@|oz7N5?5YsC%7lk;$zBN=0WtR3@PKT$t1OhxoEsCsl7$_@z5*zF%vTi2 z*3BPB_y8}?yg&in3Ohifh24PIw&(k7L$GcnnqB{@0}8OGtLyh3K3Qi=)o1MK(&NWV z)%9n8g_&R}M#U?U?n7J!>%bH#-5~A(X6M13$IJIHf9b2$`>X5E05)4$U9a9*TVpE^ zAG0O)XzB6#>hhEOOOM&3CyyUJT)Q(5j)5$|-}w{dBuQw8QZO!J*YhKw`V7pA0FuvH z%Y7xl#D?%*0U(z(AQnRszI{{2^}}FOA%^2gYEwZ0M5|2}gz;4t0kWH|IPTn7SlHg) zp5F|*^P${a0H=w|(ZZMW&KoG5R;}nsel>{iK!#DZm&0~D3>q{v{>!0p3s3Yj6hf=% z&{k+^&4gwd+=OPC!h)vdL~hF-f4KE)tEnq()NA=v3u}c;|B8Hqm!O9ZpM)MxC8xep zq9uFFVZc2Smi`@w;kQhhr?jdS`rHHZ8`(^dESs8+Z~EZ|ggP7cM5NXq!$%+D!;av? zjhe-Q2e)cA&jDf^kmdVlk3lxS-s_0|Kd9{qL?U1+om@l>sH8}Z1~joLBy0$au|R3y z39yg0f!tPCt9Ng(2T#`4*@LC^<$G-X-qJdIxB^s(H}fsFT4l>mZZC}%Rdh)RRrEL6 z3-(NVW1^^53Q3_}58Sq>*Au5+Pc0kjrFtFa$j?Ud^*Rq5=q}#&$fo%sPV!aJZM#8` z6du94!K5@2=doz|q7g$}ztdzH2dLurKVRrHqXoYUTO>Ny7a)&YZl~j`3Z~b8`1y|( z@%!TQg%&nnSn&cc7C#Gp*v-3-p43}jX94duuib@R2xYV|s?K+MiAAKoio+suCk5TI zES>)344M@y$pxW|TS+d0nzs^5xKW%~+aih^M2F#z>hvt*iF_0To$D90fmWxQp=vs6 zrm7jJS*m6Zv@I>RwjkIqjOZJ*^g}J^aIpEL4J0{OMI||jT5`G?#;Zf0CiopG zLs>BrD-wP)$tn3fV5&StOCUVNta|-(w4(DUw4$S{KVwQSeQy}0KN@D~e+;XC@wko* z7dJ>Qhd-+4@hC>jp~Ce&E!II;;}+Ic;anRG}dt&FUT&!`JP~O;8^33g#ki`AQqcqOd5$tZ>Fgr+Poo6e^WqF zu}>mlVg#KehfO~Fv|=b~CdNkNC`BDNS`NA3@HSO!6_rqEQ01BQ&+cD3PTqqh$0%%J zwW9DhsCb}ftnGouT>ooD<$a~gH?^$S8tcuCdhM~^{HV7u)?2&@>i}9iRmpIUe4h%X zs~=EJDeQ+-Q(8-8Q>qIx5PYeHG9-duOsI6-=e+bG}V1rV_4{uXjHGA^s|0G{i6eHHaH z39IV`Vl<2pEEYTQ4F(@h9*PZ5S#+>{qv_Wp z+>dc}U4}Elt&8r=z03v9c<_MYOGfj|}X{#TgfScUG(@XMAKXltnHC@GHUmXyW`loT^8gsja1l6ptDFHk7*xIsfCCY*7S<6@(`nK&sG z2*Jd{Rnt#$>ik-96LM0|`n%k%*=va3-F9LlrlI#V#RW2O!MwCXj{$#NDWwt9bwcb@ zDn6s)M^sRln4BI48+Fhiy~suKFKPG-Dn6#-6BM->1vn-(y-kP(MJ{9BNjWBRU?HA} zE&$s3^Eq2DWq*!t>$X|Y%c@;AOU5Pr(jRRb(Wzfcdw^qQq+D3@GNNSQJK8 z--5B+qR`*kH(?MFd6p^UWMBW{2H)MG;&~+}-@yvGK!tL}2ef#X`VRc8!i2o=Z}1GG z$umdcQ;}q~K|Xc(Bfn0Mg74$ZSyA}63)uFIeQnnoz)_vkc5?_a?E!+vt}Yk3Np+a> zY}O$6Li?32KN{F_1M94Pgq&xQBYR<}c8(mvmOV>u!Hn~Jg}A7&&!uOa*IvT`?&b%$ z4ZqOkKVi(-o8b99a(38jAMohd|K&$$PY&|R`I&oD>FR-X9y4|egMwUCE2lYJ;-Elo ztxr3Y_cdL6s=ZvawD{Do^xlWMwp+y5{Ge!QTgcT7&q?>&H=6p7ag(dh{1U64A=uk3 z{$26qV)|ax7O|7c>pA^jvyp9BIUE>A=~a{N=yefr`fnIt#`pP6}wyYO*(ByMt8X?Ksue%LLvJQ6y&D3*;iz@<^MJ^;EKt zt0NZHreK5A*03LB9W5_thDi>&(8h~oYAK3@B-K{;&BSu$CR{^mLP)OVJruQkCxxw+ zk6~?G8P{@(HML^5gKVRy^SI_Tx{^{UR4=LdWSHngT^!Lig%xEJ#BZ^CG><~F9iyNZ z3`<9RtJ}tuIjfh=i!dN(b<1?j8T6KQ1IDDRvt#CCC^3g}3PjzJ`M8Njtc=HN{kHbJ zybA@~LmmOqqoHdr5z|4O%|R->P)xKep0+F{ux9_>NYIYRYi8S#KEiE{WE9!yk+X&O zRr12)(Q@U#s+C-lgR_M>C+bxoSLqwq)h=qqvi6}(_GHCniat5oa?<42_onEb&%85Sg#*nzAz{BkTPLazc zEXXdAwc;yRdhQ0~|0rb7FzerFe>zM%#;oCPhc1nDIiu+ct|?StjQ|x{AFi!w@+rD- zG69fGM7^NVl3@U+DNi!yHO!&}>JWenM~HH;{1^-bhl0UIgA1U?`X6J* z9q}ERI{!IG5&KWXY!w6lId;?$H##&H$Icq*!6hulhZ&Xc0D3yPmDHWiYMW^qCrwvc zY4D)p1{Fb2@Ng1LN>c9Z#8Ca;jX`$qFwJ3v(mBVZkqE6Q{sUA}uEWn8F_iH&e3$Z9 zI(ahqzZ}T|`TO^W0og&)ErOp=Hya8CkP0D#C`s+pK!6ZWui1A{%Y;}`{iyi<7=8bR zCn7!3V1N3T594xTJa~-cBP}aqmpDsxY{6H*M#>_UXEV)1sC&PH)nSLDs@5K3s0v8jDN6J=G;BkP3Y$$zlGy4xlrt1G-=`y>Oe(@lCbJX1Z{&H0a zxNo>cd=on2f3aHm-YPPE=K#qeqc=g@)ziz!Q7%2fv>{dB z8CCfl`$O2SDJh7wM4J(T8a${8b2K|DdJ<8c~Y zWBAO8Soxq6va4kK{@Vb3hyi?)9RYg!u=uLOjsf{r!pLvvoVQZtrx-;*I6CYIRYpNe z1zrXJGe`4c2RX1cI)}WPBnKkNh)Xi9l8m1GnhH`)`Ex3W{fZ-!u$2VGn1m3aRSr+i zq1MI|QGj1C3O^}W&eekBn1$h=g_@1WQgaWPCt}=)c?2kBFuzn^ByLfb*iiXPsI=0W zs?wT2mFpdbr1U8m;UazgMaUO2cNjHL060wb$`5d$_x<8q z={0ZzohM#;K5oy2VXZbv1Gq@01EZOnoi2U9?bYmbX)>8@fp7Boten{CTi&9RCJ%jG zQbBQIhZ|e?HkH~(;-La|48;+LlfupPv)Px#sfezL))_>SddVoIrCq*T{(J4}{{j@a BH8B7H diff --git a/GPUSimulators/__pycache__/__init__.cpython-39.pyc b/GPUSimulators/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index a966589566cf0036cb97a40facf080e9dc3106f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 182 zcmYe~<>g`kg7xe&sV+eJF^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5w0=Qav3^cz zaY<2XfuVjuQGQlpK|v0fk(yi*Z(?R@00fq1`WczY8TxLSd6^}tVfi_wxvA~}q461+ s1^PfbI5W32C$S{Is8~Nf9;75bUaz3?7Kcr4eoARhsvXGE&p^xo01!1T>Hq)$ diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu deleted file mode 100644 index 238718c..0000000 --- a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu +++ /dev/null @@ -1,250 +0,0 @@ - /* -This kernel implements the Central Upwind flux function to -solve the Euler equations - -Copyright (C) 2018 SINTEF Digital - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "EulerCommon.h" -#include "limiters.h" - - -__device__ -void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float gamma_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, gamma_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, gamma_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Gravity source term - if (g_ > 0.0f) { - const int i = threadIdx.x + gc_x; - const int j = threadIdx.y + gc_y; - const float rho_v = Q[2][j][i]; - Q[2][j][i] -= g_*Q[0][j][i]*dt_; - Q[3][j][i] -= g_*rho_v*dt_; - __syncthreads(); - } - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, gamma_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, gamma_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Gravity source term - if (g_ > 0.0f) { - const int i = threadIdx.x + gc_x; - const int j = threadIdx.y + gc_y; - const float rho_v = Q[2][j][i]; - Q[2][j][i] -= g_*Q[0][j][i]*dt_; - Q[3][j][i] -= g_*rho_v*dt_; - __syncthreads(); - } - } - - - // Write to main memory for all internal cells - writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); - } -} - - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip deleted file mode 100644 index 67b701b..0000000 --- a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip +++ /dev/null @@ -1,251 +0,0 @@ -#include "hip/hip_runtime.h" - /* -This kernel implements the Central Upwind flux function to -solve the Euler equations - -Copyright (C) 2018 SINTEF Digital - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "EulerCommon.h" -#include "limiters.h" - - -__device__ -void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float gamma_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, gamma_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, gamma_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Gravity source term - if (g_ > 0.0f) { - const int i = threadIdx.x + gc_x; - const int j = threadIdx.y + gc_y; - const float rho_v = Q[2][j][i]; - Q[2][j][i] -= g_*Q[0][j][i]*dt_; - Q[3][j][i] -= g_*rho_v*dt_; - __syncthreads(); - } - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, gamma_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, gamma_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Gravity source term - if (g_ > 0.0f) { - const int i = threadIdx.x + gc_x; - const int j = threadIdx.y + gc_y; - const float rho_v = Q[2][j][i]; - Q[2][j][i] -= g_*Q[0][j][i]*dt_; - Q[3][j][i] -= g_*rho_v*dt_; - __syncthreads(); - } - } - - - // Write to main memory for all internal cells - writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); - writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); - } -} - - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EulerCommon.h b/GPUSimulators/cuda/EulerCommon.h deleted file mode 100644 index cb22a53..0000000 --- a/GPUSimulators/cuda/EulerCommon.h +++ /dev/null @@ -1,187 +0,0 @@ -/* -These CUDA functions implement different types of numerical flux -functions for the shallow water equations - -Copyright (C) 2016, 2017, 2018 SINTEF Digital - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -#pragma once -#include "limiters.h" - - - -template -__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], - float shmem[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_, - const float dx_, const float dy_, const float gamma_, - float* output_) { - //Index of thread within block - const int tx = threadIdx.x + gc_x; - const int ty = threadIdx.y + gc_y; - - //Index of cell within domain - const int ti = blockDim.x*blockIdx.x + tx; - const int tj = blockDim.y*blockIdx.y + ty; - - //Only internal cells - if (ti < nx_+gc_x && tj < ny_+gc_y) { - const float rho = Q[0][ty][tx]; - const float u = Q[1][ty][tx] / rho; - const float v = Q[2][ty][tx] / rho; - - const float max_u = dx_ / (fabsf(u) + sqrtf(gamma_*rho)); - const float max_v = dy_ / (fabsf(v) + sqrtf(gamma_*rho)); - - shmem[ty][tx] = fminf(max_u, max_v); - } - __syncthreads(); - - //One row of threads loop over all rows - if (ti < nx_+gc_x && tj < ny_+gc_y) { - if (ty == gc_y) { - float min_val = shmem[ty][tx]; - const int max_y = min(h, ny_+gc_y - tj); - for (int j=gc_y; j h_l) ? q_l_tmp : 1.0f; - const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; - - // Compute wave speed estimates - const float S_l = u_l - c_l*q_l; - const float S_r = u_r + c_r*q_r; - - //Upwind selection - if (S_l >= 0.0f) { - return F_func(Q_l, P_l); - } - else if (S_r <= 0.0f) { - return F_func(Q_r, P_r); - } - //Or estimate flux in the star region - else { - const float4 F_l = F_func(Q_l, P_l); - const float4 F_r = F_func(Q_r, P_r); - const float4 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); - return flux; - } -} - - - - - - - -/** - * Central upwind flux function - */ -__device__ float4 CentralUpwindFlux(const float4 Qm, const float4 Qp, const float gamma) { - - const float Pp = pressure(Qp, gamma); - const float4 Fp = F_func(Qp, Pp); - const float up = Qp.y / Qp.x; // rho*u / rho - const float cp = sqrt(gamma*Pp/Qp.x); // sqrt(gamma*P/rho) - - const float Pm = pressure(Qm, gamma); - const float4 Fm = F_func(Qm, Pm); - const float um = Qm.y / Qm.x; // rho*u / rho - const float cm = sqrt(gamma*Pm/Qm.x); // sqrt(gamma*P/rho) - - const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed - const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed - - return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); -} \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu b/GPUSimulators/cuda/SWE2D_FORCE.cu deleted file mode 100644 index dac46be..0000000 --- a/GPUSimulators/cuda/SWE2D_FORCE.cu +++ /dev/null @@ -1,143 +0,0 @@ -/* -This OpenCL kernel implements the classical Lax-Friedrichs scheme -for the shallow water equations, with edge fluxes. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - const float g_, const float dx_, const float dt_) { - //Compute fluxes along the x axis - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - __syncthreads(); - - //Compute flux along x, and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute flux along y, and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Write to main memory - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu.hip b/GPUSimulators/cuda/SWE2D_FORCE.cu.hip deleted file mode 100644 index aa4e968..0000000 --- a/GPUSimulators/cuda/SWE2D_FORCE.cu.hip +++ /dev/null @@ -1,144 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the classical Lax-Friedrichs scheme -for the shallow water equations, with edge fluxes. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - const float g_, const float dx_, const float dt_) { - //Compute fluxes along the x axis - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - __syncthreads(); - - //Compute flux along x, and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute flux along y, and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Write to main memory - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu b/GPUSimulators/cuda/SWE2D_HLL.cu deleted file mode 100644 index 3ed6b35..0000000 --- a/GPUSimulators/cuda/SWE2D_HLL.cu +++ /dev/null @@ -1,161 +0,0 @@ -/* -This GPU kernel implements the HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" - - - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - const float g_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Compute F flux - computeFluxF(Q, F, g_); - __syncthreads(); - - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute G flux - computeFluxG(Q, F, g_); - __syncthreads(); - - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu.hip b/GPUSimulators/cuda/SWE2D_HLL.cu.hip deleted file mode 100644 index c2f449d..0000000 --- a/GPUSimulators/cuda/SWE2D_HLL.cu.hip +++ /dev/null @@ -1,162 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This GPU kernel implements the HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" - - - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - const float g_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Compute F flux - computeFluxF(Q, F, g_); - __syncthreads(); - - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute G flux - computeFluxG(Q, F, g_); - __syncthreads(); - - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu b/GPUSimulators/cuda/SWE2D_HLL2.cu deleted file mode 100644 index 94f92b5..0000000 --- a/GPUSimulators/cuda/SWE2D_HLL2.cu +++ /dev/null @@ -1,216 +0,0 @@ -/* -This OpenCL kernel implements the second order HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - - - - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - - - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu.hip b/GPUSimulators/cuda/SWE2D_HLL2.cu.hip deleted file mode 100644 index c0bc9d1..0000000 --- a/GPUSimulators/cuda/SWE2D_HLL2.cu.hip +++ /dev/null @@ -1,217 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the second order HLL flux - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - - - - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - - - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu b/GPUSimulators/cuda/SWE2D_KP07.cu deleted file mode 100644 index 6fa6154..0000000 --- a/GPUSimulators/cuda/SWE2D_KP07.cu +++ /dev/null @@ -1,233 +0,0 @@ -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], - const float g_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - { - int j=ty; - const int l = j + 2; //Skip ghost cells - for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - - //Reconstruct slopes along x and axis - minmodSlopeX(Q, Qx, theta_); - minmodSlopeY(Q, Qy, theta_); - __syncthreads(); - - - //Compute fluxes along the x and y axis - computeFluxF(Q, Qx, F, g_); - computeFluxG(Q, Qy, G, g_); - __syncthreads(); - - - //Sum fluxes and advance in time for all internal cells - if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { - const int i = tx + 2; //Skip local ghost cells, i.e., +2 - const int j = ty + 2; - - Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ - + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; - Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ - + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; - Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ - + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; - - float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); - float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); - float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); - - if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { - //Write to main memory - h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); - hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); - hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); - } - else { - h_row[ti] = Q[0][j][i]; - hu_row[ti] = Q[1][j][i]; - hv_row[ti] = Q[2][j][i]; - } - } - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} -} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu.hip b/GPUSimulators/cuda/SWE2D_KP07.cu.hip deleted file mode 100644 index fd9ef0d..0000000 --- a/GPUSimulators/cuda/SWE2D_KP07.cu.hip +++ /dev/null @@ -1,234 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], - float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], - const float g_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - { - int j=ty; - const int l = j + 2; //Skip ghost cells - for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - - //Reconstruct slopes along x and axis - minmodSlopeX(Q, Qx, theta_); - minmodSlopeY(Q, Qy, theta_); - __syncthreads(); - - - //Compute fluxes along the x and y axis - computeFluxF(Q, Qx, F, g_); - computeFluxG(Q, Qy, G, g_); - __syncthreads(); - - - //Sum fluxes and advance in time for all internal cells - if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { - const int i = tx + 2; //Skip local ghost cells, i.e., +2 - const int j = ty + 2; - - Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ - + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; - Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ - + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; - Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ - + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; - - float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); - float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); - float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); - - if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { - //Write to main memory - h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); - hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); - hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); - } - else { - h_row[ti] = Q[0][j][i]; - hu_row[ti] = Q[1][j][i]; - hv_row[ti] = Q[2][j][i]; - } - } - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} -} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu deleted file mode 100644 index ac256e3..0000000 --- a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu +++ /dev/null @@ -1,216 +0,0 @@ -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - -template -__device__ -void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], - float Qx[3][h+2*gc_y][w+2*gc_x], - float F[3][h+2*gc_y][w+2*gc_x], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j -__device__ -void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], - float Qy[3][h+2*gc_y][w+2*gc_x], - float G[3][h+2*gc_y][w+2*gc_x], - const float g_, const float dy_, const float dt_) { - for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - if (step_ == 0) { - //Along X - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Along Y - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - else { - //Along Y - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Along X - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - - - - - - - - - - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip deleted file mode 100644 index f366b0a..0000000 --- a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip +++ /dev/null @@ -1,217 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" -#include "limiters.h" - - -template -__device__ -void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], - float Qx[3][h+2*gc_y][w+2*gc_x], - float F[3][h+2*gc_y][w+2*gc_x], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j -__device__ -void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], - float Qy[3][h+2*gc_y][w+2*gc_x], - float G[3][h+2*gc_y][w+2*gc_x], - const float g_, const float dy_, const float dt_) { - for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - if (step_ == 0) { - //Along X - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Along Y - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - else { - //Along Y - minmodSlopeY(Q, Qx, theta_); - __syncthreads(); - computeFluxG(Q, Qx, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Along X - minmodSlopeX(Q, Qx, theta_); - __syncthreads(); - computeFluxF(Q, Qx, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - - - - - - - - - - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu b/GPUSimulators/cuda/SWE2D_LxF.cu deleted file mode 100644 index 1f197fd..0000000 --- a/GPUSimulators/cuda/SWE2D_LxF.cu +++ /dev/null @@ -1,168 +0,0 @@ -/* -This OpenCL kernel implements the classical Lax-Friedrichs scheme -for the shallow water equations, with edge fluxes. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" - - -/** - * Computes the flux along the x axis for all faces - */ -template -__device__ -void computeFluxF(float Q[3][block_height+2][block_width+2], - float F[3][block_height][block_width+1], - const float g_, const float dx_, const float dt_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - { - const int j=ty; - const int l = j + 1; //Skip ghost cells - for (int i=tx; i -__device__ -void computeFluxG(float Q[3][block_height+2][block_width+2], - float G[3][block_height+1][block_width], - const float g_, const float dy_, const float dt_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Compute fluxes along the x and y axis - computeFluxF(Q, F, g_, dx_, dt_); - computeFluxG(Q, G, g_, dy_, dt_); - __syncthreads(); - - //Evolve for all cells - const int tx = threadIdx.x; - const int ty = threadIdx.y; - const int i = tx + 1; //Skip local ghost cells, i.e., +1 - const int j = ty + 1; - - Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ - + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; - Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ - + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; - Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ - + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; - __syncthreads(); - - //Write to main memory - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" - diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu.hip b/GPUSimulators/cuda/SWE2D_LxF.cu.hip deleted file mode 100644 index 588d691..0000000 --- a/GPUSimulators/cuda/SWE2D_LxF.cu.hip +++ /dev/null @@ -1,169 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the classical Lax-Friedrichs scheme -for the shallow water equations, with edge fluxes. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - -#include "common.h" -#include "SWECommon.h" - - -/** - * Computes the flux along the x axis for all faces - */ -template -__device__ -void computeFluxF(float Q[3][block_height+2][block_width+2], - float F[3][block_height][block_width+1], - const float g_, const float dx_, const float dt_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - { - const int j=ty; - const int l = j + 1; //Skip ghost cells - for (int i=tx; i -__device__ -void computeFluxG(float Q[3][block_height+2][block_width+2], - float G[3][block_height+1][block_width], - const float g_, const float dy_, const float dt_) { - //Index of thread within block - const int tx = threadIdx.x; - const int ty = threadIdx.y; - - for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - - //Compute fluxes along the x and y axis - computeFluxF(Q, F, g_, dx_, dt_); - computeFluxG(Q, G, g_, dy_, dt_); - __syncthreads(); - - //Evolve for all cells - const int tx = threadIdx.x; - const int ty = threadIdx.y; - const int i = tx + 1; //Skip local ghost cells, i.e., +1 - const int j = ty + 1; - - Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ - + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; - Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ - + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; - Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ - + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; - __syncthreads(); - - //Write to main memory - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); - - //Compute the CFL for this block - if (cfl_ != NULL) { - writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); - } -} - -} // extern "C" - diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu b/GPUSimulators/cuda/SWE2D_WAF.cu deleted file mode 100644 index 2c38cdf..0000000 --- a/GPUSimulators/cuda/SWE2D_WAF.cu +++ /dev/null @@ -1,178 +0,0 @@ -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - __syncthreads(); - - - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu.hip b/GPUSimulators/cuda/SWE2D_WAF.cu.hip deleted file mode 100644 index ddfad9d..0000000 --- a/GPUSimulators/cuda/SWE2D_WAF.cu.hip +++ /dev/null @@ -1,179 +0,0 @@ -#include "hip/hip_runtime.h" -/* -This OpenCL kernel implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. - -Copyright (C) 2016 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - - - -#include "common.h" -#include "SWECommon.h" - - - -/** - * Computes the flux along the x axis for all faces - */ -__device__ -void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], - const float g_, const float dx_, const float dt_) { - for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); - readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); - readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); - __syncthreads(); - - - - //Step 0 => evolve x first, then y - if (step_ == 0) { - //Compute fluxes along the x axis and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - - //Compute fluxes along the y axis and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - } - //Step 1 => evolve y first, then x - else { - //Compute fluxes along the y axis and evolve - computeFluxG(Q, F, g_, dy_, dt_); - __syncthreads(); - evolveG(Q, F, dy_, dt_); - __syncthreads(); - - //Compute fluxes along the x axis and evolve - computeFluxF(Q, F, g_, dx_, dt_); - __syncthreads(); - evolveF(Q, F, dx_, dt_); - __syncthreads(); - } - - - - // Write to main memory for all internal cells - writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); - writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); - writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); -} - -} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWECommon.h b/GPUSimulators/cuda/SWECommon.h deleted file mode 100644 index 52f8b31..0000000 --- a/GPUSimulators/cuda/SWECommon.h +++ /dev/null @@ -1,533 +0,0 @@ -/* -These CUDA functions implement different types of numerical flux -functions for the shallow water equations - -Copyright (C) 2016, 2017, 2018 SINTEF Digital - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -#pragma once -#include "limiters.h" - - - - - - -__device__ float3 F_func(const float3 Q, const float g) { - float3 F; - - F.x = Q.y; //hu - F.y = Q.y*Q.y / Q.x + 0.5f*g*Q.x*Q.x; //hu*hu/h + 0.5f*g*h*h; - F.z = Q.y*Q.z / Q.x; //hu*hv/h; - - return F; -} - - - - - - - - - - - - - -/** - * Superbee flux limiter for WAF. - * Related to superbee limiter so that WAF_superbee(r, c) = 1 - (1-|c|)*superbee(r) - * @param r_ the ratio of upwind change (see Toro 2001, p. 203/204) - * @param c_ the courant number for wave k, dt*S_k/dx - */ -__device__ float WAF_superbee(float r_, float c_) { - // r <= 0.0 - if (r_ <= 0.0f) { - return 1.0f; - } - // 0.0 <= r <= 1/2 - else if (r_ <= 0.5f) { - return 1.0f - 2.0f*(1.0f - fabsf(c_))*r_; - } - // 1/2 <= r <= 1 - else if (r_ <= 1.0f) { - return fabs(c_); - } - // 1 <= r <= 2 - else if (r_ <= 2.0f) { - return 1.0f - (1.0f - fabsf(c_))*r_; - } - // r >= 2 - else { - return 2.0f*fabsf(c_) - 1.0f; - } -} - - - - -__device__ float WAF_albada(float r_, float c_) { - if (r_ <= 0.0f) { - return 1.0f; - } - else { - return 1.0f - (1.0f - fabsf(c_)) * r_ * (1.0f + r_) / (1.0f + r_*r_); - } -} - -__device__ float WAF_minbee(float r_, float c_) { - r_ = fmaxf(-1.0f, fminf(2.0f, r_)); - if (r_ <= 0.0f) { - return 1.0f; - } - if (r_ >= 0.0f && r_ <= 1.0f) { - return 1.0f - (1.0f - fabsf(c_)) * r_; - } - else { - return fabsf(c_); - } -} - -__device__ float WAF_minmod(float r_, float c_) { - return 1.0f - (1.0f - fabsf(c_)) * fmaxf(0.0f, fminf(1.0f, r_)); -} - -__device__ float limiterToWAFLimiter(float r_, float c_) { - return 1.0f - (1.0f - fabsf(c_))*r_; -} - -// Compute h in the "star region", h^dagger -__device__ __inline__ float computeHStar(float h_l, float h_r, float u_l, float u_r, float c_l, float c_r, float g_) { - - //This estimate for the h* gives rise to spurious oscillations. - //return 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); - - const float h_tmp = 0.5f * (c_l + c_r) + 0.25f * (u_l - u_r); - return h_tmp*h_tmp / g_; -} - -/** - * Weighted average flux (Toro 2001, p 200) for interface {i+1/2} - * @param r_ The flux limiter parameter (see Toro 2001, p. 203) - * @param Q_l2 Q_{i-1} - * @param Q_l1 Q_{i} - * @param Q_r1 Q_{i+1} - * @param Q_r2 Q_{i+2} - */ -__device__ float3 WAF_1D_flux(const float3 Q_l2, const float3 Q_l1, const float3 Q_r1, const float3 Q_r2, const float g_, const float dx_, const float dt_) { - const float h_l = Q_l1.x; - const float h_r = Q_r1.x; - - const float h_l2 = Q_l2.x; - const float h_r2 = Q_r2.x; - - // Calculate velocities - const float u_l = Q_l1.y / h_l; - const float u_r = Q_r1.y / h_r; - - const float u_l2 = Q_l2.y / h_l2; - const float u_r2 = Q_r2.y / h_r2; - - const float v_l = Q_l1.z / h_l; - const float v_r = Q_r1.z / h_r; - - const float v_l2 = Q_l2.z / h_l2; - const float v_r2 = Q_r2.z / h_r2; - - // Estimate the potential wave speeds - const float c_l = sqrt(g_*h_l); - const float c_r = sqrt(g_*h_r); - - const float c_l2 = sqrt(g_*h_l2); - const float c_r2 = sqrt(g_*h_r2); - - // Compute h in the "star region", h^dagger - const float h_dag_l = computeHStar(h_l2, h_l, u_l2, u_l, c_l2, c_l, g_); - const float h_dag = computeHStar( h_l, h_r, u_l, u_r, c_l, c_r, g_); - const float h_dag_r = computeHStar( h_r, h_r2, u_r, u_r2, c_r, c_r2, g_); - - const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag ) ) / h_l; - const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag ) ) / h_r; - - const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; - const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; - - // Compute wave speed estimates - const float S_l = u_l - c_l*q_l; - const float S_r = u_r + c_r*q_r; - const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); - - const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1.0, S_star, v_l); - const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1.0, S_star, v_r); - - // Estimate the fluxes in the four regions - const float3 F_1 = F_func(Q_l1, g_); - const float3 F_4 = F_func(Q_r1, g_); - - const float3 F_2 = F_1 + S_l*(Q_star_l - Q_l1); - const float3 F_3 = F_4 + S_r*(Q_star_r - Q_r1); - //const float3 F_2 = F_func(Q_star_l, g_); - //const float3 F_3 = F_func(Q_star_r, g_); - - // Compute the courant numbers for the waves - const float c_1 = S_l * dt_ / dx_; - const float c_2 = S_star * dt_ / dx_; - const float c_3 = S_r * dt_ / dx_; - - // Compute the "upwind change" vectors for the i-3/2 and i+3/2 interfaces - const float eps = 1.0e-6f; - const float r_1 = desingularize( (c_1 > 0.0f) ? (h_dag_l - h_l2) : (h_dag_r - h_r), eps) / desingularize((h_dag - h_l), eps); - const float r_2 = desingularize( (c_2 > 0.0f) ? (v_l - v_l2) : (v_r2 - v_r), eps ) / desingularize((v_r - v_l), eps); - const float r_3 = desingularize( (c_3 > 0.0f) ? (h_l - h_dag_l) : (h_r2 - h_dag_r), eps ) / desingularize((h_r - h_dag), eps); - - // Compute the limiter - // We use h for the nonlinear waves, and v for the middle shear wave - const float A_1 = copysign(1.0f, c_1) * limiterToWAFLimiter(generalized_minmod(r_1, 1.9f), c_1); - const float A_2 = copysign(1.0f, c_2) * limiterToWAFLimiter(generalized_minmod(r_2, 1.9f), c_2); - const float A_3 = copysign(1.0f, c_3) * limiterToWAFLimiter(generalized_minmod(r_3, 1.9f), c_3); - - //Average the fluxes - const float3 flux = 0.5f*( F_1 + F_4 ) - - 0.5f*( A_1 * (F_2 - F_1) - + A_2 * (F_3 - F_2) - + A_3 * (F_4 - F_3) ); - - return flux; -} - - - - - - - - - - - - - - -/** - * Central upwind flux function - */ -__device__ float3 CentralUpwindFlux(const float3 Qm, float3 Qp, const float g) { - const float3 Fp = F_func(Qp, g); - const float up = Qp.y / Qp.x; // hu / h - const float cp = sqrt(g*Qp.x); // sqrt(g*h) - - const float3 Fm = F_func(Qm, g); - const float um = Qm.y / Qm.x; // hu / h - const float cm = sqrt(g*Qm.x); // sqrt(g*h) - - const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed - const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed - - return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); -} - - - - - - - - - -/** - * Godunovs centered scheme (Toro 2001, p 165) - */ -__device__ float3 GodC_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - - const float3 Q_godc = 0.5f*(Q_l + Q_r) + (dt_/dx_)*(F_l - F_r); - - return F_func(Q_godc, g_); -} - - - - - - - - - -/** - * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 180) - */ -__device__ float3 HLL_flux(const float3 Q_l, const float3 Q_r, const float g_) { - const float h_l = Q_l.x; - const float h_r = Q_r.x; - - // Calculate velocities - const float u_l = Q_l.y / h_l; - const float u_r = Q_r.y / h_r; - - // Estimate the potential wave speeds - const float c_l = sqrt(g_*h_l); - const float c_r = sqrt(g_*h_r); - - // Compute h in the "star region", h^dagger - const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); - - const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); - const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); - - const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; - const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; - - // Compute wave speed estimates - const float S_l = u_l - c_l*q_l; - const float S_r = u_r + c_r*q_r; - - //Upwind selection - if (S_l >= 0.0f) { - return F_func(Q_l, g_); - } - else if (S_r <= 0.0f) { - return F_func(Q_r, g_); - } - //Or estimate flux in the star region - else { - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - const float3 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); - return flux; - } -} - - - - - - - - - - - - - - - - - -/** - * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 181) - */ -__device__ float3 HLLC_flux(const float3 Q_l, const float3 Q_r, const float g_) { - const float h_l = Q_l.x; - const float h_r = Q_r.x; - - // Calculate velocities - const float u_l = Q_l.y / h_l; - const float u_r = Q_r.y / h_r; - - // Estimate the potential wave speeds - const float c_l = sqrt(g_*h_l); - const float c_r = sqrt(g_*h_r); - - // Compute h in the "star region", h^dagger - const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); - - const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); - const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); - - const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; - const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; - - // Compute wave speed estimates - const float S_l = u_l - c_l*q_l; - const float S_r = u_r + c_r*q_r; - const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); - - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - - //Upwind selection - if (S_l >= 0.0f) { - return F_l; - } - else if (S_r <= 0.0f) { - return F_r; - } - //Or estimate flux in the "left star" region - else if (S_l <= 0.0f && 0.0f <=S_star) { - const float v_l = Q_l.z / h_l; - const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1, S_star, v_l); - const float3 flux = F_l + S_l*(Q_star_l - Q_l); - return flux; - } - //Or estimate flux in the "righ star" region - else if (S_star <= 0.0f && 0.0f <=S_r) { - const float v_r = Q_r.z / h_r; - const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1, S_star, v_r); - const float3 flux = F_r + S_r*(Q_star_r - Q_r); - return flux; - } - else { - return make_float3(-99999.9f, -99999.9f, -99999.9f); //Something wrong here - } -} - - - - - - - - - - - - - -/** - * Lax-Friedrichs flux (Toro 2001, p 163) - */ -__device__ float3 LxF_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - - return 0.5f*(F_l + F_r) + (dx_/(2.0f*dt_))*(Q_l - Q_r); -} - - - - - - - - -/** - * Lax-Friedrichs extended to 2D - */ -__device__ float3 LxF_2D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - - //Note numerical diffusion for 2D here (0.25) - return 0.5f*(F_l + F_r) + (dx_/(4.0f*dt_))*(Q_l - Q_r); -} - - - - - - - - - - - - -/** - * Richtmeyer / Two-step Lax-Wendroff flux (Toro 2001, p 164) - */ -__device__ float3 LxW2_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { - const float3 F_l = F_func(Q_l, g_); - const float3 F_r = F_func(Q_r, g_); - - const float3 Q_lw2 = 0.5f*(Q_l + Q_r) + (dt_/(2.0f*dx_))*(F_l - F_r); - - return F_func(Q_lw2, g_); -} - - - - - - - - - - - - -/** - * First Ordered Centered (Toro 2001, p.163) - */ -__device__ float3 FORCE_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { - const float3 F_lf = LxF_1D_flux(Q_l, Q_r, g_, dx_, dt_); - const float3 F_lw2 = LxW2_1D_flux(Q_l, Q_r, g_, dx_, dt_); - return 0.5f*(F_lf + F_lw2); -} - - - - - - - - - - -template -__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], - float shmem[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_, - const float dx_, const float dy_, const float g_, - float* output_) { - //Index of thread within block - const int tx = threadIdx.x + gc_x; - const int ty = threadIdx.y + gc_y; - - //Index of cell within domain - const int ti = blockDim.x*blockIdx.x + tx; - const int tj = blockDim.y*blockIdx.y + ty; - - //Only internal cells - if (ti < nx_+gc_x && tj < ny_+gc_y) { - const float h = Q[0][ty][tx]; - const float u = Q[1][ty][tx] / h; - const float v = Q[2][ty][tx] / h; - - const float max_u = dx_ / (fabsf(u) + sqrtf(g_*h)); - const float max_v = dy_ / (fabsf(v) + sqrtf(g_*h)); - - shmem[ty][tx] = fminf(max_u, max_v); - } - __syncthreads(); - - //One row of threads loop over all rows - if (ti < nx_+gc_x && tj < ny_+gc_y) { - if (ty == gc_y) { - float min_val = shmem[ty][tx]; - const int max_y = min(h, ny_+gc_y - tj); - for (int j=gc_y; j. -*/ - -#pragma once - - -/** - * Float3 operators - */ -inline __device__ float3 operator*(const float a, const float3 b) { - return make_float3(a*b.x, a*b.y, a*b.z); -} - -inline __device__ float3 operator/(const float3 a, const float b) { - return make_float3(a.x/b, a.y/b, a.z/b); -} - -inline __device__ float3 operator-(const float3 a, const float3 b) { - return make_float3(a.x-b.x, a.y-b.y, a.z-b.z); -} - -inline __device__ float3 operator+(const float3 a, const float3 b) { - return make_float3(a.x+b.x, a.y+b.y, a.z+b.z); -} - -/** - * Float4 operators - */ -inline __device__ float4 operator*(const float a, const float4 b) { - return make_float4(a*b.x, a*b.y, a*b.z, a*b.w); -} - -inline __device__ float4 operator/(const float4 a, const float b) { - return make_float4(a.x/b, a.y/b, a.z/b, a.w/b); -} - -inline __device__ float4 operator-(const float4 a, const float4 b) { - return make_float4(a.x-b.x, a.y-b.y, a.z-b.z, a.w-b.w); -} - -inline __device__ float4 operator+(const float4 a, const float4 b) { - return make_float4(a.x+b.x, a.y+b.y, a.z+b.z, a.w+b.w); -} - - - - -inline __device__ __host__ float clamp(const float f, const float a, const float b) { - return fmaxf(a, fminf(f, b)); -} - -inline __device__ __host__ int clamp(const int f, const int a, const int b) { - return (f < b) ? ( (f > a) ? f : a) : b; -} - - - - - -__device__ float desingularize(float x_, float eps_) { - return copysign(1.0f, x_)*fmaxf(fabsf(x_), fminf(x_*x_/(2.0f*eps_)+0.5f*eps_, eps_)); -} - - - - - - - - -/** - * Returns the step stored in the leftmost 16 bits - * of the 32 bit step-order integer - */ -inline __device__ int getStep(int step_order_) { - return step_order_ >> 16; -} - -/** - * Returns the order stored in the rightmost 16 bits - * of the 32 bit step-order integer - */ -inline __device__ int getOrder(int step_order_) { - return step_order_ & 0x0000FFFF; -} - - -enum BoundaryCondition { - Dirichlet = 0, - Neumann = 1, - Periodic = 2, - Reflective = 3 -}; - -inline __device__ BoundaryCondition getBCNorth(int bc_) { - return static_cast((bc_ >> 24) & 0x0000000F); -} - -inline __device__ BoundaryCondition getBCSouth(int bc_) { - return static_cast((bc_ >> 16) & 0x0000000F); -} - -inline __device__ BoundaryCondition getBCEast(int bc_) { - return static_cast((bc_ >> 8) & 0x0000000F); -} - -inline __device__ BoundaryCondition getBCWest(int bc_) { - return static_cast((bc_ >> 0) & 0x0000000F); -} - - -// West boundary -template -__device__ void bcWestReflective(float Q[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_) { - for (int j=threadIdx.y; j= 1 && ti == gc_x) { - Q[j][i-1] = sign*Q[j][i]; - } - if (gc_x >= 2 && ti == gc_x + 1) { - Q[j][i-3] = sign*Q[j][i]; - } - if (gc_x >= 3 && ti == gc_x + 2) { - Q[j][i-5] = sign*Q[j][i]; - } - if (gc_x >= 4 && ti == gc_x + 3) { - Q[j][i-7] = sign*Q[j][i]; - } - if (gc_x >= 5 && ti == gc_x + 4) { - Q[j][i-9] = sign*Q[j][i]; - } - } -} - - -// East boundary -template -__device__ void bcEastReflective(float Q[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_) { - for (int j=threadIdx.y; j= 1 && ti == nx_ + gc_x - 1) { - Q[j][i+1] = sign*Q[j][i]; - } - if (gc_x >= 2 && ti == nx_ + gc_x - 2) { - Q[j][i+3] = sign*Q[j][i]; - } - if (gc_x >= 3 && ti == nx_ + gc_x - 3) { - Q[j][i+5] = sign*Q[j][i]; - } - if (gc_x >= 4 && ti == nx_ + gc_x - 4) { - Q[j][i+7] = sign*Q[j][i]; - } - if (gc_x >= 5 && ti == nx_ + gc_x - 5) { - Q[j][i+9] = sign*Q[j][i]; - } - } -} - - -// South boundary -template -__device__ void bcSouthReflective(float Q[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_) { - for (int i=threadIdx.x; i= 1 && tj == gc_y) { - Q[j-1][i] = sign*Q[j][i]; - } - if (gc_y >= 2 && tj == gc_y + 1) { - Q[j-3][i] = sign*Q[j][i]; - } - if (gc_y >= 3 && tj == gc_y + 2) { - Q[j-5][i] = sign*Q[j][i]; - } - if (gc_y >= 4 && tj == gc_y + 3) { - Q[j-7][i] = sign*Q[j][i]; - } - if (gc_y >= 5 && tj == gc_y + 4) { - Q[j-9][i] = sign*Q[j][i]; - } - } -} - - - - -// North boundary -template -__device__ void bcNorthReflective(float Q[h+2*gc_y][w+2*gc_x], const int nx_, const int ny_) { - for (int i=threadIdx.x; i= 1 && tj == ny_ + gc_y - 1) { - Q[j+1][i] = sign*Q[j][i]; - } - if (gc_y >= 2 && tj == ny_ + gc_y - 2) { - Q[j+3][i] = sign*Q[j][i]; - } - if (gc_y >= 3 && tj == ny_ + gc_y - 3) { - Q[j+5][i] = sign*Q[j][i]; - } - if (gc_y >= 4 && tj == ny_ + gc_y - 4) { - Q[j+7][i] = sign*Q[j][i]; - } - if (gc_y >= 5 && tj == ny_ + gc_y - 5) { - Q[j+9][i] = sign*Q[j][i]; - } - } -} - - - - -/** - * Alter the index l so that it gives periodic boundary conditions when reading - */ -template -inline __device__ int handlePeriodicBoundaryX(int k, int nx_, int boundary_conditions_) { - const int gc_pad = gc_x; - - //West boundary: add an offset to read from east of domain - if (gc_x > 0) { - if ((k < gc_pad) - && getBCWest(boundary_conditions_) == Periodic) { - k += (nx_+2*gc_x - 2*gc_pad); - } - //East boundary: subtract an offset to read from west of domain - else if ((k >= nx_+2*gc_x-gc_pad) - && getBCEast(boundary_conditions_) == Periodic) { - k -= (nx_+2*gc_x - 2*gc_pad); - } - } - - return k; -} - -/** - * Alter the index l so that it gives periodic boundary conditions when reading - */ -template -inline __device__ int handlePeriodicBoundaryY(int l, int ny_, int boundary_conditions_) { - const int gc_pad = gc_y; - - //South boundary: add an offset to read from north of domain - if (gc_y > 0) { - if ((l < gc_pad) - && getBCSouth(boundary_conditions_) == Periodic) { - l += (ny_+2*gc_y - 2*gc_pad); - } - //North boundary: subtract an offset to read from south of domain - else if ((l >= ny_+2*gc_y-gc_pad) - && getBCNorth(boundary_conditions_) == Periodic) { - l -= (ny_+2*gc_y - 2*gc_pad); - } - } - - return l; -} - - -template -inline __device__ -void handleReflectiveBoundary( - float Q[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_, - const int boundary_conditions_) { - - //Handle reflective boundary conditions - if (getBCNorth(boundary_conditions_) == Reflective) { - bcNorthReflective(Q, nx_, ny_); - __syncthreads(); - } - if (getBCSouth(boundary_conditions_) == Reflective) { - bcSouthReflective(Q, nx_, ny_); - __syncthreads(); - } - if (getBCEast(boundary_conditions_) == Reflective) { - bcEastReflective(Q, nx_, ny_); - __syncthreads(); - } - if (getBCWest(boundary_conditions_) == Reflective) { - bcWestReflective(Q, nx_, ny_); - __syncthreads(); - } -} - -/** - * Reads a block of data with ghost cells - */ -template -inline __device__ void readBlock(float* ptr_, int pitch_, - float Q[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_, - const int boundary_conditions_, - int x0, int y0, - int x1, int y1) { - //Index of block within domain - const int bx = blockDim.x * blockIdx.x; - const int by = blockDim.y * blockIdx.y; - - //Read into shared memory - //Loop over all variables - for (int j=threadIdx.y; j(by + j + y0, ny_, boundary_conditions_); - l = min(l, min(ny_+2*gc_y-1, y1+2*gc_y-1)); - float* row = (float*) ((char*) ptr_ + pitch_*l); - - for (int i=threadIdx.x; i(bx + i + x0, nx_, boundary_conditions_); - k = min(k, min(nx_+2*gc_x-1, x1+2*gc_x-1)); - - //Read from global memory - Q[j][i] = row[k]; - } - } - __syncthreads(); - - handleReflectiveBoundary(Q, nx_, ny_, boundary_conditions_); -} - - - - -/** - * Writes a block of data to global memory for the shallow water equations. - */ -template -inline __device__ void writeBlock(float* ptr_, int pitch_, - float shmem[h+2*gc_y][w+2*gc_x], - const int nx_, const int ny_, - int rk_step_, int rk_order_, - int x0, int y0, - int x1, int y1) { - - //Index of cell within domain - const int ti = blockDim.x*blockIdx.x + threadIdx.x + gc_x + x0; - const int tj = blockDim.y*blockIdx.y + threadIdx.y + gc_y + y0; - - //In case we are writing only to a subarea given by (x0, y0) x (x1, y1) - const int max_ti = min(nx_+gc_x, x1+gc_x); - const int max_tj = min(ny_+gc_y, y1+gc_y); - - //Only write internal cells - if ((x0+gc_x <= ti) && (ti < max_ti) && (y0+gc_y <= tj) && (tj < max_tj)) { - //Index of thread within block - const int tx = threadIdx.x + gc_x; - const int ty = threadIdx.y + gc_y; - - float* const row = (float*) ((char*) ptr_ + pitch_*tj); - - //Handle runge-kutta timestepping here - row[ti] = shmem[ty][tx]; - - - - /** - * SSPRK1 (forward Euler) - * u^1 = u^n + dt*f(u^n) - */ - if (rk_order_ == 1) { - row[ti] = shmem[ty][tx]; - } - /** - * SSPRK2 - * u^1 = u^n + dt*f(u^n) - * u^n+1 = 1/2*u^n + 1/2*(u^1 + dt*f(u^1)) - */ - else if (rk_order_ == 2) { - if (rk_step_ == 0) { - row[ti] = shmem[ty][tx]; - } - else if (rk_step_ == 1) { - row[ti] = 0.5f*row[ti] + 0.5f*shmem[ty][tx]; - } - } - /** - * SSPRK3 - * u^1 = u^n + dt*f(u^n) - * u^2 = 3/4 * u^n + 1/4 * (u^1 + dt*f(u^1)) - * u^n+1 = 1/3 * u^n + 2/3 * (u^2 + dt*f(u^2)) - * FIXME: This is not correct now, need a temporary to hold intermediate step u^2 - */ - else if (rk_order_ == 3) { - if (rk_step_ == 0) { - row[ti] = shmem[ty][tx]; - } - else if (rk_step_ == 1) { - row[ti] = 0.75f*row[ti] + 0.25f*shmem[ty][tx]; - } - else if (rk_step_ == 2) { - const float t = 1.0f / 3.0f; //Not representable in base 2 - row[ti] = t*row[ti] + (1.0f-t)*shmem[ty][tx]; - } - } - - // DEBUG - //row[ti] = 99.0; - } -} - - - - - - - - - - - -template -__device__ void evolveF(float Q[vars][h+2*gc_y][w+2*gc_x], - float F[vars][h+2*gc_y][w+2*gc_x], - const float dx_, const float dt_) { - for (int var=0; var < vars; ++var) { - for (int j=threadIdx.y; j -__device__ void evolveG(float Q[vars][h+2*gc_y][w+2*gc_x], - float G[vars][h+2*gc_y][w+2*gc_x], - const float dy_, const float dt_) { - for (int var=0; var < vars; ++var) { - for (int j=threadIdx.y+gc_y; j -__device__ void memset(float Q[vars][shmem_height][shmem_width], float value) { - for (int k=0; k -__device__ void reduce_max(float* data, unsigned int n) { - __shared__ float sdata[threads]; - unsigned int tid = threadIdx.x; - - //Reduce to "threads" elements - sdata[tid] = FLT_MIN; - for (unsigned int i=tid; i= 512) { - if (tid < 256) { - sdata[tid] = max(sdata[tid], sdata[tid + 256]); - } - __syncthreads(); - } - if (threads >= 256) { - if (tid < 128) { - sdata[tid] = max(sdata[tid], sdata[tid + 128]); - } - __syncthreads(); - } - if (threads >= 128) { - if (tid < 64) { - sdata[tid] = max(sdata[tid], sdata[tid + 64]); - } - __syncthreads(); - } - if (tid < 32) { - volatile float* sdata_volatile = sdata; - if (threads >= 64) { - sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 32]); - } - if (tid < 16) { - if (threads >= 32) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 16]); - if (threads >= 16) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 8]); - if (threads >= 8) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 4]); - if (threads >= 4) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 2]); - if (threads >= 2) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 1]); - } - - if (tid == 0) { - return sdata_volatile[0]; - } - } -} - - - - - - - - - - diff --git a/GPUSimulators/cuda/limiters.h b/GPUSimulators/cuda/limiters.h deleted file mode 100644 index c2effa7..0000000 --- a/GPUSimulators/cuda/limiters.h +++ /dev/null @@ -1,118 +0,0 @@ -/* -This file implements different flux and slope limiters - -Copyright (C) 2016, 2017, 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -#pragma once - - - - - - -/** - * Reconstructs a slope using the generalized minmod limiter based on three - * consecutive values - */ -__device__ __inline__ float minmodSlope(float left, float center, float right, float theta) { - const float backward = (center - left) * theta; - const float central = (right - left) * 0.5f; - const float forward = (right - center) * theta; - - return 0.25f - *copysign(1.0f, backward) - *(copysign(1.0f, backward) + copysign(1.0f, central)) - *(copysign(1.0f, central) + copysign(1.0f, forward)) - *min( min(fabs(backward), fabs(central)), fabs(forward) ); -} - - - - -/** - * Reconstructs a minmod slope for a whole block along the abscissa - */ -template -__device__ void minmodSlopeX(float Q[vars][h+2*gc_y][w+2*gc_x], - float Qx[vars][h+2*gc_y][w+2*gc_x], - const float theta_) { - //Reconstruct slopes along x axis - for (int p=0; p -__device__ void minmodSlopeY(float Q[vars][h+2*gc_y][w+2*gc_x], - float Qy[vars][h+2*gc_y][w+2*gc_x], - const float theta_) { - //Reconstruct slopes along y axis - for (int p=0; p. -""" - - -from GPUSimulators.Simulator import BoundaryCondition -import numpy as np -import gc - - -def getExtent(width, height, nx, ny, grid, index=None): - if grid is not None: - gx = grid.grid[0] - gy = grid.grid[1] - if index is not None: - i, j = grid.getCoordinate(index) - else: - i, j = grid.getCoordinate() - - dx = (width / gx) / nx - dy = (height / gy) / ny - - x0 = width*i/gx + 0.5*dx - y0 = height*j/gy + 0.5*dy - x1 = width*(i+1)/gx - 0.5*dx - y1 = height*(j+1)/gy - 0.5*dx - - else: - dx = width / nx - dy = height / ny - - x0 = 0.5*dx - y0 = 0.5*dy - x1 = width-0.5*dx - y1 = height-0.5*dy - - return [x0, x1, y0, y1, dx, dy] - - -def downsample(highres_solution, x_factor, y_factor=None): - if (y_factor == None): - y_factor = x_factor - - assert(highres_solution.shape[1] % x_factor == 0) - assert(highres_solution.shape[0] % y_factor == 0) - - if (x_factor*y_factor == 1): - return highres_solution - - if (len(highres_solution.shape) == 1): - highres_solution = highres_solution.reshape((1, highres_solution.size)) - - nx = highres_solution.shape[1] / x_factor - ny = highres_solution.shape[0] / y_factor - - return highres_solution.reshape([int(ny), int(y_factor), int(nx), int(x_factor)]).mean(3).mean(1) - - - - - -def bump(nx, ny, width, height, - bump_size=None, - ref_nx=None, ref_ny=None, - x_center=0.5, y_center=0.5, - h_ref=0.5, h_amp=0.1, u_ref=0.0, u_amp=0.1, v_ref=0.0, v_amp=0.1): - - if (ref_nx == None): - ref_nx = nx - assert(ref_nx >= nx) - - if (ref_ny == None): - ref_ny = ny - assert(ref_ny >= ny) - - if (bump_size == None): - bump_size = width/5.0 - - ref_dx = width / float(ref_nx) - ref_dy = height / float(ref_ny) - - x_center = ref_dx*ref_nx*x_center - y_center = ref_dy*ref_ny*y_center - - x = ref_dx*(np.arange(0, ref_nx, dtype=np.float32)+0.5) - x_center - y = ref_dy*(np.arange(0, ref_ny, dtype=np.float32)+0.5) - y_center - xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') - r = np.sqrt(xv**2 + yv**2) - xv = None - yv = None - gc.collect() - - #Generate highres then downsample - #h_highres = 0.5 + 0.1*np.exp(-(xv**2/size + yv**2/size)) - h_highres = h_ref + h_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) - h = downsample(h_highres, ref_nx/nx, ref_ny/ny) - h_highres = None - gc.collect() - - #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) - u_highres = u_ref + u_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) - hu = downsample(u_highres, ref_nx/nx, ref_ny/ny)*h - u_highres = None - gc.collect() - - #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) - v_highres = v_ref + v_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) - hv = downsample(v_highres, ref_nx/nx, ref_ny/ny)*h - v_highres = None - gc.collect() - - dx = width/nx - dy = height/ny - - return h, hu, hv, dx, dy - - -def genShockBubble(nx, ny, gamma, grid=None): - """ - Generate Shock-bubble interaction case for the Euler equations - """ - - width = 4.0 - height = 1.0 - g = 0.0 - - - rho = np.ones((ny, nx), dtype=np.float32) - u = np.zeros((ny, nx), dtype=np.float32) - v = np.zeros((ny, nx), dtype=np.float32) - E = np.zeros((ny, nx), dtype=np.float32) - p = np.ones((ny, nx), dtype=np.float32) - - - x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) - x = np.linspace(x0, x1, nx, dtype=np.float32) - y = np.linspace(y0, y1, ny, dtype=np.float32) - xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') - - #Bubble - radius = 0.25 - x_center = 0.5 - y_center = 0.5 - bubble = np.sqrt((xv-x_center)**2+(yv-y_center)**2) <= radius - rho = np.where(bubble, 0.1, rho) - - #Left boundary - left = (xv < 0.1) - rho = np.where(left, 3.81250, rho) - u = np.where(left, 2.57669, u) - - #Energy - p = np.where(left, 10.0, p) - E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) - - - bc = BoundaryCondition({ - 'north': BoundaryCondition.Type.Reflective, - 'south': BoundaryCondition.Type.Reflective, - 'east': BoundaryCondition.Type.Periodic, - 'west': BoundaryCondition.Type.Periodic - }) - - #Construct simulator - arguments = { - 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, - 'nx': nx, 'ny': ny, - 'dx': dx, 'dy': dy, - 'g': g, - 'gamma': gamma, - 'boundary_conditions': bc - } - return arguments - - - - - - - -def genKelvinHelmholtz(nx, ny, gamma, roughness=0.125, grid=None, index=None): - """ - Roughness parameter in (0, 1.0] determines how "squiggly" - the interface betweeen the zones is - """ - - def genZones(nx, ny, n): - """ - Generates the zones of the two fluids of K-H - """ - zone = np.zeros((ny, nx), dtype=np.int32) - - - def genSmoothRandom(nx, n): - n = max(1, min(n, nx)) - - if n == nx: - return np.random.random(nx)-0.5 - else: - from scipy.interpolate import interp1d - - #Control points and interpolator - xp = np.linspace(0.0, 1.0, n) - yp = np.random.random(n) - 0.5 - - if (n == 1): - kind = 'nearest' - elif (n == 2): - kind = 'linear' - elif (n == 3): - kind = 'quadratic' - else: - kind = 'cubic' - - f = interp1d(xp, yp, kind=kind) - - #Interpolation points - x = np.linspace(0.0, 1.0, nx) - return f(x) - - - - x0, x1, y0, y1, _, dy = getExtent(1.0, 1.0, nx, ny, grid, index) - x = np.linspace(x0, x1, nx) - y = np.linspace(y0, y1, ny) - _, y = np.meshgrid(x, y) - - #print(y+a[0]) - - a = genSmoothRandom(nx, n)*dy - zone = np.where(y > 0.25+a, zone, 1) - - a = genSmoothRandom(nx, n)*dy - zone = np.where(y < 0.75+a, zone, 1) - - return zone - - width = 2.0 - height = 1.0 - g = 0.0 - gamma = 1.4 - - rho = np.empty((ny, nx), dtype=np.float32) - u = np.empty((ny, nx), dtype=np.float32) - v = np.zeros((ny, nx), dtype=np.float32) - p = 2.5*np.ones((ny, nx), dtype=np.float32) - - #Generate the different zones - zones = genZones(nx, ny, max(1, min(nx, int(nx*roughness)))) - - #Zone 0 - zone0 = zones == 0 - rho = np.where(zone0, 1.0, rho) - u = np.where(zone0, 0.5, u) - - #Zone 1 - zone1 = zones == 1 - rho = np.where(zone1, 2.0, rho) - u = np.where(zone1, -0.5, u) - - E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) - - _, _, _, _, dx, dy = getExtent(width, height, nx, ny, grid, index) - - - bc = BoundaryCondition({ - 'north': BoundaryCondition.Type.Periodic, - 'south': BoundaryCondition.Type.Periodic, - 'east': BoundaryCondition.Type.Periodic, - 'west': BoundaryCondition.Type.Periodic - }) - - #Construct simulator - arguments = { - 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, - 'nx': nx, 'ny': ny, - 'dx': dx, 'dy': dy, - 'g': g, - 'gamma': gamma, - 'boundary_conditions': bc - } - - return arguments - - - -def genRayleighTaylor(nx, ny, gamma, version=0, grid=None): - """ - Generates Rayleigh-Taylor instability case - """ - width = 0.5 - height = 1.5 - g = 0.1 - - rho = np.zeros((ny, nx), dtype=np.float32) - u = np.zeros((ny, nx), dtype=np.float32) - v = np.zeros((ny, nx), dtype=np.float32) - p = np.zeros((ny, nx), dtype=np.float32) - - - x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) - x = np.linspace(x0, x1, nx, dtype=np.float32)-width*0.5 - y = np.linspace(y0, y1, ny, dtype=np.float32)-height*0.5 - xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') - - #This gives a squigly interfact - if (version == 0): - y_threshold = 0.01*np.cos(2*np.pi*np.abs(x)/0.5) - rho = np.where(yv <= y_threshold, 1.0, rho) - rho = np.where(yv > y_threshold, 2.0, rho) - elif (version == 1): - rho = np.where(yv <= 0.0, 1.0, rho) - rho = np.where(yv > 0.0, 2.0, rho) - v = 0.01*(1.0 + np.cos(2*np.pi*xv/0.5))/4 - else: - assert False, "Invalid version" - - p = 2.5 - rho*g*yv - E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) - - bc = BoundaryCondition({ - 'north': BoundaryCondition.Type.Reflective, - 'south': BoundaryCondition.Type.Reflective, - 'east': BoundaryCondition.Type.Reflective, - 'west': BoundaryCondition.Type.Reflective - }) - - #Construct simulator - arguments = { - 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, - 'nx': nx, 'ny': ny, - 'dx': dx, 'dy': dy, - 'g': g, - 'gamma': gamma, - 'boundary_conditions': bc - } - - return arguments \ No newline at end of file diff --git a/GPUSimulators/helpers/Visualization.py b/GPUSimulators/helpers/Visualization.py deleted file mode 100644 index a2ff8f1..0000000 --- a/GPUSimulators/helpers/Visualization.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements visualization techniques/modes - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - - -import numpy as np - -from matplotlib.colors import Normalize - - - -def genSchlieren(rho): - #Compute length of z-component of normalized gradient vector - normal = np.gradient(rho) #[x, y, 1] - length = 1.0 / np.sqrt(normal[0]**2 + normal[1]**2 + 1.0) - schlieren = np.power(length, 128) - return schlieren - - -def genVorticity(rho, rho_u, rho_v): - u = rho_u / rho - v = rho_v / rho - u = np.sqrt(u**2 + v**2) - u_max = u.max() - - du_dy, _ = np.gradient(u) - _, dv_dx = np.gradient(v) - - #Length of curl - curl = dv_dx - du_dy - return curl - - -def genColors(rho, rho_u, rho_v, cmap, vmax, vmin): - schlieren = genSchlieren(rho) - curl = genVorticity(rho, rho_u, rho_v) - - colors = Normalize(vmin, vmax, clip=True)(curl) - colors = cmap(colors) - for k in range(3): - colors[:,:,k] = colors[:,:,k]*schlieren - - return colors \ No newline at end of file diff --git a/Jobs/job_lumi.slrum b/Jobs/job_lumi.slrum deleted file mode 100644 index b946bbf..0000000 --- a/Jobs/job_lumi.slrum +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -e -#SBATCH --job-name=lumi -#SBATCH --account=project_4650000xx -#SBATCH --time=00:10:00 -#SBATCH --partition=dev-g -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=2 -#SBATCH --gpus=2 -#SBATCH --gpus-per-node=2 -#SBATCH -o %x-%j.out -# - -N=$SLURM_JOB_NUM_NODES -echo "--nbr of nodes:", $N -echo "--total nbr of gpus:", $SLURM_NTASKS - -Mydir=/project/project_4650000xx -Myapplication=${Mydir}/FiniteVolumeGPU_hip/mpiTesting.py - -#modules -ml LUMI/23.03 partition/G -ml lumi-container-wrapper -ml cray-python/3.9.13.1 -ml rocm/5.2.3 - -ml craype-accel-amd-gfx90a -ml cray-mpich/8.1.27 - -#Enable GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - -export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH" - -srun python ${Myapplication} -nx 1024 -ny 1024 --profile diff --git a/LICENSE b/LICENSE deleted file mode 100644 index f288702..0000000 --- a/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/README.md b/README.md deleted file mode 100644 index 5f68f36..0000000 --- a/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# FiniteVolumeGPU - -This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. - -## Setup -A good place to start exploring this codebase is the notebooks. Complete the following steps to run the notebooks: - -1. Install conda (see e.g. Miniconda or Anaconda) -2. Change directory to the repository root and run the following commands -3. conda env create -f conda_environment.yml -4. conda activate ShallowWaterGPU -5. jupyter notebook - -Make sure you are running the correct kernel ("conda:ShallowWaterGPU"). If not, change kernel using the "Kernel"-menu in the notebook. - -If you do not need to run notebooks you may use the conda environment found in conda_environment_hpc.yml - -## Troubleshooting -Have a look at the conda documentation and https://towardsdatascience.com/how-to-set-up-anaconda-and-jupyter-notebook-the-right-way-de3b7623ea4a - -## Setup on LUMI-G -Here is a step-by-step guide on installing packages on LUMI-G - -### Step 0: load modules -``` -ml LUMI/23.03 -ml lumi-container-wrapper -ml cray-python/3.9.13.1 -``` - -### Step 1: run conda-container -Installation via conda can be done as: -``` -conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml -``` -where the file `conda_environment_lumi.yml` contains packages to be installed. - -### Step 2: Set the env. variable to search for binaries -``` -export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" -``` -### An alternative: Convert to a singularity container with cotainr -``` -cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml -``` diff --git a/SYSTEMS.md b/SYSTEMS.md deleted file mode 100644 index 5b0e770..0000000 --- a/SYSTEMS.md +++ /dev/null @@ -1,62 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -61316265663939333638336466323036663861343233316466646432313138653633623662353937 -3232313165656633346432376237383566363537366534310a303231343936663438653835373161 -35616161323432653062323164623861353065333761663136313137333732313230626665386336 -6166656538396463370a356166316363326133313864386536323236346634323537393639653038 -66336337336132613061353964613638326233356336323962366531333932366539366339623563 -36333365326463616634323939333062363263636663373635653064626138363464666233316561 -63393735393233616437386537393739393433663631313864646535636262616336333631396166 -38643636323530386565396338623366393232313838356536303537393338393634666632656234 -65353930303762333639376638336364303439306132626531326132376264623063376464636430 -32333536386134333136313139313861306364333037323363393463333664633764653937623866 -34313064346261313330373132353563343761323435393930303136353865303163373937623831 -64343038373162333039653161643233353764633337366434396638376530636261323362373434 -38393630613065356632663533333331633039663935663732353234643131306665343339373265 -64356563653838613337663132663234356462343333623139626662363764656239326637653832 -35396636643937336431623531306133643137623831333936313839333738333730373136666336 -35623965643664343164373630313362656663386638376237616134343631386366313336626138 -62376436383837376539663438346431633138383363633862356366376537393932626262383637 -31323365333139653736623233636233323162343039663035346135326638633430303134396337 -36353264313835346130643736663665386364343835643166383361316631373338663731373335 -30313530326662663937666330643565363565616566633333363535656539656531666266613638 -30306264613438363265646332386535383238373433396337633636616532626161343236336533 -36366362653137333739353737386563613136653164383437316237643533633133313735633363 -64326433356266363133343339626333633063326533383632353639613163663966376465396231 -36663034363534396430316463386564663465323036613636343136643262666566303533346439 -63396466656639623836613130363835346435633437666463363333356231343038356434343861 -66643636633739336666316566653136363862346336353862653130346335363334616430366435 -30376365383262326438306266366265363030353764633630333034663037643037343132303631 -39316364366234363339363130333765616432306331373566393530653963356539636437383062 -34633938643563656363633864656361643539663833356638356365373061663964363530393535 -37646533386235613763396638393539303062326239633238373763326561313634313265613135 -64646138313562313732393732303133343234323438616165326530333234626363393735636530 -62353735313231353662353533636134306530623339383730306332613636663366653566313935 -32343935353566656130393533323639353863666436333839386463396337336635356663373136 -61323734613239396236393266363631313465363630306565636663396235626132336339623938 -62383435643661623938393662363262376566613365613465323432343534356433323330666133 -30303963656635303734316539333038663962626331313366666337663165323230646564623935 -61316630353739386365323339626166323562616630383538393733353864396565353039656333 -30343038636231363531383061613836653038373937616163643963393231356235626531366239 -62343333326434636665363931376235313535343135626261336439636663323233383565633964 -65333830613131396630336337646230393038386536336365313738316335386261393838383961 -64656331363738616539346663613261386639353437316231636533353031336464383432623939 -65386164396231393735643563663337643563633233373338643630313739373861356166616463 -35306263333963663434376263396464323135346663376334356134393066653439376263376231 -30333730383163366636323533393334336331633234306536376634313735613263366537346536 -62366564383861656662353738366665396639313833323038356661306135393338333466333563 -32653861346166663163383036386432343833333137663462343030363762663139366534326466 -66313864623438336164333430613766373430656536323964633863333931643036656563353639 -30313835666366383035343031643265386263316165323537613636656533376239633964393866 -61646163343032313036303738643763383364663134356634373262633361383035306231636364 -39333232636538643033313438396332383962656131363365666566633239366532326336363133 -38393064643030333538333562643435663434343863383834663266373337336433313663646164 -36343334343965623830613736393231666361643239663062393239613233376335383362666161 -66383035653330373736613234303631386163656561383138613363613539396332376162316131 -61313532653531653836343731636535623066383231613635316432323331623761383833623333 -39343632623961613561373261653939636363366531303839336237383166363733303538363237 -36373362636263666334316163633766303334373033636539353464393536356466636664333665 -32643135626366666137626464393961366165383334343063356334373534633764326162363837 -38643662326266313464343464646166643235663663303761313639376537306337353863336264 -66376335333738366265343636376363366365306137336665623466626261653937656461303332 -32613561616662383032393562613831626666373134303032626134313262363830326530643632 -61366133663564313933366430396430353762386133396436633839303766653765 diff --git a/conda_environment_lumi.yml b/conda_environment_lumi.yml deleted file mode 100644 index 661ebc5..0000000 --- a/conda_environment_lumi.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Assumes that conda, pip, build-essentials and cuda are installed ---- -name: ShallowWaterGPU_HPC -channels: -- conda-forge - -dependencies: -- python=3.9 -- numpy -- mpi4py -- six -- pytools -- netcdf4 -- scipy -- pip: - - hip-python - - hip-python-as-cuda - - -i https://test.pypi.org/simple/ - - -#On LUMI-G -#module load LUMI/23.03 -#module load lumi-container-wrapper -#ml cray-python/3.9.13.1 -#conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml -# export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" -# -# -# -# Install conda environment (one-time operation): -# $ conda env create -f conda_environment_hpc.yml -# Activate environment and install the following packages using pip: -# $ conda activate ShallowWaterGPU_HPC -# - pycuda: $ pip3 install --no-deps -U pycuda -# on Windows: make sure your visual studio c++ compiler is available in PATH -# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\ diff --git a/mpiTesting.py b/mpiTesting.py deleted file mode 100644 index 5869fdd..0000000 --- a/mpiTesting.py +++ /dev/null @@ -1,230 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements MPI simulations for benchmarking - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - -import numpy as np -import gc -import time -import json -import logging -import os - -#GPU-aware MPI -from os import environ -if environ.get("MPICH_GPU_SUPPORT_ENABLED", False): - from ctypes import CDLL, RTLD_GLOBAL - CDLL(f"{environ.get('CRAY_MPICH_ROOTDIR')}/gtl/lib/libmpi_gtl_hsa.so", mode=RTLD_GLOBAL) - -# MPI -from mpi4py import MPI - -# CUDA -#import pycuda.driver as cuda -from hip import hip - -# Simulator engine etc -from GPUSimulators import MPISimulator, Common, CudaContext -from GPUSimulators import EE2D_KP07_dimsplit -from GPUSimulators.helpers import InitialConditions as IC -from GPUSimulators.Simulator import BoundaryCondition as BC - -import argparse -parser = argparse.ArgumentParser(description='Strong and weak scaling experiments.') -parser.add_argument('-nx', type=int, default=128) -parser.add_argument('-ny', type=int, default=128) -parser.add_argument('--profile', action='store_true') # default: False - -def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - -args = parser.parse_args() - -if(args.profile): - profiling_data = {} - # profiling: total run time - t_total_start = time.time() - t_init_start = time.time() - - -# Get MPI COMM to use -comm = MPI.COMM_WORLD - - -#### -# Initialize logging -#### -log_level_console = 20 -log_level_file = 10 -log_filename = 'mpi_' + str(comm.rank) + '.log' -logger = logging.getLogger('GPUSimulators') -logger.setLevel(min(log_level_console, log_level_file)) - -ch = logging.StreamHandler() -ch.setLevel(log_level_console) -logger.addHandler(ch) -logger.info("Console logger using level %s", - logging.getLevelName(log_level_console)) - -fh = logging.FileHandler(log_filename) -formatter = logging.Formatter( - '%(asctime)s:%(name)s:%(levelname)s: %(message)s') -fh.setFormatter(formatter) -fh.setLevel(log_level_file) -logger.addHandler(fh) -logger.info("File logger using level %s to %s", - logging.getLevelName(log_level_file), log_filename) - - -#### -# Initialize MPI grid etc -#### -logger.info("Creating MPI grid") -grid = MPISimulator.MPIGrid(MPI.COMM_WORLD) - - -#### -# Initialize CUDA -#### -#cuda.init(flags=0) -#logger.info("Initializing CUDA") -local_rank = grid.getLocalRank() -#num_cuda_devices = cuda.Device.count() -num_cuda_devices = hip_check(hip.hipGetDeviceCount()) -cuda_device = local_rank % num_cuda_devices -logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device)) -cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False) - - -#### -# Set initial conditions -#### - -# DEBUGGING - setting random seed -np.random.seed(42) - -logger.info("Generating initial conditions") -nx = args.nx -ny = args.ny - -dt = 0.000001 - -gamma = 1.4 -#save_times = np.linspace(0, 0.000009, 2) -#save_times = np.linspace(0, 0.000099, 11) -#save_times = np.linspace(0, 0.000099, 2) -save_times = np.linspace(0, 0.0000999, 2) -outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc" -save_var_names = ['rho', 'rho_u', 'rho_v', 'E'] - -arguments = IC.genKelvinHelmholtz(nx, ny, gamma, grid=grid) -arguments['context'] = cuda_context -arguments['theta'] = 1.2 -arguments['grid'] = grid - -if(args.profile): - t_init_end = time.time() - t_init = t_init_end - t_init_start - profiling_data["t_init"] = t_init - -#### -# Run simulation -#### -logger.info("Running simulation") -# Helper function to create MPI simulator - - -def genSim(grid, **kwargs): - local_sim = EE2D_KP07_dimsplit.EE2D_KP07_dimsplit(**kwargs) - sim = MPISimulator.MPISimulator(local_sim, grid) - return sim - - -outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation( - genSim, arguments, outfile, save_times, save_var_names, dt) - -if(args.profile): - t_total_end = time.time() - t_total = t_total_end - t_total_start - profiling_data["t_total"] = t_total - print("Total run time on rank " + str(MPI.COMM_WORLD.rank) + " is " + str(t_total) + " s") - -# write profiling to json file -if(args.profile and MPI.COMM_WORLD.rank == 0): - job_id = "" - if "SLURM_JOB_ID" in os.environ: - job_id = int(os.environ["SLURM_JOB_ID"]) - allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"]) - allocated_gpus = int(os.environ["HIP_VISIBLE_DEVICES"].count(",") + 1) - # allocated_gpus = int(os.environ["CUDA_VISIBLE_DEVICES"].count(",") + 1) - profiling_file = "MPI_jobid_" + \ - str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json" - profiling_data["outfile"] = outfile - else: - profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json" - - for stage in sim_runner_profiling_data["start"].keys(): - profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage] - - for stage in sim_profiling_data["start"].keys(): - profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage] - - profiling_data["nx"] = nx - profiling_data["ny"] = ny - profiling_data["dt"] = dt - profiling_data["n_time_steps"] = sim_profiling_data["n_time_steps"] - - profiling_data["slurm_job_id"] = job_id - profiling_data["n_cuda_devices"] = str(num_cuda_devices) - profiling_data["n_processes"] = str(MPI.COMM_WORLD.size) - profiling_data["git_hash"] = Common.getGitHash() - profiling_data["git_status"] = Common.getGitStatus() - - with open(profiling_file, "w") as write_file: - json.dump(profiling_data, write_file) - -#### -# Clean shutdown -#### -sim = None -local_sim = None -cuda_context = None -arguments = None -logging.shutdown() -gc.collect() - - - -#### -# Print completion and exit -#### -print("Completed!") -exit(0) diff --git a/reference/swashes_1_nx=1024.csv b/reference/swashes_1_nx=1024.csv deleted file mode 100644 index bf7bda7..0000000 --- a/reference/swashes_1_nx=1024.csv +++ /dev/null @@ -1,1042 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.00976562 meters -# Number of cells: 1024 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.00488281 0.005 0 0 0 0.005 0 0 -0.0146484 0.005 0 0 0 0.005 0 0 -0.0244141 0.005 0 0 0 0.005 0 0 -0.0341797 0.005 0 0 0 0.005 0 0 -0.0439453 0.005 0 0 0 0.005 0 0 -0.0537109 0.005 0 0 0 0.005 0 0 -0.0634766 0.005 0 0 0 0.005 0 0 -0.0732422 0.005 0 0 0 0.005 0 0 -0.0830078 0.005 0 0 0 0.005 0 0 -0.0927734 0.005 0 0 0 0.005 0 0 -0.102539 0.005 0 0 0 0.005 0 0 -0.112305 0.005 0 0 0 0.005 0 0 -0.12207 0.005 0 0 0 0.005 0 0 -0.131836 0.005 0 0 0 0.005 0 0 -0.141602 0.005 0 0 0 0.005 0 0 -0.151367 0.005 0 0 0 0.005 0 0 -0.161133 0.005 0 0 0 0.005 0 0 -0.170898 0.005 0 0 0 0.005 0 0 -0.180664 0.005 0 0 0 0.005 0 0 -0.19043 0.005 0 0 0 0.005 0 0 -0.200195 0.005 0 0 0 0.005 0 0 -0.209961 0.005 0 0 0 0.005 0 0 -0.219727 0.005 0 0 0 0.005 0 0 -0.229492 0.005 0 0 0 0.005 0 0 -0.239258 0.005 0 0 0 0.005 0 0 -0.249023 0.005 0 0 0 0.005 0 0 -0.258789 0.005 0 0 0 0.005 0 0 -0.268555 0.005 0 0 0 0.005 0 0 -0.27832 0.005 0 0 0 0.005 0 0 -0.288086 0.005 0 0 0 0.005 0 0 -0.297852 0.005 0 0 0 0.005 0 0 -0.307617 0.005 0 0 0 0.005 0 0 -0.317383 0.005 0 0 0 0.005 0 0 -0.327148 0.005 0 0 0 0.005 0 0 -0.336914 0.005 0 0 0 0.005 0 0 -0.34668 0.005 0 0 0 0.005 0 0 -0.356445 0.005 0 0 0 0.005 0 0 -0.366211 0.005 0 0 0 0.005 0 0 -0.375977 0.005 0 0 0 0.005 0 0 -0.385742 0.005 0 0 0 0.005 0 0 -0.395508 0.005 0 0 0 0.005 0 0 -0.405273 0.005 0 0 0 0.005 0 0 -0.415039 0.005 0 0 0 0.005 0 0 -0.424805 0.005 0 0 0 0.005 0 0 -0.43457 0.005 0 0 0 0.005 0 0 -0.444336 0.005 0 0 0 0.005 0 0 -0.454102 0.005 0 0 0 0.005 0 0 -0.463867 0.005 0 0 0 0.005 0 0 -0.473633 0.005 0 0 0 0.005 0 0 -0.483398 0.005 0 0 0 0.005 0 0 -0.493164 0.005 0 0 0 0.005 0 0 -0.50293 0.005 0 0 0 0.005 0 0 -0.512695 0.005 0 0 0 0.005 0 0 -0.522461 0.005 0 0 0 0.005 0 0 -0.532227 0.005 0 0 0 0.005 0 0 -0.541992 0.005 0 0 0 0.005 0 0 -0.551758 0.005 0 0 0 0.005 0 0 -0.561523 0.005 0 0 0 0.005 0 0 -0.571289 0.005 0 0 0 0.005 0 0 -0.581055 0.005 0 0 0 0.005 0 0 -0.59082 0.005 0 0 0 0.005 0 0 -0.600586 0.005 0 0 0 0.005 0 0 -0.610352 0.005 0 0 0 0.005 0 0 -0.620117 0.005 0 0 0 0.005 0 0 -0.629883 0.005 0 0 0 0.005 0 0 -0.639648 0.005 0 0 0 0.005 0 0 -0.649414 0.005 0 0 0 0.005 0 0 -0.65918 0.005 0 0 0 0.005 0 0 -0.668945 0.005 0 0 0 0.005 0 0 -0.678711 0.005 0 0 0 0.005 0 0 -0.688477 0.005 0 0 0 0.005 0 0 -0.698242 0.005 0 0 0 0.005 0 0 -0.708008 0.005 0 0 0 0.005 0 0 -0.717773 0.005 0 0 0 0.005 0 0 -0.727539 0.005 0 0 0 0.005 0 0 -0.737305 0.005 0 0 0 0.005 0 0 -0.74707 0.005 0 0 0 0.005 0 0 -0.756836 0.005 0 0 0 0.005 0 0 -0.766602 0.005 0 0 0 0.005 0 0 -0.776367 0.005 0 0 0 0.005 0 0 -0.786133 0.005 0 0 0 0.005 0 0 -0.795898 0.005 0 0 0 0.005 0 0 -0.805664 0.005 0 0 0 0.005 0 0 -0.81543 0.005 0 0 0 0.005 0 0 -0.825195 0.005 0 0 0 0.005 0 0 -0.834961 0.005 0 0 0 0.005 0 0 -0.844727 0.005 0 0 0 0.005 0 0 -0.854492 0.005 0 0 0 0.005 0 0 -0.864258 0.005 0 0 0 0.005 0 0 -0.874023 0.005 0 0 0 0.005 0 0 -0.883789 0.005 0 0 0 0.005 0 0 -0.893555 0.005 0 0 0 0.005 0 0 -0.90332 0.005 0 0 0 0.005 0 0 -0.913086 0.005 0 0 0 0.005 0 0 -0.922852 0.005 0 0 0 0.005 0 0 -0.932617 0.005 0 0 0 0.005 0 0 -0.942383 0.005 0 0 0 0.005 0 0 -0.952148 0.005 0 0 0 0.005 0 0 -0.961914 0.005 0 0 0 0.005 0 0 -0.97168 0.005 0 0 0 0.005 0 0 -0.981445 0.005 0 0 0 0.005 0 0 -0.991211 0.005 0 0 0 0.005 0 0 -1.00098 0.005 0 0 0 0.005 0 0 -1.01074 0.005 0 0 0 0.005 0 0 -1.02051 0.005 0 0 0 0.005 0 0 -1.03027 0.005 0 0 0 0.005 0 0 -1.04004 0.005 0 0 0 0.005 0 0 -1.0498 0.005 0 0 0 0.005 0 0 -1.05957 0.005 0 0 0 0.005 0 0 -1.06934 0.005 0 0 0 0.005 0 0 -1.0791 0.005 0 0 0 0.005 0 0 -1.08887 0.005 0 0 0 0.005 0 0 -1.09863 0.005 0 0 0 0.005 0 0 -1.1084 0.005 0 0 0 0.005 0 0 -1.11816 0.005 0 0 0 0.005 0 0 -1.12793 0.005 0 0 0 0.005 0 0 -1.1377 0.005 0 0 0 0.005 0 0 -1.14746 0.005 0 0 0 0.005 0 0 -1.15723 0.005 0 0 0 0.005 0 0 -1.16699 0.005 0 0 0 0.005 0 0 -1.17676 0.005 0 0 0 0.005 0 0 -1.18652 0.005 0 0 0 0.005 0 0 -1.19629 0.005 0 0 0 0.005 0 0 -1.20605 0.005 0 0 0 0.005 0 0 -1.21582 0.005 0 0 0 0.005 0 0 -1.22559 0.005 0 0 0 0.005 0 0 -1.23535 0.005 0 0 0 0.005 0 0 -1.24512 0.005 0 0 0 0.005 0 0 -1.25488 0.005 0 0 0 0.005 0 0 -1.26465 0.005 0 0 0 0.005 0 0 -1.27441 0.005 0 0 0 0.005 0 0 -1.28418 0.005 0 0 0 0.005 0 0 -1.29395 0.005 0 0 0 0.005 0 0 -1.30371 0.005 0 0 0 0.005 0 0 -1.31348 0.005 0 0 0 0.005 0 0 -1.32324 0.005 0 0 0 0.005 0 0 -1.33301 0.005 0 0 0 0.005 0 0 -1.34277 0.005 0 0 0 0.005 0 0 -1.35254 0.005 0 0 0 0.005 0 0 -1.3623 0.005 0 0 0 0.005 0 0 -1.37207 0.005 0 0 0 0.005 0 0 -1.38184 0.005 0 0 0 0.005 0 0 -1.3916 0.005 0 0 0 0.005 0 0 -1.40137 0.005 0 0 0 0.005 0 0 -1.41113 0.005 0 0 0 0.005 0 0 -1.4209 0.005 0 0 0 0.005 0 0 -1.43066 0.005 0 0 0 0.005 0 0 -1.44043 0.005 0 0 0 0.005 0 0 -1.4502 0.005 0 0 0 0.005 0 0 -1.45996 0.005 0 0 0 0.005 0 0 -1.46973 0.005 0 0 0 0.005 0 0 -1.47949 0.005 0 0 0 0.005 0 0 -1.48926 0.005 0 0 0 0.005 0 0 -1.49902 0.005 0 0 0 0.005 0 0 -1.50879 0.005 0 0 0 0.005 0 0 -1.51855 0.005 0 0 0 0.005 0 0 -1.52832 0.005 0 0 0 0.005 0 0 -1.53809 0.005 0 0 0 0.005 0 0 -1.54785 0.005 0 0 0 0.005 0 0 -1.55762 0.005 0 0 0 0.005 0 0 -1.56738 0.005 0 0 0 0.005 0 0 -1.57715 0.005 0 0 0 0.005 0 0 -1.58691 0.005 0 0 0 0.005 0 0 -1.59668 0.005 0 0 0 0.005 0 0 -1.60645 0.005 0 0 0 0.005 0 0 -1.61621 0.005 0 0 0 0.005 0 0 -1.62598 0.005 0 0 0 0.005 0 0 -1.63574 0.005 0 0 0 0.005 0 0 -1.64551 0.005 0 0 0 0.005 0 0 -1.65527 0.005 0 0 0 0.005 0 0 -1.66504 0.005 0 0 0 0.005 0 0 -1.6748 0.005 0 0 0 0.005 0 0 -1.68457 0.005 0 0 0 0.005 0 0 -1.69434 0.005 0 0 0 0.005 0 0 -1.7041 0.005 0 0 0 0.005 0 0 -1.71387 0.005 0 0 0 0.005 0 0 -1.72363 0.005 0 0 0 0.005 0 0 -1.7334 0.005 0 0 0 0.005 0 0 -1.74316 0.005 0 0 0 0.005 0 0 -1.75293 0.005 0 0 0 0.005 0 0 -1.7627 0.005 0 0 0 0.005 0 0 -1.77246 0.005 0 0 0 0.005 0 0 -1.78223 0.005 0 0 0 0.005 0 0 -1.79199 0.005 0 0 0 0.005 0 0 -1.80176 0.005 0 0 0 0.005 0 0 -1.81152 0.005 0 0 0 0.005 0 0 -1.82129 0.005 0 0 0 0.005 0 0 -1.83105 0.005 0 0 0 0.005 0 0 -1.84082 0.005 0 0 0 0.005 0 0 -1.85059 0.005 0 0 0 0.005 0 0 -1.86035 0.005 0 0 0 0.005 0 0 -1.87012 0.005 0 0 0 0.005 0 0 -1.87988 0.005 0 0 0 0.005 0 0 -1.88965 0.005 0 0 0 0.005 0 0 -1.89941 0.005 0 0 0 0.005 0 0 -1.90918 0.005 0 0 0 0.005 0 0 -1.91895 0.005 0 0 0 0.005 0 0 -1.92871 0.005 0 0 0 0.005 0 0 -1.93848 0.005 0 0 0 0.005 0 0 -1.94824 0.005 0 0 0 0.005 0 0 -1.95801 0.005 0 0 0 0.005 0 0 -1.96777 0.005 0 0 0 0.005 0 0 -1.97754 0.005 0 0 0 0.005 0 0 -1.9873 0.005 0 0 0 0.005 0 0 -1.99707 0.005 0 0 0 0.005 0 0 -2.00684 0.005 0 0 0 0.005 0 0 -2.0166 0.005 0 0 0 0.005 0 0 -2.02637 0.005 0 0 0 0.005 0 0 -2.03613 0.005 0 0 0 0.005 0 0 -2.0459 0.005 0 0 0 0.005 0 0 -2.05566 0.005 0 0 0 0.005 0 0 -2.06543 0.005 0 0 0 0.005 0 0 -2.0752 0.005 0 0 0 0.005 0 0 -2.08496 0.005 0 0 0 0.005 0 0 -2.09473 0.005 0 0 0 0.005 0 0 -2.10449 0.005 0 0 0 0.005 0 0 -2.11426 0.005 0 0 0 0.005 0 0 -2.12402 0.005 0 0 0 0.005 0 0 -2.13379 0.005 0 0 0 0.005 0 0 -2.14355 0.005 0 0 0 0.005 0 0 -2.15332 0.005 0 0 0 0.005 0 0 -2.16309 0.005 0 0 0 0.005 0 0 -2.17285 0.005 0 0 0 0.005 0 0 -2.18262 0.005 0 0 0 0.005 0 0 -2.19238 0.005 0 0 0 0.005 0 0 -2.20215 0.005 0 0 0 0.005 0 0 -2.21191 0.005 0 0 0 0.005 0 0 -2.22168 0.005 0 0 0 0.005 0 0 -2.23145 0.005 0 0 0 0.005 0 0 -2.24121 0.005 0 0 0 0.005 0 0 -2.25098 0.005 0 0 0 0.005 0 0 -2.26074 0.005 0 0 0 0.005 0 0 -2.27051 0.005 0 0 0 0.005 0 0 -2.28027 0.005 0 0 0 0.005 0 0 -2.29004 0.005 0 0 0 0.005 0 0 -2.2998 0.005 0 0 0 0.005 0 0 -2.30957 0.005 0 0 0 0.005 0 0 -2.31934 0.005 0 0 0 0.005 0 0 -2.3291 0.005 0 0 0 0.005 0 0 -2.33887 0.005 0 0 0 0.005 0 0 -2.34863 0.005 0 0 0 0.005 0 0 -2.3584 0.005 0 0 0 0.005 0 0 -2.36816 0.005 0 0 0 0.005 0 0 -2.37793 0.005 0 0 0 0.005 0 0 -2.3877 0.005 0 0 0 0.005 0 0 -2.39746 0.005 0 0 0 0.005 0 0 -2.40723 0.005 0 0 0 0.005 0 0 -2.41699 0.005 0 0 0 0.005 0 0 -2.42676 0.005 0 0 0 0.005 0 0 -2.43652 0.005 0 0 0 0.005 0 0 -2.44629 0.005 0 0 0 0.005 0 0 -2.45605 0.005 0 0 0 0.005 0 0 -2.46582 0.005 0 0 0 0.005 0 0 -2.47559 0.005 0 0 0 0.005 0 0 -2.48535 0.005 0 0 0 0.005 0 0 -2.49512 0.005 0 0 0 0.005 0 0 -2.50488 0.005 0 0 0 0.005 0 0 -2.51465 0.005 0 0 0 0.005 0 0 -2.52441 0.005 0 0 0 0.005 0 0 -2.53418 0.005 0 0 0 0.005 0 0 -2.54395 0.005 0 0 0 0.005 0 0 -2.55371 0.005 0 0 0 0.005 0 0 -2.56348 0.005 0 0 0 0.005 0 0 -2.57324 0.005 0 0 0 0.005 0 0 -2.58301 0.005 0 0 0 0.005 0 0 -2.59277 0.005 0 0 0 0.005 0 0 -2.60254 0.005 0 0 0 0.005 0 0 -2.6123 0.005 0 0 0 0.005 0 0 -2.62207 0.005 0 0 0 0.005 0 0 -2.63184 0.005 0 0 0 0.005 0 0 -2.6416 0.005 0 0 0 0.005 0 0 -2.65137 0.005 0 0 0 0.005 0 0 -2.66113 0.005 0 0 0 0.005 0 0 -2.6709 0.005 0 0 0 0.005 0 0 -2.68066 0.005 0 0 0 0.005 0 0 -2.69043 0.005 0 0 0 0.005 0 0 -2.7002 0.005 0 0 0 0.005 0 0 -2.70996 0.005 0 0 0 0.005 0 0 -2.71973 0.005 0 0 0 0.005 0 0 -2.72949 0.005 0 0 0 0.005 0 0 -2.73926 0.005 0 0 0 0.005 0 0 -2.74902 0.005 0 0 0 0.005 0 0 -2.75879 0.005 0 0 0 0.005 0 0 -2.76855 0.005 0 0 0 0.005 0 0 -2.77832 0.005 0 0 0 0.005 0 0 -2.78809 0.005 0 0 0 0.005 0 0 -2.79785 0.005 0 0 0 0.005 0 0 -2.80762 0.005 0 0 0 0.005 0 0 -2.81738 0.005 0 0 0 0.005 0 0 -2.82715 0.005 0 0 0 0.005 0 0 -2.83691 0.005 0 0 0 0.005 0 0 -2.84668 0.005 0 0 0 0.005 0 0 -2.85645 0.005 0 0 0 0.005 0 0 -2.86621 0.005 0 0 0 0.005 0 0 -2.87598 0.005 0 0 0 0.005 0 0 -2.88574 0.005 0 0 0 0.005 0 0 -2.89551 0.005 0 0 0 0.005 0 0 -2.90527 0.005 0 0 0 0.005 0 0 -2.91504 0.005 0 0 0 0.005 0 0 -2.9248 0.005 0 0 0 0.005 0 0 -2.93457 0.005 0 0 0 0.005 0 0 -2.94434 0.005 0 0 0 0.005 0 0 -2.9541 0.005 0 0 0 0.005 0 0 -2.96387 0.005 0 0 0 0.005 0 0 -2.97363 0.005 0 0 0 0.005 0 0 -2.9834 0.005 0 0 0 0.005 0 0 -2.99316 0.005 0 0 0 0.005 0 0 -3.00293 0.005 0 0 0 0.005 0 0 -3.0127 0.005 0 0 0 0.005 0 0 -3.02246 0.005 0 0 0 0.005 0 0 -3.03223 0.005 0 0 0 0.005 0 0 -3.04199 0.005 0 0 0 0.005 0 0 -3.05176 0.005 0 0 0 0.005 0 0 -3.06152 0.005 0 0 0 0.005 0 0 -3.07129 0.005 0 0 0 0.005 0 0 -3.08105 0.005 0 0 0 0.005 0 0 -3.09082 0.005 0 0 0 0.005 0 0 -3.10059 0.005 0 0 0 0.005 0 0 -3.11035 0.005 0 0 0 0.005 0 0 -3.12012 0.005 0 0 0 0.005 0 0 -3.12988 0.005 0 0 0 0.005 0 0 -3.13965 0.005 0 0 0 0.005 0 0 -3.14941 0.005 0 0 0 0.005 0 0 -3.15918 0.005 0 0 0 0.005 0 0 -3.16895 0.005 0 0 0 0.005 0 0 -3.17871 0.005 0 0 0 0.005 0 0 -3.18848 0.005 0 0 0 0.005 0 0 -3.19824 0.005 0 0 0 0.005 0 0 -3.20801 0.005 0 0 0 0.005 0 0 -3.21777 0.005 0 0 0 0.005 0 0 -3.22754 0.005 0 0 0 0.005 0 0 -3.2373 0.005 0 0 0 0.005 0 0 -3.24707 0.005 0 0 0 0.005 0 0 -3.25684 0.005 0 0 0 0.005 0 0 -3.2666 0.005 0 0 0 0.005 0 0 -3.27637 0.005 0 0 0 0.005 0 0 -3.28613 0.005 0 0 0 0.005 0 0 -3.2959 0.005 0 0 0 0.005 0 0 -3.30566 0.005 0 0 0 0.005 0 0 -3.31543 0.005 0 0 0 0.005 0 0 -3.3252 0.005 0 0 0 0.005 0 0 -3.33496 0.005 0 0 0 0.005 0 0 -3.34473 0.005 0 0 0 0.005 0 0 -3.35449 0.005 0 0 0 0.005 0 0 -3.36426 0.005 0 0 0 0.005 0 0 -3.37402 0.005 0 0 0 0.005 0 0 -3.38379 0.005 0 0 0 0.005 0 0 -3.39355 0.005 0 0 0 0.005 0 0 -3.40332 0.005 0 0 0 0.005 0 0 -3.41309 0.005 0 0 0 0.005 0 0 -3.42285 0.005 0 0 0 0.005 0 0 -3.43262 0.005 0 0 0 0.005 0 0 -3.44238 0.005 0 0 0 0.005 0 0 -3.45215 0.005 0 0 0 0.005 0 0 -3.46191 0.005 0 0 0 0.005 0 0 -3.47168 0.005 0 0 0 0.005 0 0 -3.48145 0.005 0 0 0 0.005 0 0 -3.49121 0.005 0 0 0 0.005 0 0 -3.50098 0.005 0 0 0 0.005 0 0 -3.51074 0.005 0 0 0 0.005 0 0 -3.52051 0.005 0 0 0 0.005 0 0 -3.53027 0.005 0 0 0 0.005 0 0 -3.54004 0.005 0 0 0 0.005 0 0 -3.5498 0.005 0 0 0 0.005 0 0 -3.55957 0.005 0 0 0 0.005 0 0 -3.56934 0.005 0 0 0 0.005 0 0 -3.5791 0.005 0 0 0 0.005 0 0 -3.58887 0.005 0 0 0 0.005 0 0 -3.59863 0.005 0 0 0 0.005 0 0 -3.6084 0.005 0 0 0 0.005 0 0 -3.61816 0.005 0 0 0 0.005 0 0 -3.62793 0.005 0 0 0 0.005 0 0 -3.6377 0.005 0 0 0 0.005 0 0 -3.64746 0.005 0 0 0 0.005 0 0 -3.65723 0.005 0 0 0 0.005 0 0 -3.66699 0.005 0 0 0 0.005 0 0 -3.67676 0.00498598 0.000621321 0 3.0979e-006 0.00498598 0.00280935 9.92708e-005 -3.68652 0.00496155 0.00170639 0 8.46634e-006 0.00496155 0.00773455 0.000194047 -3.69629 0.00493718 0.00279146 0 1.37819e-005 0.00493718 0.012684 0.000268524 -3.70605 0.00491287 0.00387653 0 1.90449e-005 0.00491287 0.017658 0.000333142 -3.71582 0.00488861 0.0049616 0 2.42553e-005 0.00488861 0.0226566 0.000391425 -3.72559 0.00486442 0.00604667 0 2.94135e-005 0.00486442 0.02768 0.000445118 -3.73535 0.00484029 0.00713174 0 3.45197e-005 0.00484029 0.0327284 0.000495246 -3.74512 0.00481622 0.00821681 0 3.95739e-005 0.00481622 0.0378021 0.000542479 -3.75488 0.0047922 0.00930188 0 4.45765e-005 0.0047922 0.0429011 0.000587283 -3.76465 0.00476825 0.0103869 0 4.95276e-005 0.00476825 0.0480257 0.000630002 -3.77441 0.00474436 0.011472 0 5.44274e-005 0.00474436 0.0531761 0.000670896 -3.78418 0.00472053 0.0125571 0 5.92761e-005 0.00472053 0.0583524 0.000710171 -3.79395 0.00469676 0.0136422 0 6.40739e-005 0.00469676 0.063555 0.000747993 -3.80371 0.00467304 0.0147272 0 6.88209e-005 0.00467304 0.0687839 0.000784496 -3.81348 0.00464939 0.0158123 0 7.35175e-005 0.00464939 0.0740393 0.000819793 -3.82324 0.0046258 0.0168974 0 7.81638e-005 0.0046258 0.0793215 0.000853979 -3.83301 0.00460227 0.0179824 0 8.27599e-005 0.00460227 0.0846307 0.000887136 -3.84277 0.00457879 0.0190675 0 8.73062e-005 0.00457879 0.0899671 0.000919335 -3.85254 0.00455538 0.0201526 0 9.18027e-005 0.00455538 0.0953309 0.000950635 -3.8623 0.00453203 0.0212376 0 9.62496e-005 0.00453203 0.100722 0.000981092 -3.87207 0.00450874 0.0223227 0 0.000100647 0.00450874 0.106141 0.00101075 -3.88184 0.00448551 0.0234078 0 0.000104996 0.00448551 0.111589 0.00103966 -3.8916 0.00446233 0.0244928 0 0.000109295 0.00446233 0.117064 0.00106785 -3.90137 0.00443922 0.0255779 0 0.000113546 0.00443922 0.122568 0.00109536 -3.91113 0.00441617 0.026663 0 0.000117748 0.00441617 0.128101 0.00112223 -3.9209 0.00439318 0.0277481 0 0.000121902 0.00439318 0.133662 0.00114847 -3.93066 0.00437024 0.0288331 0 0.000126008 0.00437024 0.139253 0.00117411 -3.94043 0.00434737 0.0299182 0 0.000130066 0.00434737 0.144873 0.00119918 -3.9502 0.00432456 0.0310033 0 0.000134075 0.00432456 0.150523 0.0012237 -3.95996 0.00430181 0.0320883 0 0.000138038 0.00430181 0.156202 0.0012477 -3.96973 0.00427912 0.0331734 0 0.000141953 0.00427912 0.161912 0.00127118 -3.97949 0.00425648 0.0342585 0 0.000145821 0.00425648 0.167652 0.00129417 -3.98926 0.00423391 0.0353435 0 0.000149641 0.00423391 0.173422 0.00131667 -3.99902 0.0042114 0.0364286 0 0.000153415 0.0042114 0.179223 0.00133872 -4.00879 0.00418895 0.0375137 0 0.000157143 0.00418895 0.185056 0.00136032 -4.01855 0.00416656 0.0385988 0 0.000160824 0.00416656 0.190919 0.00138148 -4.02832 0.00414422 0.0396838 0 0.000164459 0.00414422 0.196815 0.00140222 -4.03809 0.00412195 0.0407689 0 0.000168047 0.00412195 0.202742 0.00142254 -4.04785 0.00409974 0.041854 0 0.00017159 0.00409974 0.208701 0.00144247 -4.05762 0.00407759 0.042939 0 0.000175088 0.00407759 0.214692 0.001462 -4.06738 0.0040555 0.0440241 0 0.00017854 0.0040555 0.220716 0.00148115 -4.07715 0.00403346 0.0451092 0 0.000181946 0.00403346 0.226773 0.00149993 -4.08691 0.00401149 0.0461942 0 0.000185308 0.00401149 0.232863 0.00151835 -4.09668 0.00398958 0.0472793 0 0.000188625 0.00398958 0.238986 0.00153642 -4.10645 0.00396773 0.0483644 0 0.000191897 0.00396773 0.245143 0.00155413 -4.11621 0.00394594 0.0494494 0 0.000195124 0.00394594 0.251334 0.00157151 -4.12598 0.0039242 0.0505345 0 0.000198308 0.0039242 0.25756 0.00158856 -4.13574 0.00390253 0.0516196 0 0.000201447 0.00390253 0.263819 0.00160528 -4.14551 0.00388092 0.0527047 0 0.000204543 0.00388092 0.270114 0.00162168 -4.15527 0.00385937 0.0537897 0 0.000207594 0.00385937 0.276444 0.00163777 -4.16504 0.00383788 0.0548748 0 0.000210603 0.00383788 0.282809 0.00165356 -4.1748 0.00381644 0.0559599 0 0.000213568 0.00381644 0.28921 0.00166904 -4.18457 0.00379507 0.0570449 0 0.00021649 0.00379507 0.295646 0.00168423 -4.19434 0.00377376 0.05813 0 0.000219369 0.00377376 0.302119 0.00169913 -4.2041 0.00375251 0.0592151 0 0.000222205 0.00375251 0.308629 0.00171375 -4.21387 0.00373132 0.0603001 0 0.000224999 0.00373132 0.315176 0.00172808 -4.22363 0.00371018 0.0613852 0 0.00022775 0.00371018 0.32176 0.00174214 -4.2334 0.00368911 0.0624703 0 0.00023046 0.00368911 0.328381 0.00175593 -4.24316 0.0036681 0.0635553 0 0.000233127 0.0036681 0.33504 0.00176945 -4.25293 0.00364715 0.0646404 0 0.000235753 0.00364715 0.341738 0.00178272 -4.2627 0.00362626 0.0657255 0 0.000238338 0.00362626 0.348474 0.00179572 -4.27246 0.00360543 0.0668106 0 0.00024088 0.00360543 0.355249 0.00180847 -4.28223 0.00358465 0.0678956 0 0.000243382 0.00358465 0.362063 0.00182097 -4.29199 0.00356394 0.0689807 0 0.000245843 0.00356394 0.368916 0.00183322 -4.30176 0.00354329 0.0700658 0 0.000248263 0.00354329 0.37581 0.00184524 -4.31152 0.0035227 0.0711508 0 0.000250643 0.0035227 0.382743 0.00185701 -4.32129 0.00350217 0.0722359 0 0.000252982 0.00350217 0.389718 0.00186855 -4.33105 0.00348169 0.073321 0 0.000255281 0.00348169 0.396733 0.00187985 -4.34082 0.00346128 0.074406 0 0.00025754 0.00346128 0.40379 0.00189092 -4.35059 0.00344093 0.0754911 0 0.00025976 0.00344093 0.410888 0.00190177 -4.36035 0.00342064 0.0765762 0 0.000261939 0.00342064 0.418028 0.0019124 -4.37012 0.00340041 0.0776613 0 0.00026408 0.00340041 0.425211 0.0019228 -4.37988 0.00338024 0.0787463 0 0.000266181 0.00338024 0.432436 0.00193299 -4.38965 0.00336012 0.0798314 0 0.000268243 0.00336012 0.439705 0.00194296 -4.39941 0.00334007 0.0809165 0 0.000270267 0.00334007 0.447017 0.00195271 -4.40918 0.00332008 0.0820015 0 0.000272252 0.00332008 0.454374 0.00196226 -4.41895 0.00330015 0.0830866 0 0.000274198 0.00330015 0.461774 0.00197161 -4.42871 0.00328028 0.0841717 0 0.000276106 0.00328028 0.46922 0.00198074 -4.43848 0.00326047 0.0852567 0 0.000277977 0.00326047 0.47671 0.00198968 -4.44824 0.00324071 0.0863418 0 0.000279809 0.00324071 0.484246 0.00199841 -4.45801 0.00322102 0.0874269 0 0.000281604 0.00322102 0.491828 0.00200695 -4.46777 0.00320139 0.0885119 0 0.000283361 0.00320139 0.499457 0.00201529 -4.47754 0.00318182 0.089597 0 0.000285081 0.00318182 0.507132 0.00202344 -4.4873 0.00316231 0.0906821 0 0.000286765 0.00316231 0.514855 0.00203139 -4.49707 0.00314286 0.0917672 0 0.000288411 0.00314286 0.522625 0.00203916 -4.50684 0.00312346 0.0928522 0 0.000290021 0.00312346 0.530444 0.00204674 -4.5166 0.00310413 0.0939373 0 0.000291594 0.00310413 0.538311 0.00205414 -4.52637 0.00308486 0.0950224 0 0.000293131 0.00308486 0.546227 0.00206135 -4.53613 0.00306565 0.0961074 0 0.000294632 0.00306565 0.554193 0.00206838 -4.5459 0.0030465 0.0971925 0 0.000296097 0.0030465 0.562209 0.00207523 -4.55566 0.00302741 0.0982776 0 0.000297526 0.00302741 0.570275 0.0020819 -4.56543 0.00300837 0.0993626 0 0.00029892 0.00300837 0.578392 0.0020884 -4.5752 0.0029894 0.100448 0 0.000300279 0.0029894 0.586561 0.00209472 -4.58496 0.00297049 0.101533 0 0.000301602 0.00297049 0.594782 0.00210087 -4.59473 0.00295164 0.102618 0 0.000302891 0.00295164 0.603055 0.00210685 -4.60449 0.00293285 0.103703 0 0.000304145 0.00293285 0.611381 0.00211267 -4.61426 0.00291412 0.104788 0 0.000305364 0.00291412 0.61976 0.00211831 -4.62402 0.00289545 0.105873 0 0.00030655 0.00289545 0.628193 0.00212379 -4.63379 0.00287683 0.106958 0 0.000307701 0.00287683 0.636681 0.0021291 -4.64355 0.00285828 0.108043 0 0.000308818 0.00285828 0.645224 0.00213425 -4.65332 0.00283979 0.109128 0 0.000309901 0.00283979 0.653822 0.00213924 -4.66309 0.00282136 0.110213 0 0.000310951 0.00282136 0.662476 0.00214407 -4.67285 0.00280299 0.111298 0 0.000311968 0.00280299 0.671187 0.00214874 -4.68262 0.00278468 0.112383 0 0.000312952 0.00278468 0.679956 0.00215325 -4.69238 0.00276643 0.113469 0 0.000313902 0.00276643 0.688782 0.00215761 -4.70215 0.00274823 0.114554 0 0.00031482 0.00274823 0.697666 0.00216182 -4.71191 0.0027301 0.115639 0 0.000315705 0.0027301 0.706609 0.00216587 -4.72168 0.00271203 0.116724 0 0.000316558 0.00271203 0.715612 0.00216977 -4.73145 0.00269402 0.117809 0 0.000317379 0.00269402 0.724674 0.00217352 -4.74121 0.00267607 0.118894 0 0.000318168 0.00267607 0.733798 0.00217712 -4.75098 0.00265818 0.119979 0 0.000318925 0.00265818 0.742983 0.00218057 -4.76074 0.00264035 0.121064 0 0.000319651 0.00264035 0.752229 0.00218387 -4.77051 0.00262257 0.122149 0 0.000320345 0.00262257 0.761539 0.00218703 -4.78027 0.00260486 0.123234 0 0.000321008 0.00260486 0.770911 0.00219005 -4.79004 0.00258721 0.124319 0 0.00032164 0.00258721 0.780347 0.00219293 -4.7998 0.00256962 0.125404 0 0.000322241 0.00256962 0.789848 0.00219566 -4.80957 0.00255209 0.126489 0 0.000322812 0.00255209 0.799414 0.00219825 -4.81934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.92676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01465 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02441 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04395 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05371 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06348 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.07324 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08301 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09277 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10254 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13184 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15137 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16113 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18066 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19043 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20996 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21973 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22949 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23926 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24902 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25879 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26855 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28809 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29785 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30762 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31738 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32715 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33691 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36621 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37598 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38574 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39551 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40527 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41504 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43457 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44434 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46387 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47363 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49316 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52246 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53223 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54199 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55176 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56152 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57129 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58105 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60059 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61035 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62012 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62988 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63965 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64941 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65918 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67871 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68848 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69824 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70801 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71777 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72754 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75684 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77637 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78613 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80566 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83496 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84473 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85449 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86426 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87402 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88379 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90332 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91309 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92285 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93262 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94238 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95215 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96191 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97168 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99121 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00098 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01074 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02051 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03027 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04004 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.17676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.26465 0.001 0 0 0 0.001 0 0 -6.27441 0.001 0 0 0 0.001 0 0 -6.28418 0.001 0 0 0 0.001 0 0 -6.29395 0.001 0 0 0 0.001 0 0 -6.30371 0.001 0 0 0 0.001 0 0 -6.31348 0.001 0 0 0 0.001 0 0 -6.32324 0.001 0 0 0 0.001 0 0 -6.33301 0.001 0 0 0 0.001 0 0 -6.34277 0.001 0 0 0 0.001 0 0 -6.35254 0.001 0 0 0 0.001 0 0 -6.3623 0.001 0 0 0 0.001 0 0 -6.37207 0.001 0 0 0 0.001 0 0 -6.38184 0.001 0 0 0 0.001 0 0 -6.3916 0.001 0 0 0 0.001 0 0 -6.40137 0.001 0 0 0 0.001 0 0 -6.41113 0.001 0 0 0 0.001 0 0 -6.4209 0.001 0 0 0 0.001 0 0 -6.43066 0.001 0 0 0 0.001 0 0 -6.44043 0.001 0 0 0 0.001 0 0 -6.4502 0.001 0 0 0 0.001 0 0 -6.45996 0.001 0 0 0 0.001 0 0 -6.46973 0.001 0 0 0 0.001 0 0 -6.47949 0.001 0 0 0 0.001 0 0 -6.48926 0.001 0 0 0 0.001 0 0 -6.49902 0.001 0 0 0 0.001 0 0 -6.50879 0.001 0 0 0 0.001 0 0 -6.51855 0.001 0 0 0 0.001 0 0 -6.52832 0.001 0 0 0 0.001 0 0 -6.53809 0.001 0 0 0 0.001 0 0 -6.54785 0.001 0 0 0 0.001 0 0 -6.55762 0.001 0 0 0 0.001 0 0 -6.56738 0.001 0 0 0 0.001 0 0 -6.57715 0.001 0 0 0 0.001 0 0 -6.58691 0.001 0 0 0 0.001 0 0 -6.59668 0.001 0 0 0 0.001 0 0 -6.60645 0.001 0 0 0 0.001 0 0 -6.61621 0.001 0 0 0 0.001 0 0 -6.62598 0.001 0 0 0 0.001 0 0 -6.63574 0.001 0 0 0 0.001 0 0 -6.64551 0.001 0 0 0 0.001 0 0 -6.65527 0.001 0 0 0 0.001 0 0 -6.66504 0.001 0 0 0 0.001 0 0 -6.6748 0.001 0 0 0 0.001 0 0 -6.68457 0.001 0 0 0 0.001 0 0 -6.69434 0.001 0 0 0 0.001 0 0 -6.7041 0.001 0 0 0 0.001 0 0 -6.71387 0.001 0 0 0 0.001 0 0 -6.72363 0.001 0 0 0 0.001 0 0 -6.7334 0.001 0 0 0 0.001 0 0 -6.74316 0.001 0 0 0 0.001 0 0 -6.75293 0.001 0 0 0 0.001 0 0 -6.7627 0.001 0 0 0 0.001 0 0 -6.77246 0.001 0 0 0 0.001 0 0 -6.78223 0.001 0 0 0 0.001 0 0 -6.79199 0.001 0 0 0 0.001 0 0 -6.80176 0.001 0 0 0 0.001 0 0 -6.81152 0.001 0 0 0 0.001 0 0 -6.82129 0.001 0 0 0 0.001 0 0 -6.83105 0.001 0 0 0 0.001 0 0 -6.84082 0.001 0 0 0 0.001 0 0 -6.85059 0.001 0 0 0 0.001 0 0 -6.86035 0.001 0 0 0 0.001 0 0 -6.87012 0.001 0 0 0 0.001 0 0 -6.87988 0.001 0 0 0 0.001 0 0 -6.88965 0.001 0 0 0 0.001 0 0 -6.89941 0.001 0 0 0 0.001 0 0 -6.90918 0.001 0 0 0 0.001 0 0 -6.91895 0.001 0 0 0 0.001 0 0 -6.92871 0.001 0 0 0 0.001 0 0 -6.93848 0.001 0 0 0 0.001 0 0 -6.94824 0.001 0 0 0 0.001 0 0 -6.95801 0.001 0 0 0 0.001 0 0 -6.96777 0.001 0 0 0 0.001 0 0 -6.97754 0.001 0 0 0 0.001 0 0 -6.9873 0.001 0 0 0 0.001 0 0 -6.99707 0.001 0 0 0 0.001 0 0 -7.00684 0.001 0 0 0 0.001 0 0 -7.0166 0.001 0 0 0 0.001 0 0 -7.02637 0.001 0 0 0 0.001 0 0 -7.03613 0.001 0 0 0 0.001 0 0 -7.0459 0.001 0 0 0 0.001 0 0 -7.05566 0.001 0 0 0 0.001 0 0 -7.06543 0.001 0 0 0 0.001 0 0 -7.0752 0.001 0 0 0 0.001 0 0 -7.08496 0.001 0 0 0 0.001 0 0 -7.09473 0.001 0 0 0 0.001 0 0 -7.10449 0.001 0 0 0 0.001 0 0 -7.11426 0.001 0 0 0 0.001 0 0 -7.12402 0.001 0 0 0 0.001 0 0 -7.13379 0.001 0 0 0 0.001 0 0 -7.14355 0.001 0 0 0 0.001 0 0 -7.15332 0.001 0 0 0 0.001 0 0 -7.16309 0.001 0 0 0 0.001 0 0 -7.17285 0.001 0 0 0 0.001 0 0 -7.18262 0.001 0 0 0 0.001 0 0 -7.19238 0.001 0 0 0 0.001 0 0 -7.20215 0.001 0 0 0 0.001 0 0 -7.21191 0.001 0 0 0 0.001 0 0 -7.22168 0.001 0 0 0 0.001 0 0 -7.23145 0.001 0 0 0 0.001 0 0 -7.24121 0.001 0 0 0 0.001 0 0 -7.25098 0.001 0 0 0 0.001 0 0 -7.26074 0.001 0 0 0 0.001 0 0 -7.27051 0.001 0 0 0 0.001 0 0 -7.28027 0.001 0 0 0 0.001 0 0 -7.29004 0.001 0 0 0 0.001 0 0 -7.2998 0.001 0 0 0 0.001 0 0 -7.30957 0.001 0 0 0 0.001 0 0 -7.31934 0.001 0 0 0 0.001 0 0 -7.3291 0.001 0 0 0 0.001 0 0 -7.33887 0.001 0 0 0 0.001 0 0 -7.34863 0.001 0 0 0 0.001 0 0 -7.3584 0.001 0 0 0 0.001 0 0 -7.36816 0.001 0 0 0 0.001 0 0 -7.37793 0.001 0 0 0 0.001 0 0 -7.3877 0.001 0 0 0 0.001 0 0 -7.39746 0.001 0 0 0 0.001 0 0 -7.40723 0.001 0 0 0 0.001 0 0 -7.41699 0.001 0 0 0 0.001 0 0 -7.42676 0.001 0 0 0 0.001 0 0 -7.43652 0.001 0 0 0 0.001 0 0 -7.44629 0.001 0 0 0 0.001 0 0 -7.45605 0.001 0 0 0 0.001 0 0 -7.46582 0.001 0 0 0 0.001 0 0 -7.47559 0.001 0 0 0 0.001 0 0 -7.48535 0.001 0 0 0 0.001 0 0 -7.49512 0.001 0 0 0 0.001 0 0 -7.50488 0.001 0 0 0 0.001 0 0 -7.51465 0.001 0 0 0 0.001 0 0 -7.52441 0.001 0 0 0 0.001 0 0 -7.53418 0.001 0 0 0 0.001 0 0 -7.54395 0.001 0 0 0 0.001 0 0 -7.55371 0.001 0 0 0 0.001 0 0 -7.56348 0.001 0 0 0 0.001 0 0 -7.57324 0.001 0 0 0 0.001 0 0 -7.58301 0.001 0 0 0 0.001 0 0 -7.59277 0.001 0 0 0 0.001 0 0 -7.60254 0.001 0 0 0 0.001 0 0 -7.6123 0.001 0 0 0 0.001 0 0 -7.62207 0.001 0 0 0 0.001 0 0 -7.63184 0.001 0 0 0 0.001 0 0 -7.6416 0.001 0 0 0 0.001 0 0 -7.65137 0.001 0 0 0 0.001 0 0 -7.66113 0.001 0 0 0 0.001 0 0 -7.6709 0.001 0 0 0 0.001 0 0 -7.68066 0.001 0 0 0 0.001 0 0 -7.69043 0.001 0 0 0 0.001 0 0 -7.7002 0.001 0 0 0 0.001 0 0 -7.70996 0.001 0 0 0 0.001 0 0 -7.71973 0.001 0 0 0 0.001 0 0 -7.72949 0.001 0 0 0 0.001 0 0 -7.73926 0.001 0 0 0 0.001 0 0 -7.74902 0.001 0 0 0 0.001 0 0 -7.75879 0.001 0 0 0 0.001 0 0 -7.76855 0.001 0 0 0 0.001 0 0 -7.77832 0.001 0 0 0 0.001 0 0 -7.78809 0.001 0 0 0 0.001 0 0 -7.79785 0.001 0 0 0 0.001 0 0 -7.80762 0.001 0 0 0 0.001 0 0 -7.81738 0.001 0 0 0 0.001 0 0 -7.82715 0.001 0 0 0 0.001 0 0 -7.83691 0.001 0 0 0 0.001 0 0 -7.84668 0.001 0 0 0 0.001 0 0 -7.85645 0.001 0 0 0 0.001 0 0 -7.86621 0.001 0 0 0 0.001 0 0 -7.87598 0.001 0 0 0 0.001 0 0 -7.88574 0.001 0 0 0 0.001 0 0 -7.89551 0.001 0 0 0 0.001 0 0 -7.90527 0.001 0 0 0 0.001 0 0 -7.91504 0.001 0 0 0 0.001 0 0 -7.9248 0.001 0 0 0 0.001 0 0 -7.93457 0.001 0 0 0 0.001 0 0 -7.94434 0.001 0 0 0 0.001 0 0 -7.9541 0.001 0 0 0 0.001 0 0 -7.96387 0.001 0 0 0 0.001 0 0 -7.97363 0.001 0 0 0 0.001 0 0 -7.9834 0.001 0 0 0 0.001 0 0 -7.99316 0.001 0 0 0 0.001 0 0 -8.00293 0.001 0 0 0 0.001 0 0 -8.0127 0.001 0 0 0 0.001 0 0 -8.02246 0.001 0 0 0 0.001 0 0 -8.03223 0.001 0 0 0 0.001 0 0 -8.04199 0.001 0 0 0 0.001 0 0 -8.05176 0.001 0 0 0 0.001 0 0 -8.06152 0.001 0 0 0 0.001 0 0 -8.07129 0.001 0 0 0 0.001 0 0 -8.08105 0.001 0 0 0 0.001 0 0 -8.09082 0.001 0 0 0 0.001 0 0 -8.10059 0.001 0 0 0 0.001 0 0 -8.11035 0.001 0 0 0 0.001 0 0 -8.12012 0.001 0 0 0 0.001 0 0 -8.12988 0.001 0 0 0 0.001 0 0 -8.13965 0.001 0 0 0 0.001 0 0 -8.14941 0.001 0 0 0 0.001 0 0 -8.15918 0.001 0 0 0 0.001 0 0 -8.16895 0.001 0 0 0 0.001 0 0 -8.17871 0.001 0 0 0 0.001 0 0 -8.18848 0.001 0 0 0 0.001 0 0 -8.19824 0.001 0 0 0 0.001 0 0 -8.20801 0.001 0 0 0 0.001 0 0 -8.21777 0.001 0 0 0 0.001 0 0 -8.22754 0.001 0 0 0 0.001 0 0 -8.2373 0.001 0 0 0 0.001 0 0 -8.24707 0.001 0 0 0 0.001 0 0 -8.25684 0.001 0 0 0 0.001 0 0 -8.2666 0.001 0 0 0 0.001 0 0 -8.27637 0.001 0 0 0 0.001 0 0 -8.28613 0.001 0 0 0 0.001 0 0 -8.2959 0.001 0 0 0 0.001 0 0 -8.30566 0.001 0 0 0 0.001 0 0 -8.31543 0.001 0 0 0 0.001 0 0 -8.3252 0.001 0 0 0 0.001 0 0 -8.33496 0.001 0 0 0 0.001 0 0 -8.34473 0.001 0 0 0 0.001 0 0 -8.35449 0.001 0 0 0 0.001 0 0 -8.36426 0.001 0 0 0 0.001 0 0 -8.37402 0.001 0 0 0 0.001 0 0 -8.38379 0.001 0 0 0 0.001 0 0 -8.39355 0.001 0 0 0 0.001 0 0 -8.40332 0.001 0 0 0 0.001 0 0 -8.41309 0.001 0 0 0 0.001 0 0 -8.42285 0.001 0 0 0 0.001 0 0 -8.43262 0.001 0 0 0 0.001 0 0 -8.44238 0.001 0 0 0 0.001 0 0 -8.45215 0.001 0 0 0 0.001 0 0 -8.46191 0.001 0 0 0 0.001 0 0 -8.47168 0.001 0 0 0 0.001 0 0 -8.48145 0.001 0 0 0 0.001 0 0 -8.49121 0.001 0 0 0 0.001 0 0 -8.50098 0.001 0 0 0 0.001 0 0 -8.51074 0.001 0 0 0 0.001 0 0 -8.52051 0.001 0 0 0 0.001 0 0 -8.53027 0.001 0 0 0 0.001 0 0 -8.54004 0.001 0 0 0 0.001 0 0 -8.5498 0.001 0 0 0 0.001 0 0 -8.55957 0.001 0 0 0 0.001 0 0 -8.56934 0.001 0 0 0 0.001 0 0 -8.5791 0.001 0 0 0 0.001 0 0 -8.58887 0.001 0 0 0 0.001 0 0 -8.59863 0.001 0 0 0 0.001 0 0 -8.6084 0.001 0 0 0 0.001 0 0 -8.61816 0.001 0 0 0 0.001 0 0 -8.62793 0.001 0 0 0 0.001 0 0 -8.6377 0.001 0 0 0 0.001 0 0 -8.64746 0.001 0 0 0 0.001 0 0 -8.65723 0.001 0 0 0 0.001 0 0 -8.66699 0.001 0 0 0 0.001 0 0 -8.67676 0.001 0 0 0 0.001 0 0 -8.68652 0.001 0 0 0 0.001 0 0 -8.69629 0.001 0 0 0 0.001 0 0 -8.70605 0.001 0 0 0 0.001 0 0 -8.71582 0.001 0 0 0 0.001 0 0 -8.72559 0.001 0 0 0 0.001 0 0 -8.73535 0.001 0 0 0 0.001 0 0 -8.74512 0.001 0 0 0 0.001 0 0 -8.75488 0.001 0 0 0 0.001 0 0 -8.76465 0.001 0 0 0 0.001 0 0 -8.77441 0.001 0 0 0 0.001 0 0 -8.78418 0.001 0 0 0 0.001 0 0 -8.79395 0.001 0 0 0 0.001 0 0 -8.80371 0.001 0 0 0 0.001 0 0 -8.81348 0.001 0 0 0 0.001 0 0 -8.82324 0.001 0 0 0 0.001 0 0 -8.83301 0.001 0 0 0 0.001 0 0 -8.84277 0.001 0 0 0 0.001 0 0 -8.85254 0.001 0 0 0 0.001 0 0 -8.8623 0.001 0 0 0 0.001 0 0 -8.87207 0.001 0 0 0 0.001 0 0 -8.88184 0.001 0 0 0 0.001 0 0 -8.8916 0.001 0 0 0 0.001 0 0 -8.90137 0.001 0 0 0 0.001 0 0 -8.91113 0.001 0 0 0 0.001 0 0 -8.9209 0.001 0 0 0 0.001 0 0 -8.93066 0.001 0 0 0 0.001 0 0 -8.94043 0.001 0 0 0 0.001 0 0 -8.9502 0.001 0 0 0 0.001 0 0 -8.95996 0.001 0 0 0 0.001 0 0 -8.96973 0.001 0 0 0 0.001 0 0 -8.97949 0.001 0 0 0 0.001 0 0 -8.98926 0.001 0 0 0 0.001 0 0 -8.99902 0.001 0 0 0 0.001 0 0 -9.00879 0.001 0 0 0 0.001 0 0 -9.01855 0.001 0 0 0 0.001 0 0 -9.02832 0.001 0 0 0 0.001 0 0 -9.03809 0.001 0 0 0 0.001 0 0 -9.04785 0.001 0 0 0 0.001 0 0 -9.05762 0.001 0 0 0 0.001 0 0 -9.06738 0.001 0 0 0 0.001 0 0 -9.07715 0.001 0 0 0 0.001 0 0 -9.08691 0.001 0 0 0 0.001 0 0 -9.09668 0.001 0 0 0 0.001 0 0 -9.10645 0.001 0 0 0 0.001 0 0 -9.11621 0.001 0 0 0 0.001 0 0 -9.12598 0.001 0 0 0 0.001 0 0 -9.13574 0.001 0 0 0 0.001 0 0 -9.14551 0.001 0 0 0 0.001 0 0 -9.15527 0.001 0 0 0 0.001 0 0 -9.16504 0.001 0 0 0 0.001 0 0 -9.1748 0.001 0 0 0 0.001 0 0 -9.18457 0.001 0 0 0 0.001 0 0 -9.19434 0.001 0 0 0 0.001 0 0 -9.2041 0.001 0 0 0 0.001 0 0 -9.21387 0.001 0 0 0 0.001 0 0 -9.22363 0.001 0 0 0 0.001 0 0 -9.2334 0.001 0 0 0 0.001 0 0 -9.24316 0.001 0 0 0 0.001 0 0 -9.25293 0.001 0 0 0 0.001 0 0 -9.2627 0.001 0 0 0 0.001 0 0 -9.27246 0.001 0 0 0 0.001 0 0 -9.28223 0.001 0 0 0 0.001 0 0 -9.29199 0.001 0 0 0 0.001 0 0 -9.30176 0.001 0 0 0 0.001 0 0 -9.31152 0.001 0 0 0 0.001 0 0 -9.32129 0.001 0 0 0 0.001 0 0 -9.33105 0.001 0 0 0 0.001 0 0 -9.34082 0.001 0 0 0 0.001 0 0 -9.35059 0.001 0 0 0 0.001 0 0 -9.36035 0.001 0 0 0 0.001 0 0 -9.37012 0.001 0 0 0 0.001 0 0 -9.37988 0.001 0 0 0 0.001 0 0 -9.38965 0.001 0 0 0 0.001 0 0 -9.39941 0.001 0 0 0 0.001 0 0 -9.40918 0.001 0 0 0 0.001 0 0 -9.41895 0.001 0 0 0 0.001 0 0 -9.42871 0.001 0 0 0 0.001 0 0 -9.43848 0.001 0 0 0 0.001 0 0 -9.44824 0.001 0 0 0 0.001 0 0 -9.45801 0.001 0 0 0 0.001 0 0 -9.46777 0.001 0 0 0 0.001 0 0 -9.47754 0.001 0 0 0 0.001 0 0 -9.4873 0.001 0 0 0 0.001 0 0 -9.49707 0.001 0 0 0 0.001 0 0 -9.50684 0.001 0 0 0 0.001 0 0 -9.5166 0.001 0 0 0 0.001 0 0 -9.52637 0.001 0 0 0 0.001 0 0 -9.53613 0.001 0 0 0 0.001 0 0 -9.5459 0.001 0 0 0 0.001 0 0 -9.55566 0.001 0 0 0 0.001 0 0 -9.56543 0.001 0 0 0 0.001 0 0 -9.5752 0.001 0 0 0 0.001 0 0 -9.58496 0.001 0 0 0 0.001 0 0 -9.59473 0.001 0 0 0 0.001 0 0 -9.60449 0.001 0 0 0 0.001 0 0 -9.61426 0.001 0 0 0 0.001 0 0 -9.62402 0.001 0 0 0 0.001 0 0 -9.63379 0.001 0 0 0 0.001 0 0 -9.64355 0.001 0 0 0 0.001 0 0 -9.65332 0.001 0 0 0 0.001 0 0 -9.66309 0.001 0 0 0 0.001 0 0 -9.67285 0.001 0 0 0 0.001 0 0 -9.68262 0.001 0 0 0 0.001 0 0 -9.69238 0.001 0 0 0 0.001 0 0 -9.70215 0.001 0 0 0 0.001 0 0 -9.71191 0.001 0 0 0 0.001 0 0 -9.72168 0.001 0 0 0 0.001 0 0 -9.73145 0.001 0 0 0 0.001 0 0 -9.74121 0.001 0 0 0 0.001 0 0 -9.75098 0.001 0 0 0 0.001 0 0 -9.76074 0.001 0 0 0 0.001 0 0 -9.77051 0.001 0 0 0 0.001 0 0 -9.78027 0.001 0 0 0 0.001 0 0 -9.79004 0.001 0 0 0 0.001 0 0 -9.7998 0.001 0 0 0 0.001 0 0 -9.80957 0.001 0 0 0 0.001 0 0 -9.81934 0.001 0 0 0 0.001 0 0 -9.8291 0.001 0 0 0 0.001 0 0 -9.83887 0.001 0 0 0 0.001 0 0 -9.84863 0.001 0 0 0 0.001 0 0 -9.8584 0.001 0 0 0 0.001 0 0 -9.86816 0.001 0 0 0 0.001 0 0 -9.87793 0.001 0 0 0 0.001 0 0 -9.8877 0.001 0 0 0 0.001 0 0 -9.89746 0.001 0 0 0 0.001 0 0 -9.90723 0.001 0 0 0 0.001 0 0 -9.91699 0.001 0 0 0 0.001 0 0 -9.92676 0.001 0 0 0 0.001 0 0 -9.93652 0.001 0 0 0 0.001 0 0 -9.94629 0.001 0 0 0 0.001 0 0 -9.95605 0.001 0 0 0 0.001 0 0 -9.96582 0.001 0 0 0 0.001 0 0 -9.97559 0.001 0 0 0 0.001 0 0 -9.98535 0.001 0 0 0 0.001 0 0 -9.99512 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=128.csv b/reference/swashes_1_nx=128.csv deleted file mode 100644 index b5357d4..0000000 --- a/reference/swashes_1_nx=128.csv +++ /dev/null @@ -1,146 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.078125 meters -# Number of cells: 128 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.0390625 0.005 0 0 0 0.005 0 0 -0.117188 0.005 0 0 0 0.005 0 0 -0.195312 0.005 0 0 0 0.005 0 0 -0.273438 0.005 0 0 0 0.005 0 0 -0.351562 0.005 0 0 0 0.005 0 0 -0.429688 0.005 0 0 0 0.005 0 0 -0.507812 0.005 0 0 0 0.005 0 0 -0.585938 0.005 0 0 0 0.005 0 0 -0.664062 0.005 0 0 0 0.005 0 0 -0.742188 0.005 0 0 0 0.005 0 0 -0.820312 0.005 0 0 0 0.005 0 0 -0.898438 0.005 0 0 0 0.005 0 0 -0.976562 0.005 0 0 0 0.005 0 0 -1.05469 0.005 0 0 0 0.005 0 0 -1.13281 0.005 0 0 0 0.005 0 0 -1.21094 0.005 0 0 0 0.005 0 0 -1.28906 0.005 0 0 0 0.005 0 0 -1.36719 0.005 0 0 0 0.005 0 0 -1.44531 0.005 0 0 0 0.005 0 0 -1.52344 0.005 0 0 0 0.005 0 0 -1.60156 0.005 0 0 0 0.005 0 0 -1.67969 0.005 0 0 0 0.005 0 0 -1.75781 0.005 0 0 0 0.005 0 0 -1.83594 0.005 0 0 0 0.005 0 0 -1.91406 0.005 0 0 0 0.005 0 0 -1.99219 0.005 0 0 0 0.005 0 0 -2.07031 0.005 0 0 0 0.005 0 0 -2.14844 0.005 0 0 0 0.005 0 0 -2.22656 0.005 0 0 0 0.005 0 0 -2.30469 0.005 0 0 0 0.005 0 0 -2.38281 0.005 0 0 0 0.005 0 0 -2.46094 0.005 0 0 0 0.005 0 0 -2.53906 0.005 0 0 0 0.005 0 0 -2.61719 0.005 0 0 0 0.005 0 0 -2.69531 0.005 0 0 0 0.005 0 0 -2.77344 0.005 0 0 0 0.005 0 0 -2.85156 0.005 0 0 0 0.005 0 0 -2.92969 0.005 0 0 0 0.005 0 0 -3.00781 0.005 0 0 0 0.005 0 0 -3.08594 0.005 0 0 0 0.005 0 0 -3.16406 0.005 0 0 0 0.005 0 0 -3.24219 0.005 0 0 0 0.005 0 0 -3.32031 0.005 0 0 0 0.005 0 0 -3.39844 0.005 0 0 0 0.005 0 0 -3.47656 0.005 0 0 0 0.005 0 0 -3.55469 0.005 0 0 0 0.005 0 0 -3.63281 0.005 0 0 0 0.005 0 0 -3.71094 0.00490073 0.00441906 0 2.16566e-005 0.00490073 0.0201542 0.000362943 -3.78906 0.00470863 0.0130996 0 6.16813e-005 0.00470863 0.0609504 0.000729255 -3.86719 0.00452038 0.0217802 0 9.84546e-005 0.00452038 0.103428 0.000996019 -3.94531 0.00433596 0.0304607 0 0.000132076 0.00433596 0.147694 0.00121151 -4.02344 0.00415538 0.0391413 0 0.000162647 0.00415538 0.193863 0.0013919 -4.10156 0.00397865 0.0478218 0 0.000190266 0.00397865 0.242061 0.00154532 -4.17969 0.00380575 0.0565024 0 0.000215034 0.00380575 0.292423 0.00167667 -4.25781 0.0036367 0.065183 0 0.000237051 0.0036367 0.345101 0.00178925 -4.33594 0.00347148 0.0738635 0 0.000256416 0.00347148 0.400256 0.00188541 -4.41406 0.00331011 0.0825441 0 0.00027323 0.00331011 0.458068 0.00196696 -4.49219 0.00315257 0.0912246 0 0.000287592 0.00315257 0.518734 0.0020353 -4.57031 0.00299888 0.0999052 0 0.000299604 0.00299888 0.58247 0.00209158 -4.64844 0.00284903 0.108586 0 0.000309364 0.00284903 0.649516 0.00213677 -4.72656 0.00270302 0.117266 0 0.000316973 0.00270302 0.720135 0.00217166 -4.80469 0.00256085 0.125947 0 0.000322531 0.00256085 0.794623 0.00219697 -4.88281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03906 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11719 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19531 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27344 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35156 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42969 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50781 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58594 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66406 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74219 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82031 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89844 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97656 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05469 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.28906 0.001 0 0 0 0.001 0 0 -6.36719 0.001 0 0 0 0.001 0 0 -6.44531 0.001 0 0 0 0.001 0 0 -6.52344 0.001 0 0 0 0.001 0 0 -6.60156 0.001 0 0 0 0.001 0 0 -6.67969 0.001 0 0 0 0.001 0 0 -6.75781 0.001 0 0 0 0.001 0 0 -6.83594 0.001 0 0 0 0.001 0 0 -6.91406 0.001 0 0 0 0.001 0 0 -6.99219 0.001 0 0 0 0.001 0 0 -7.07031 0.001 0 0 0 0.001 0 0 -7.14844 0.001 0 0 0 0.001 0 0 -7.22656 0.001 0 0 0 0.001 0 0 -7.30469 0.001 0 0 0 0.001 0 0 -7.38281 0.001 0 0 0 0.001 0 0 -7.46094 0.001 0 0 0 0.001 0 0 -7.53906 0.001 0 0 0 0.001 0 0 -7.61719 0.001 0 0 0 0.001 0 0 -7.69531 0.001 0 0 0 0.001 0 0 -7.77344 0.001 0 0 0 0.001 0 0 -7.85156 0.001 0 0 0 0.001 0 0 -7.92969 0.001 0 0 0 0.001 0 0 -8.00781 0.001 0 0 0 0.001 0 0 -8.08594 0.001 0 0 0 0.001 0 0 -8.16406 0.001 0 0 0 0.001 0 0 -8.24219 0.001 0 0 0 0.001 0 0 -8.32031 0.001 0 0 0 0.001 0 0 -8.39844 0.001 0 0 0 0.001 0 0 -8.47656 0.001 0 0 0 0.001 0 0 -8.55469 0.001 0 0 0 0.001 0 0 -8.63281 0.001 0 0 0 0.001 0 0 -8.71094 0.001 0 0 0 0.001 0 0 -8.78906 0.001 0 0 0 0.001 0 0 -8.86719 0.001 0 0 0 0.001 0 0 -8.94531 0.001 0 0 0 0.001 0 0 -9.02344 0.001 0 0 0 0.001 0 0 -9.10156 0.001 0 0 0 0.001 0 0 -9.17969 0.001 0 0 0 0.001 0 0 -9.25781 0.001 0 0 0 0.001 0 0 -9.33594 0.001 0 0 0 0.001 0 0 -9.41406 0.001 0 0 0 0.001 0 0 -9.49219 0.001 0 0 0 0.001 0 0 -9.57031 0.001 0 0 0 0.001 0 0 -9.64844 0.001 0 0 0 0.001 0 0 -9.72656 0.001 0 0 0 0.001 0 0 -9.80469 0.001 0 0 0 0.001 0 0 -9.88281 0.001 0 0 0 0.001 0 0 -9.96094 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=2048.csv b/reference/swashes_1_nx=2048.csv deleted file mode 100644 index 184b240..0000000 --- a/reference/swashes_1_nx=2048.csv +++ /dev/null @@ -1,2066 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.00488281 meters -# Number of cells: 2048 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.00244141 0.005 0 0 0 0.005 0 0 -0.00732422 0.005 0 0 0 0.005 0 0 -0.012207 0.005 0 0 0 0.005 0 0 -0.0170898 0.005 0 0 0 0.005 0 0 -0.0219727 0.005 0 0 0 0.005 0 0 -0.0268555 0.005 0 0 0 0.005 0 0 -0.0317383 0.005 0 0 0 0.005 0 0 -0.0366211 0.005 0 0 0 0.005 0 0 -0.0415039 0.005 0 0 0 0.005 0 0 -0.0463867 0.005 0 0 0 0.005 0 0 -0.0512695 0.005 0 0 0 0.005 0 0 -0.0561523 0.005 0 0 0 0.005 0 0 -0.0610352 0.005 0 0 0 0.005 0 0 -0.065918 0.005 0 0 0 0.005 0 0 -0.0708008 0.005 0 0 0 0.005 0 0 -0.0756836 0.005 0 0 0 0.005 0 0 -0.0805664 0.005 0 0 0 0.005 0 0 -0.0854492 0.005 0 0 0 0.005 0 0 -0.090332 0.005 0 0 0 0.005 0 0 -0.0952148 0.005 0 0 0 0.005 0 0 -0.100098 0.005 0 0 0 0.005 0 0 -0.10498 0.005 0 0 0 0.005 0 0 -0.109863 0.005 0 0 0 0.005 0 0 -0.114746 0.005 0 0 0 0.005 0 0 -0.119629 0.005 0 0 0 0.005 0 0 -0.124512 0.005 0 0 0 0.005 0 0 -0.129395 0.005 0 0 0 0.005 0 0 -0.134277 0.005 0 0 0 0.005 0 0 -0.13916 0.005 0 0 0 0.005 0 0 -0.144043 0.005 0 0 0 0.005 0 0 -0.148926 0.005 0 0 0 0.005 0 0 -0.153809 0.005 0 0 0 0.005 0 0 -0.158691 0.005 0 0 0 0.005 0 0 -0.163574 0.005 0 0 0 0.005 0 0 -0.168457 0.005 0 0 0 0.005 0 0 -0.17334 0.005 0 0 0 0.005 0 0 -0.178223 0.005 0 0 0 0.005 0 0 -0.183105 0.005 0 0 0 0.005 0 0 -0.187988 0.005 0 0 0 0.005 0 0 -0.192871 0.005 0 0 0 0.005 0 0 -0.197754 0.005 0 0 0 0.005 0 0 -0.202637 0.005 0 0 0 0.005 0 0 -0.20752 0.005 0 0 0 0.005 0 0 -0.212402 0.005 0 0 0 0.005 0 0 -0.217285 0.005 0 0 0 0.005 0 0 -0.222168 0.005 0 0 0 0.005 0 0 -0.227051 0.005 0 0 0 0.005 0 0 -0.231934 0.005 0 0 0 0.005 0 0 -0.236816 0.005 0 0 0 0.005 0 0 -0.241699 0.005 0 0 0 0.005 0 0 -0.246582 0.005 0 0 0 0.005 0 0 -0.251465 0.005 0 0 0 0.005 0 0 -0.256348 0.005 0 0 0 0.005 0 0 -0.26123 0.005 0 0 0 0.005 0 0 -0.266113 0.005 0 0 0 0.005 0 0 -0.270996 0.005 0 0 0 0.005 0 0 -0.275879 0.005 0 0 0 0.005 0 0 -0.280762 0.005 0 0 0 0.005 0 0 -0.285645 0.005 0 0 0 0.005 0 0 -0.290527 0.005 0 0 0 0.005 0 0 -0.29541 0.005 0 0 0 0.005 0 0 -0.300293 0.005 0 0 0 0.005 0 0 -0.305176 0.005 0 0 0 0.005 0 0 -0.310059 0.005 0 0 0 0.005 0 0 -0.314941 0.005 0 0 0 0.005 0 0 -0.319824 0.005 0 0 0 0.005 0 0 -0.324707 0.005 0 0 0 0.005 0 0 -0.32959 0.005 0 0 0 0.005 0 0 -0.334473 0.005 0 0 0 0.005 0 0 -0.339355 0.005 0 0 0 0.005 0 0 -0.344238 0.005 0 0 0 0.005 0 0 -0.349121 0.005 0 0 0 0.005 0 0 -0.354004 0.005 0 0 0 0.005 0 0 -0.358887 0.005 0 0 0 0.005 0 0 -0.36377 0.005 0 0 0 0.005 0 0 -0.368652 0.005 0 0 0 0.005 0 0 -0.373535 0.005 0 0 0 0.005 0 0 -0.378418 0.005 0 0 0 0.005 0 0 -0.383301 0.005 0 0 0 0.005 0 0 -0.388184 0.005 0 0 0 0.005 0 0 -0.393066 0.005 0 0 0 0.005 0 0 -0.397949 0.005 0 0 0 0.005 0 0 -0.402832 0.005 0 0 0 0.005 0 0 -0.407715 0.005 0 0 0 0.005 0 0 -0.412598 0.005 0 0 0 0.005 0 0 -0.41748 0.005 0 0 0 0.005 0 0 -0.422363 0.005 0 0 0 0.005 0 0 -0.427246 0.005 0 0 0 0.005 0 0 -0.432129 0.005 0 0 0 0.005 0 0 -0.437012 0.005 0 0 0 0.005 0 0 -0.441895 0.005 0 0 0 0.005 0 0 -0.446777 0.005 0 0 0 0.005 0 0 -0.45166 0.005 0 0 0 0.005 0 0 -0.456543 0.005 0 0 0 0.005 0 0 -0.461426 0.005 0 0 0 0.005 0 0 -0.466309 0.005 0 0 0 0.005 0 0 -0.471191 0.005 0 0 0 0.005 0 0 -0.476074 0.005 0 0 0 0.005 0 0 -0.480957 0.005 0 0 0 0.005 0 0 -0.48584 0.005 0 0 0 0.005 0 0 -0.490723 0.005 0 0 0 0.005 0 0 -0.495605 0.005 0 0 0 0.005 0 0 -0.500488 0.005 0 0 0 0.005 0 0 -0.505371 0.005 0 0 0 0.005 0 0 -0.510254 0.005 0 0 0 0.005 0 0 -0.515137 0.005 0 0 0 0.005 0 0 -0.52002 0.005 0 0 0 0.005 0 0 -0.524902 0.005 0 0 0 0.005 0 0 -0.529785 0.005 0 0 0 0.005 0 0 -0.534668 0.005 0 0 0 0.005 0 0 -0.539551 0.005 0 0 0 0.005 0 0 -0.544434 0.005 0 0 0 0.005 0 0 -0.549316 0.005 0 0 0 0.005 0 0 -0.554199 0.005 0 0 0 0.005 0 0 -0.559082 0.005 0 0 0 0.005 0 0 -0.563965 0.005 0 0 0 0.005 0 0 -0.568848 0.005 0 0 0 0.005 0 0 -0.57373 0.005 0 0 0 0.005 0 0 -0.578613 0.005 0 0 0 0.005 0 0 -0.583496 0.005 0 0 0 0.005 0 0 -0.588379 0.005 0 0 0 0.005 0 0 -0.593262 0.005 0 0 0 0.005 0 0 -0.598145 0.005 0 0 0 0.005 0 0 -0.603027 0.005 0 0 0 0.005 0 0 -0.60791 0.005 0 0 0 0.005 0 0 -0.612793 0.005 0 0 0 0.005 0 0 -0.617676 0.005 0 0 0 0.005 0 0 -0.622559 0.005 0 0 0 0.005 0 0 -0.627441 0.005 0 0 0 0.005 0 0 -0.632324 0.005 0 0 0 0.005 0 0 -0.637207 0.005 0 0 0 0.005 0 0 -0.64209 0.005 0 0 0 0.005 0 0 -0.646973 0.005 0 0 0 0.005 0 0 -0.651855 0.005 0 0 0 0.005 0 0 -0.656738 0.005 0 0 0 0.005 0 0 -0.661621 0.005 0 0 0 0.005 0 0 -0.666504 0.005 0 0 0 0.005 0 0 -0.671387 0.005 0 0 0 0.005 0 0 -0.67627 0.005 0 0 0 0.005 0 0 -0.681152 0.005 0 0 0 0.005 0 0 -0.686035 0.005 0 0 0 0.005 0 0 -0.690918 0.005 0 0 0 0.005 0 0 -0.695801 0.005 0 0 0 0.005 0 0 -0.700684 0.005 0 0 0 0.005 0 0 -0.705566 0.005 0 0 0 0.005 0 0 -0.710449 0.005 0 0 0 0.005 0 0 -0.715332 0.005 0 0 0 0.005 0 0 -0.720215 0.005 0 0 0 0.005 0 0 -0.725098 0.005 0 0 0 0.005 0 0 -0.72998 0.005 0 0 0 0.005 0 0 -0.734863 0.005 0 0 0 0.005 0 0 -0.739746 0.005 0 0 0 0.005 0 0 -0.744629 0.005 0 0 0 0.005 0 0 -0.749512 0.005 0 0 0 0.005 0 0 -0.754395 0.005 0 0 0 0.005 0 0 -0.759277 0.005 0 0 0 0.005 0 0 -0.76416 0.005 0 0 0 0.005 0 0 -0.769043 0.005 0 0 0 0.005 0 0 -0.773926 0.005 0 0 0 0.005 0 0 -0.778809 0.005 0 0 0 0.005 0 0 -0.783691 0.005 0 0 0 0.005 0 0 -0.788574 0.005 0 0 0 0.005 0 0 -0.793457 0.005 0 0 0 0.005 0 0 -0.79834 0.005 0 0 0 0.005 0 0 -0.803223 0.005 0 0 0 0.005 0 0 -0.808105 0.005 0 0 0 0.005 0 0 -0.812988 0.005 0 0 0 0.005 0 0 -0.817871 0.005 0 0 0 0.005 0 0 -0.822754 0.005 0 0 0 0.005 0 0 -0.827637 0.005 0 0 0 0.005 0 0 -0.83252 0.005 0 0 0 0.005 0 0 -0.837402 0.005 0 0 0 0.005 0 0 -0.842285 0.005 0 0 0 0.005 0 0 -0.847168 0.005 0 0 0 0.005 0 0 -0.852051 0.005 0 0 0 0.005 0 0 -0.856934 0.005 0 0 0 0.005 0 0 -0.861816 0.005 0 0 0 0.005 0 0 -0.866699 0.005 0 0 0 0.005 0 0 -0.871582 0.005 0 0 0 0.005 0 0 -0.876465 0.005 0 0 0 0.005 0 0 -0.881348 0.005 0 0 0 0.005 0 0 -0.88623 0.005 0 0 0 0.005 0 0 -0.891113 0.005 0 0 0 0.005 0 0 -0.895996 0.005 0 0 0 0.005 0 0 -0.900879 0.005 0 0 0 0.005 0 0 -0.905762 0.005 0 0 0 0.005 0 0 -0.910645 0.005 0 0 0 0.005 0 0 -0.915527 0.005 0 0 0 0.005 0 0 -0.92041 0.005 0 0 0 0.005 0 0 -0.925293 0.005 0 0 0 0.005 0 0 -0.930176 0.005 0 0 0 0.005 0 0 -0.935059 0.005 0 0 0 0.005 0 0 -0.939941 0.005 0 0 0 0.005 0 0 -0.944824 0.005 0 0 0 0.005 0 0 -0.949707 0.005 0 0 0 0.005 0 0 -0.95459 0.005 0 0 0 0.005 0 0 -0.959473 0.005 0 0 0 0.005 0 0 -0.964355 0.005 0 0 0 0.005 0 0 -0.969238 0.005 0 0 0 0.005 0 0 -0.974121 0.005 0 0 0 0.005 0 0 -0.979004 0.005 0 0 0 0.005 0 0 -0.983887 0.005 0 0 0 0.005 0 0 -0.98877 0.005 0 0 0 0.005 0 0 -0.993652 0.005 0 0 0 0.005 0 0 -0.998535 0.005 0 0 0 0.005 0 0 -1.00342 0.005 0 0 0 0.005 0 0 -1.0083 0.005 0 0 0 0.005 0 0 -1.01318 0.005 0 0 0 0.005 0 0 -1.01807 0.005 0 0 0 0.005 0 0 -1.02295 0.005 0 0 0 0.005 0 0 -1.02783 0.005 0 0 0 0.005 0 0 -1.03271 0.005 0 0 0 0.005 0 0 -1.0376 0.005 0 0 0 0.005 0 0 -1.04248 0.005 0 0 0 0.005 0 0 -1.04736 0.005 0 0 0 0.005 0 0 -1.05225 0.005 0 0 0 0.005 0 0 -1.05713 0.005 0 0 0 0.005 0 0 -1.06201 0.005 0 0 0 0.005 0 0 -1.06689 0.005 0 0 0 0.005 0 0 -1.07178 0.005 0 0 0 0.005 0 0 -1.07666 0.005 0 0 0 0.005 0 0 -1.08154 0.005 0 0 0 0.005 0 0 -1.08643 0.005 0 0 0 0.005 0 0 -1.09131 0.005 0 0 0 0.005 0 0 -1.09619 0.005 0 0 0 0.005 0 0 -1.10107 0.005 0 0 0 0.005 0 0 -1.10596 0.005 0 0 0 0.005 0 0 -1.11084 0.005 0 0 0 0.005 0 0 -1.11572 0.005 0 0 0 0.005 0 0 -1.12061 0.005 0 0 0 0.005 0 0 -1.12549 0.005 0 0 0 0.005 0 0 -1.13037 0.005 0 0 0 0.005 0 0 -1.13525 0.005 0 0 0 0.005 0 0 -1.14014 0.005 0 0 0 0.005 0 0 -1.14502 0.005 0 0 0 0.005 0 0 -1.1499 0.005 0 0 0 0.005 0 0 -1.15479 0.005 0 0 0 0.005 0 0 -1.15967 0.005 0 0 0 0.005 0 0 -1.16455 0.005 0 0 0 0.005 0 0 -1.16943 0.005 0 0 0 0.005 0 0 -1.17432 0.005 0 0 0 0.005 0 0 -1.1792 0.005 0 0 0 0.005 0 0 -1.18408 0.005 0 0 0 0.005 0 0 -1.18896 0.005 0 0 0 0.005 0 0 -1.19385 0.005 0 0 0 0.005 0 0 -1.19873 0.005 0 0 0 0.005 0 0 -1.20361 0.005 0 0 0 0.005 0 0 -1.2085 0.005 0 0 0 0.005 0 0 -1.21338 0.005 0 0 0 0.005 0 0 -1.21826 0.005 0 0 0 0.005 0 0 -1.22314 0.005 0 0 0 0.005 0 0 -1.22803 0.005 0 0 0 0.005 0 0 -1.23291 0.005 0 0 0 0.005 0 0 -1.23779 0.005 0 0 0 0.005 0 0 -1.24268 0.005 0 0 0 0.005 0 0 -1.24756 0.005 0 0 0 0.005 0 0 -1.25244 0.005 0 0 0 0.005 0 0 -1.25732 0.005 0 0 0 0.005 0 0 -1.26221 0.005 0 0 0 0.005 0 0 -1.26709 0.005 0 0 0 0.005 0 0 -1.27197 0.005 0 0 0 0.005 0 0 -1.27686 0.005 0 0 0 0.005 0 0 -1.28174 0.005 0 0 0 0.005 0 0 -1.28662 0.005 0 0 0 0.005 0 0 -1.2915 0.005 0 0 0 0.005 0 0 -1.29639 0.005 0 0 0 0.005 0 0 -1.30127 0.005 0 0 0 0.005 0 0 -1.30615 0.005 0 0 0 0.005 0 0 -1.31104 0.005 0 0 0 0.005 0 0 -1.31592 0.005 0 0 0 0.005 0 0 -1.3208 0.005 0 0 0 0.005 0 0 -1.32568 0.005 0 0 0 0.005 0 0 -1.33057 0.005 0 0 0 0.005 0 0 -1.33545 0.005 0 0 0 0.005 0 0 -1.34033 0.005 0 0 0 0.005 0 0 -1.34521 0.005 0 0 0 0.005 0 0 -1.3501 0.005 0 0 0 0.005 0 0 -1.35498 0.005 0 0 0 0.005 0 0 -1.35986 0.005 0 0 0 0.005 0 0 -1.36475 0.005 0 0 0 0.005 0 0 -1.36963 0.005 0 0 0 0.005 0 0 -1.37451 0.005 0 0 0 0.005 0 0 -1.37939 0.005 0 0 0 0.005 0 0 -1.38428 0.005 0 0 0 0.005 0 0 -1.38916 0.005 0 0 0 0.005 0 0 -1.39404 0.005 0 0 0 0.005 0 0 -1.39893 0.005 0 0 0 0.005 0 0 -1.40381 0.005 0 0 0 0.005 0 0 -1.40869 0.005 0 0 0 0.005 0 0 -1.41357 0.005 0 0 0 0.005 0 0 -1.41846 0.005 0 0 0 0.005 0 0 -1.42334 0.005 0 0 0 0.005 0 0 -1.42822 0.005 0 0 0 0.005 0 0 -1.43311 0.005 0 0 0 0.005 0 0 -1.43799 0.005 0 0 0 0.005 0 0 -1.44287 0.005 0 0 0 0.005 0 0 -1.44775 0.005 0 0 0 0.005 0 0 -1.45264 0.005 0 0 0 0.005 0 0 -1.45752 0.005 0 0 0 0.005 0 0 -1.4624 0.005 0 0 0 0.005 0 0 -1.46729 0.005 0 0 0 0.005 0 0 -1.47217 0.005 0 0 0 0.005 0 0 -1.47705 0.005 0 0 0 0.005 0 0 -1.48193 0.005 0 0 0 0.005 0 0 -1.48682 0.005 0 0 0 0.005 0 0 -1.4917 0.005 0 0 0 0.005 0 0 -1.49658 0.005 0 0 0 0.005 0 0 -1.50146 0.005 0 0 0 0.005 0 0 -1.50635 0.005 0 0 0 0.005 0 0 -1.51123 0.005 0 0 0 0.005 0 0 -1.51611 0.005 0 0 0 0.005 0 0 -1.521 0.005 0 0 0 0.005 0 0 -1.52588 0.005 0 0 0 0.005 0 0 -1.53076 0.005 0 0 0 0.005 0 0 -1.53564 0.005 0 0 0 0.005 0 0 -1.54053 0.005 0 0 0 0.005 0 0 -1.54541 0.005 0 0 0 0.005 0 0 -1.55029 0.005 0 0 0 0.005 0 0 -1.55518 0.005 0 0 0 0.005 0 0 -1.56006 0.005 0 0 0 0.005 0 0 -1.56494 0.005 0 0 0 0.005 0 0 -1.56982 0.005 0 0 0 0.005 0 0 -1.57471 0.005 0 0 0 0.005 0 0 -1.57959 0.005 0 0 0 0.005 0 0 -1.58447 0.005 0 0 0 0.005 0 0 -1.58936 0.005 0 0 0 0.005 0 0 -1.59424 0.005 0 0 0 0.005 0 0 -1.59912 0.005 0 0 0 0.005 0 0 -1.604 0.005 0 0 0 0.005 0 0 -1.60889 0.005 0 0 0 0.005 0 0 -1.61377 0.005 0 0 0 0.005 0 0 -1.61865 0.005 0 0 0 0.005 0 0 -1.62354 0.005 0 0 0 0.005 0 0 -1.62842 0.005 0 0 0 0.005 0 0 -1.6333 0.005 0 0 0 0.005 0 0 -1.63818 0.005 0 0 0 0.005 0 0 -1.64307 0.005 0 0 0 0.005 0 0 -1.64795 0.005 0 0 0 0.005 0 0 -1.65283 0.005 0 0 0 0.005 0 0 -1.65771 0.005 0 0 0 0.005 0 0 -1.6626 0.005 0 0 0 0.005 0 0 -1.66748 0.005 0 0 0 0.005 0 0 -1.67236 0.005 0 0 0 0.005 0 0 -1.67725 0.005 0 0 0 0.005 0 0 -1.68213 0.005 0 0 0 0.005 0 0 -1.68701 0.005 0 0 0 0.005 0 0 -1.69189 0.005 0 0 0 0.005 0 0 -1.69678 0.005 0 0 0 0.005 0 0 -1.70166 0.005 0 0 0 0.005 0 0 -1.70654 0.005 0 0 0 0.005 0 0 -1.71143 0.005 0 0 0 0.005 0 0 -1.71631 0.005 0 0 0 0.005 0 0 -1.72119 0.005 0 0 0 0.005 0 0 -1.72607 0.005 0 0 0 0.005 0 0 -1.73096 0.005 0 0 0 0.005 0 0 -1.73584 0.005 0 0 0 0.005 0 0 -1.74072 0.005 0 0 0 0.005 0 0 -1.74561 0.005 0 0 0 0.005 0 0 -1.75049 0.005 0 0 0 0.005 0 0 -1.75537 0.005 0 0 0 0.005 0 0 -1.76025 0.005 0 0 0 0.005 0 0 -1.76514 0.005 0 0 0 0.005 0 0 -1.77002 0.005 0 0 0 0.005 0 0 -1.7749 0.005 0 0 0 0.005 0 0 -1.77979 0.005 0 0 0 0.005 0 0 -1.78467 0.005 0 0 0 0.005 0 0 -1.78955 0.005 0 0 0 0.005 0 0 -1.79443 0.005 0 0 0 0.005 0 0 -1.79932 0.005 0 0 0 0.005 0 0 -1.8042 0.005 0 0 0 0.005 0 0 -1.80908 0.005 0 0 0 0.005 0 0 -1.81396 0.005 0 0 0 0.005 0 0 -1.81885 0.005 0 0 0 0.005 0 0 -1.82373 0.005 0 0 0 0.005 0 0 -1.82861 0.005 0 0 0 0.005 0 0 -1.8335 0.005 0 0 0 0.005 0 0 -1.83838 0.005 0 0 0 0.005 0 0 -1.84326 0.005 0 0 0 0.005 0 0 -1.84814 0.005 0 0 0 0.005 0 0 -1.85303 0.005 0 0 0 0.005 0 0 -1.85791 0.005 0 0 0 0.005 0 0 -1.86279 0.005 0 0 0 0.005 0 0 -1.86768 0.005 0 0 0 0.005 0 0 -1.87256 0.005 0 0 0 0.005 0 0 -1.87744 0.005 0 0 0 0.005 0 0 -1.88232 0.005 0 0 0 0.005 0 0 -1.88721 0.005 0 0 0 0.005 0 0 -1.89209 0.005 0 0 0 0.005 0 0 -1.89697 0.005 0 0 0 0.005 0 0 -1.90186 0.005 0 0 0 0.005 0 0 -1.90674 0.005 0 0 0 0.005 0 0 -1.91162 0.005 0 0 0 0.005 0 0 -1.9165 0.005 0 0 0 0.005 0 0 -1.92139 0.005 0 0 0 0.005 0 0 -1.92627 0.005 0 0 0 0.005 0 0 -1.93115 0.005 0 0 0 0.005 0 0 -1.93604 0.005 0 0 0 0.005 0 0 -1.94092 0.005 0 0 0 0.005 0 0 -1.9458 0.005 0 0 0 0.005 0 0 -1.95068 0.005 0 0 0 0.005 0 0 -1.95557 0.005 0 0 0 0.005 0 0 -1.96045 0.005 0 0 0 0.005 0 0 -1.96533 0.005 0 0 0 0.005 0 0 -1.97021 0.005 0 0 0 0.005 0 0 -1.9751 0.005 0 0 0 0.005 0 0 -1.97998 0.005 0 0 0 0.005 0 0 -1.98486 0.005 0 0 0 0.005 0 0 -1.98975 0.005 0 0 0 0.005 0 0 -1.99463 0.005 0 0 0 0.005 0 0 -1.99951 0.005 0 0 0 0.005 0 0 -2.00439 0.005 0 0 0 0.005 0 0 -2.00928 0.005 0 0 0 0.005 0 0 -2.01416 0.005 0 0 0 0.005 0 0 -2.01904 0.005 0 0 0 0.005 0 0 -2.02393 0.005 0 0 0 0.005 0 0 -2.02881 0.005 0 0 0 0.005 0 0 -2.03369 0.005 0 0 0 0.005 0 0 -2.03857 0.005 0 0 0 0.005 0 0 -2.04346 0.005 0 0 0 0.005 0 0 -2.04834 0.005 0 0 0 0.005 0 0 -2.05322 0.005 0 0 0 0.005 0 0 -2.05811 0.005 0 0 0 0.005 0 0 -2.06299 0.005 0 0 0 0.005 0 0 -2.06787 0.005 0 0 0 0.005 0 0 -2.07275 0.005 0 0 0 0.005 0 0 -2.07764 0.005 0 0 0 0.005 0 0 -2.08252 0.005 0 0 0 0.005 0 0 -2.0874 0.005 0 0 0 0.005 0 0 -2.09229 0.005 0 0 0 0.005 0 0 -2.09717 0.005 0 0 0 0.005 0 0 -2.10205 0.005 0 0 0 0.005 0 0 -2.10693 0.005 0 0 0 0.005 0 0 -2.11182 0.005 0 0 0 0.005 0 0 -2.1167 0.005 0 0 0 0.005 0 0 -2.12158 0.005 0 0 0 0.005 0 0 -2.12646 0.005 0 0 0 0.005 0 0 -2.13135 0.005 0 0 0 0.005 0 0 -2.13623 0.005 0 0 0 0.005 0 0 -2.14111 0.005 0 0 0 0.005 0 0 -2.146 0.005 0 0 0 0.005 0 0 -2.15088 0.005 0 0 0 0.005 0 0 -2.15576 0.005 0 0 0 0.005 0 0 -2.16064 0.005 0 0 0 0.005 0 0 -2.16553 0.005 0 0 0 0.005 0 0 -2.17041 0.005 0 0 0 0.005 0 0 -2.17529 0.005 0 0 0 0.005 0 0 -2.18018 0.005 0 0 0 0.005 0 0 -2.18506 0.005 0 0 0 0.005 0 0 -2.18994 0.005 0 0 0 0.005 0 0 -2.19482 0.005 0 0 0 0.005 0 0 -2.19971 0.005 0 0 0 0.005 0 0 -2.20459 0.005 0 0 0 0.005 0 0 -2.20947 0.005 0 0 0 0.005 0 0 -2.21436 0.005 0 0 0 0.005 0 0 -2.21924 0.005 0 0 0 0.005 0 0 -2.22412 0.005 0 0 0 0.005 0 0 -2.229 0.005 0 0 0 0.005 0 0 -2.23389 0.005 0 0 0 0.005 0 0 -2.23877 0.005 0 0 0 0.005 0 0 -2.24365 0.005 0 0 0 0.005 0 0 -2.24854 0.005 0 0 0 0.005 0 0 -2.25342 0.005 0 0 0 0.005 0 0 -2.2583 0.005 0 0 0 0.005 0 0 -2.26318 0.005 0 0 0 0.005 0 0 -2.26807 0.005 0 0 0 0.005 0 0 -2.27295 0.005 0 0 0 0.005 0 0 -2.27783 0.005 0 0 0 0.005 0 0 -2.28271 0.005 0 0 0 0.005 0 0 -2.2876 0.005 0 0 0 0.005 0 0 -2.29248 0.005 0 0 0 0.005 0 0 -2.29736 0.005 0 0 0 0.005 0 0 -2.30225 0.005 0 0 0 0.005 0 0 -2.30713 0.005 0 0 0 0.005 0 0 -2.31201 0.005 0 0 0 0.005 0 0 -2.31689 0.005 0 0 0 0.005 0 0 -2.32178 0.005 0 0 0 0.005 0 0 -2.32666 0.005 0 0 0 0.005 0 0 -2.33154 0.005 0 0 0 0.005 0 0 -2.33643 0.005 0 0 0 0.005 0 0 -2.34131 0.005 0 0 0 0.005 0 0 -2.34619 0.005 0 0 0 0.005 0 0 -2.35107 0.005 0 0 0 0.005 0 0 -2.35596 0.005 0 0 0 0.005 0 0 -2.36084 0.005 0 0 0 0.005 0 0 -2.36572 0.005 0 0 0 0.005 0 0 -2.37061 0.005 0 0 0 0.005 0 0 -2.37549 0.005 0 0 0 0.005 0 0 -2.38037 0.005 0 0 0 0.005 0 0 -2.38525 0.005 0 0 0 0.005 0 0 -2.39014 0.005 0 0 0 0.005 0 0 -2.39502 0.005 0 0 0 0.005 0 0 -2.3999 0.005 0 0 0 0.005 0 0 -2.40479 0.005 0 0 0 0.005 0 0 -2.40967 0.005 0 0 0 0.005 0 0 -2.41455 0.005 0 0 0 0.005 0 0 -2.41943 0.005 0 0 0 0.005 0 0 -2.42432 0.005 0 0 0 0.005 0 0 -2.4292 0.005 0 0 0 0.005 0 0 -2.43408 0.005 0 0 0 0.005 0 0 -2.43896 0.005 0 0 0 0.005 0 0 -2.44385 0.005 0 0 0 0.005 0 0 -2.44873 0.005 0 0 0 0.005 0 0 -2.45361 0.005 0 0 0 0.005 0 0 -2.4585 0.005 0 0 0 0.005 0 0 -2.46338 0.005 0 0 0 0.005 0 0 -2.46826 0.005 0 0 0 0.005 0 0 -2.47314 0.005 0 0 0 0.005 0 0 -2.47803 0.005 0 0 0 0.005 0 0 -2.48291 0.005 0 0 0 0.005 0 0 -2.48779 0.005 0 0 0 0.005 0 0 -2.49268 0.005 0 0 0 0.005 0 0 -2.49756 0.005 0 0 0 0.005 0 0 -2.50244 0.005 0 0 0 0.005 0 0 -2.50732 0.005 0 0 0 0.005 0 0 -2.51221 0.005 0 0 0 0.005 0 0 -2.51709 0.005 0 0 0 0.005 0 0 -2.52197 0.005 0 0 0 0.005 0 0 -2.52686 0.005 0 0 0 0.005 0 0 -2.53174 0.005 0 0 0 0.005 0 0 -2.53662 0.005 0 0 0 0.005 0 0 -2.5415 0.005 0 0 0 0.005 0 0 -2.54639 0.005 0 0 0 0.005 0 0 -2.55127 0.005 0 0 0 0.005 0 0 -2.55615 0.005 0 0 0 0.005 0 0 -2.56104 0.005 0 0 0 0.005 0 0 -2.56592 0.005 0 0 0 0.005 0 0 -2.5708 0.005 0 0 0 0.005 0 0 -2.57568 0.005 0 0 0 0.005 0 0 -2.58057 0.005 0 0 0 0.005 0 0 -2.58545 0.005 0 0 0 0.005 0 0 -2.59033 0.005 0 0 0 0.005 0 0 -2.59521 0.005 0 0 0 0.005 0 0 -2.6001 0.005 0 0 0 0.005 0 0 -2.60498 0.005 0 0 0 0.005 0 0 -2.60986 0.005 0 0 0 0.005 0 0 -2.61475 0.005 0 0 0 0.005 0 0 -2.61963 0.005 0 0 0 0.005 0 0 -2.62451 0.005 0 0 0 0.005 0 0 -2.62939 0.005 0 0 0 0.005 0 0 -2.63428 0.005 0 0 0 0.005 0 0 -2.63916 0.005 0 0 0 0.005 0 0 -2.64404 0.005 0 0 0 0.005 0 0 -2.64893 0.005 0 0 0 0.005 0 0 -2.65381 0.005 0 0 0 0.005 0 0 -2.65869 0.005 0 0 0 0.005 0 0 -2.66357 0.005 0 0 0 0.005 0 0 -2.66846 0.005 0 0 0 0.005 0 0 -2.67334 0.005 0 0 0 0.005 0 0 -2.67822 0.005 0 0 0 0.005 0 0 -2.68311 0.005 0 0 0 0.005 0 0 -2.68799 0.005 0 0 0 0.005 0 0 -2.69287 0.005 0 0 0 0.005 0 0 -2.69775 0.005 0 0 0 0.005 0 0 -2.70264 0.005 0 0 0 0.005 0 0 -2.70752 0.005 0 0 0 0.005 0 0 -2.7124 0.005 0 0 0 0.005 0 0 -2.71729 0.005 0 0 0 0.005 0 0 -2.72217 0.005 0 0 0 0.005 0 0 -2.72705 0.005 0 0 0 0.005 0 0 -2.73193 0.005 0 0 0 0.005 0 0 -2.73682 0.005 0 0 0 0.005 0 0 -2.7417 0.005 0 0 0 0.005 0 0 -2.74658 0.005 0 0 0 0.005 0 0 -2.75146 0.005 0 0 0 0.005 0 0 -2.75635 0.005 0 0 0 0.005 0 0 -2.76123 0.005 0 0 0 0.005 0 0 -2.76611 0.005 0 0 0 0.005 0 0 -2.771 0.005 0 0 0 0.005 0 0 -2.77588 0.005 0 0 0 0.005 0 0 -2.78076 0.005 0 0 0 0.005 0 0 -2.78564 0.005 0 0 0 0.005 0 0 -2.79053 0.005 0 0 0 0.005 0 0 -2.79541 0.005 0 0 0 0.005 0 0 -2.80029 0.005 0 0 0 0.005 0 0 -2.80518 0.005 0 0 0 0.005 0 0 -2.81006 0.005 0 0 0 0.005 0 0 -2.81494 0.005 0 0 0 0.005 0 0 -2.81982 0.005 0 0 0 0.005 0 0 -2.82471 0.005 0 0 0 0.005 0 0 -2.82959 0.005 0 0 0 0.005 0 0 -2.83447 0.005 0 0 0 0.005 0 0 -2.83936 0.005 0 0 0 0.005 0 0 -2.84424 0.005 0 0 0 0.005 0 0 -2.84912 0.005 0 0 0 0.005 0 0 -2.854 0.005 0 0 0 0.005 0 0 -2.85889 0.005 0 0 0 0.005 0 0 -2.86377 0.005 0 0 0 0.005 0 0 -2.86865 0.005 0 0 0 0.005 0 0 -2.87354 0.005 0 0 0 0.005 0 0 -2.87842 0.005 0 0 0 0.005 0 0 -2.8833 0.005 0 0 0 0.005 0 0 -2.88818 0.005 0 0 0 0.005 0 0 -2.89307 0.005 0 0 0 0.005 0 0 -2.89795 0.005 0 0 0 0.005 0 0 -2.90283 0.005 0 0 0 0.005 0 0 -2.90771 0.005 0 0 0 0.005 0 0 -2.9126 0.005 0 0 0 0.005 0 0 -2.91748 0.005 0 0 0 0.005 0 0 -2.92236 0.005 0 0 0 0.005 0 0 -2.92725 0.005 0 0 0 0.005 0 0 -2.93213 0.005 0 0 0 0.005 0 0 -2.93701 0.005 0 0 0 0.005 0 0 -2.94189 0.005 0 0 0 0.005 0 0 -2.94678 0.005 0 0 0 0.005 0 0 -2.95166 0.005 0 0 0 0.005 0 0 -2.95654 0.005 0 0 0 0.005 0 0 -2.96143 0.005 0 0 0 0.005 0 0 -2.96631 0.005 0 0 0 0.005 0 0 -2.97119 0.005 0 0 0 0.005 0 0 -2.97607 0.005 0 0 0 0.005 0 0 -2.98096 0.005 0 0 0 0.005 0 0 -2.98584 0.005 0 0 0 0.005 0 0 -2.99072 0.005 0 0 0 0.005 0 0 -2.99561 0.005 0 0 0 0.005 0 0 -3.00049 0.005 0 0 0 0.005 0 0 -3.00537 0.005 0 0 0 0.005 0 0 -3.01025 0.005 0 0 0 0.005 0 0 -3.01514 0.005 0 0 0 0.005 0 0 -3.02002 0.005 0 0 0 0.005 0 0 -3.0249 0.005 0 0 0 0.005 0 0 -3.02979 0.005 0 0 0 0.005 0 0 -3.03467 0.005 0 0 0 0.005 0 0 -3.03955 0.005 0 0 0 0.005 0 0 -3.04443 0.005 0 0 0 0.005 0 0 -3.04932 0.005 0 0 0 0.005 0 0 -3.0542 0.005 0 0 0 0.005 0 0 -3.05908 0.005 0 0 0 0.005 0 0 -3.06396 0.005 0 0 0 0.005 0 0 -3.06885 0.005 0 0 0 0.005 0 0 -3.07373 0.005 0 0 0 0.005 0 0 -3.07861 0.005 0 0 0 0.005 0 0 -3.0835 0.005 0 0 0 0.005 0 0 -3.08838 0.005 0 0 0 0.005 0 0 -3.09326 0.005 0 0 0 0.005 0 0 -3.09814 0.005 0 0 0 0.005 0 0 -3.10303 0.005 0 0 0 0.005 0 0 -3.10791 0.005 0 0 0 0.005 0 0 -3.11279 0.005 0 0 0 0.005 0 0 -3.11768 0.005 0 0 0 0.005 0 0 -3.12256 0.005 0 0 0 0.005 0 0 -3.12744 0.005 0 0 0 0.005 0 0 -3.13232 0.005 0 0 0 0.005 0 0 -3.13721 0.005 0 0 0 0.005 0 0 -3.14209 0.005 0 0 0 0.005 0 0 -3.14697 0.005 0 0 0 0.005 0 0 -3.15186 0.005 0 0 0 0.005 0 0 -3.15674 0.005 0 0 0 0.005 0 0 -3.16162 0.005 0 0 0 0.005 0 0 -3.1665 0.005 0 0 0 0.005 0 0 -3.17139 0.005 0 0 0 0.005 0 0 -3.17627 0.005 0 0 0 0.005 0 0 -3.18115 0.005 0 0 0 0.005 0 0 -3.18604 0.005 0 0 0 0.005 0 0 -3.19092 0.005 0 0 0 0.005 0 0 -3.1958 0.005 0 0 0 0.005 0 0 -3.20068 0.005 0 0 0 0.005 0 0 -3.20557 0.005 0 0 0 0.005 0 0 -3.21045 0.005 0 0 0 0.005 0 0 -3.21533 0.005 0 0 0 0.005 0 0 -3.22021 0.005 0 0 0 0.005 0 0 -3.2251 0.005 0 0 0 0.005 0 0 -3.22998 0.005 0 0 0 0.005 0 0 -3.23486 0.005 0 0 0 0.005 0 0 -3.23975 0.005 0 0 0 0.005 0 0 -3.24463 0.005 0 0 0 0.005 0 0 -3.24951 0.005 0 0 0 0.005 0 0 -3.25439 0.005 0 0 0 0.005 0 0 -3.25928 0.005 0 0 0 0.005 0 0 -3.26416 0.005 0 0 0 0.005 0 0 -3.26904 0.005 0 0 0 0.005 0 0 -3.27393 0.005 0 0 0 0.005 0 0 -3.27881 0.005 0 0 0 0.005 0 0 -3.28369 0.005 0 0 0 0.005 0 0 -3.28857 0.005 0 0 0 0.005 0 0 -3.29346 0.005 0 0 0 0.005 0 0 -3.29834 0.005 0 0 0 0.005 0 0 -3.30322 0.005 0 0 0 0.005 0 0 -3.30811 0.005 0 0 0 0.005 0 0 -3.31299 0.005 0 0 0 0.005 0 0 -3.31787 0.005 0 0 0 0.005 0 0 -3.32275 0.005 0 0 0 0.005 0 0 -3.32764 0.005 0 0 0 0.005 0 0 -3.33252 0.005 0 0 0 0.005 0 0 -3.3374 0.005 0 0 0 0.005 0 0 -3.34229 0.005 0 0 0 0.005 0 0 -3.34717 0.005 0 0 0 0.005 0 0 -3.35205 0.005 0 0 0 0.005 0 0 -3.35693 0.005 0 0 0 0.005 0 0 -3.36182 0.005 0 0 0 0.005 0 0 -3.3667 0.005 0 0 0 0.005 0 0 -3.37158 0.005 0 0 0 0.005 0 0 -3.37646 0.005 0 0 0 0.005 0 0 -3.38135 0.005 0 0 0 0.005 0 0 -3.38623 0.005 0 0 0 0.005 0 0 -3.39111 0.005 0 0 0 0.005 0 0 -3.396 0.005 0 0 0 0.005 0 0 -3.40088 0.005 0 0 0 0.005 0 0 -3.40576 0.005 0 0 0 0.005 0 0 -3.41064 0.005 0 0 0 0.005 0 0 -3.41553 0.005 0 0 0 0.005 0 0 -3.42041 0.005 0 0 0 0.005 0 0 -3.42529 0.005 0 0 0 0.005 0 0 -3.43018 0.005 0 0 0 0.005 0 0 -3.43506 0.005 0 0 0 0.005 0 0 -3.43994 0.005 0 0 0 0.005 0 0 -3.44482 0.005 0 0 0 0.005 0 0 -3.44971 0.005 0 0 0 0.005 0 0 -3.45459 0.005 0 0 0 0.005 0 0 -3.45947 0.005 0 0 0 0.005 0 0 -3.46436 0.005 0 0 0 0.005 0 0 -3.46924 0.005 0 0 0 0.005 0 0 -3.47412 0.005 0 0 0 0.005 0 0 -3.479 0.005 0 0 0 0.005 0 0 -3.48389 0.005 0 0 0 0.005 0 0 -3.48877 0.005 0 0 0 0.005 0 0 -3.49365 0.005 0 0 0 0.005 0 0 -3.49854 0.005 0 0 0 0.005 0 0 -3.50342 0.005 0 0 0 0.005 0 0 -3.5083 0.005 0 0 0 0.005 0 0 -3.51318 0.005 0 0 0 0.005 0 0 -3.51807 0.005 0 0 0 0.005 0 0 -3.52295 0.005 0 0 0 0.005 0 0 -3.52783 0.005 0 0 0 0.005 0 0 -3.53271 0.005 0 0 0 0.005 0 0 -3.5376 0.005 0 0 0 0.005 0 0 -3.54248 0.005 0 0 0 0.005 0 0 -3.54736 0.005 0 0 0 0.005 0 0 -3.55225 0.005 0 0 0 0.005 0 0 -3.55713 0.005 0 0 0 0.005 0 0 -3.56201 0.005 0 0 0 0.005 0 0 -3.56689 0.005 0 0 0 0.005 0 0 -3.57178 0.005 0 0 0 0.005 0 0 -3.57666 0.005 0 0 0 0.005 0 0 -3.58154 0.005 0 0 0 0.005 0 0 -3.58643 0.005 0 0 0 0.005 0 0 -3.59131 0.005 0 0 0 0.005 0 0 -3.59619 0.005 0 0 0 0.005 0 0 -3.60107 0.005 0 0 0 0.005 0 0 -3.60596 0.005 0 0 0 0.005 0 0 -3.61084 0.005 0 0 0 0.005 0 0 -3.61572 0.005 0 0 0 0.005 0 0 -3.62061 0.005 0 0 0 0.005 0 0 -3.62549 0.005 0 0 0 0.005 0 0 -3.63037 0.005 0 0 0 0.005 0 0 -3.63525 0.005 0 0 0 0.005 0 0 -3.64014 0.005 0 0 0 0.005 0 0 -3.64502 0.005 0 0 0 0.005 0 0 -3.6499 0.005 0 0 0 0.005 0 0 -3.65479 0.005 0 0 0 0.005 0 0 -3.65967 0.005 0 0 0 0.005 0 0 -3.66455 0.005 0 0 0 0.005 0 0 -3.66943 0.005 0 0 0 0.005 0 0 -3.67432 0.0049921 0.000350054 0 1.7475e-006 0.0049921 0.00158182 6.77728e-005 -3.6792 0.00497987 0.000892588 0 4.44497e-006 0.00497987 0.00403838 0.000126286 -3.68408 0.00496765 0.00143512 0 7.12919e-006 0.00496765 0.00650098 0.000173036 -3.68896 0.00495545 0.00197766 0 9.80019e-006 0.00495545 0.00896964 0.000213927 -3.69385 0.00494327 0.00252019 0 1.2458e-005 0.00494327 0.0114444 0.00025104 -3.69873 0.00493109 0.00306273 0 1.51026e-005 0.00493109 0.0139252 0.000285416 -3.70361 0.00491894 0.00360526 0 1.77341e-005 0.00491894 0.0164122 0.000317675 -3.7085 0.0049068 0.0041478 0 2.03524e-005 0.0049068 0.0189053 0.00034822 -3.71338 0.00489467 0.00469033 0 2.29576e-005 0.00489467 0.0214046 0.000377336 -3.71826 0.00488256 0.00523287 0 2.55498e-005 0.00488256 0.0239101 0.00040523 -3.72314 0.00487046 0.0057754 0 2.81289e-005 0.00487046 0.0264218 0.000432061 -3.72803 0.00485838 0.00631794 0 3.06949e-005 0.00485838 0.0289398 0.000457954 -3.73291 0.00484632 0.00686047 0 3.3248e-005 0.00484632 0.031464 0.000483008 -3.73779 0.00483427 0.007403 0 3.57881e-005 0.00483427 0.0339945 0.000507305 -3.74268 0.00482223 0.00794554 0 3.83152e-005 0.00482223 0.0365313 0.000530914 -3.74756 0.00481021 0.00848807 0 4.08294e-005 0.00481021 0.0390744 0.000553893 -3.75244 0.0047982 0.00903061 0 4.33307e-005 0.0047982 0.041624 0.000576289 -3.75732 0.00478621 0.00957314 0 4.58191e-005 0.00478621 0.0441798 0.000598147 -3.76221 0.00477423 0.0101157 0 4.82946e-005 0.00477423 0.0467421 0.000619502 -3.76709 0.00476227 0.0106582 0 5.07573e-005 0.00476227 0.0493109 0.000640387 -3.77197 0.00475033 0.0112007 0 5.32072e-005 0.00475033 0.0518861 0.000660831 -3.77686 0.0047384 0.0117433 0 5.56443e-005 0.0047384 0.0544677 0.00068086 -3.78174 0.00472648 0.0122858 0 5.80687e-005 0.00472648 0.0570559 0.000700495 -3.78662 0.00471458 0.0128284 0 6.04803e-005 0.00471458 0.0596506 0.000719757 -3.7915 0.00470269 0.0133709 0 6.28792e-005 0.00470269 0.0622519 0.000738666 -3.79639 0.00469082 0.0139134 0 6.52654e-005 0.00469082 0.0648597 0.000757238 -3.80127 0.00467897 0.014456 0 6.76389e-005 0.00467897 0.0674741 0.000775487 -3.80615 0.00466712 0.0149985 0 6.99998e-005 0.00466712 0.0700952 0.000793429 -3.81104 0.0046553 0.015541 0 7.23481e-005 0.0046553 0.0727229 0.000811076 -3.81592 0.00464349 0.0160836 0 7.46838e-005 0.00464349 0.0753573 0.00082844 -3.8208 0.00463169 0.0166261 0 7.70069e-005 0.00463169 0.0779985 0.000845532 -3.82568 0.00461991 0.0171686 0 7.93175e-005 0.00461991 0.0806463 0.000862362 -3.83057 0.00460814 0.0177112 0 8.16156e-005 0.00460814 0.0833009 0.00087894 -3.83545 0.00459639 0.0182537 0 8.39012e-005 0.00459639 0.0859623 0.000895273 -3.84033 0.00458466 0.0187962 0 8.61743e-005 0.00458466 0.0886304 0.000911371 -3.84521 0.00457294 0.0193388 0 8.84349e-005 0.00457294 0.0913055 0.000927242 -3.8501 0.00456123 0.0198813 0 9.06832e-005 0.00456123 0.0939874 0.000942891 -3.85498 0.00454954 0.0204238 0 9.2919e-005 0.00454954 0.0966761 0.000958326 -3.85986 0.00453786 0.0209664 0 9.51425e-005 0.00453786 0.0993718 0.000973554 -3.86475 0.0045262 0.0215089 0 9.73536e-005 0.0045262 0.102074 0.00098858 -3.86963 0.00451455 0.0220514 0 9.95524e-005 0.00451455 0.104784 0.00100341 -3.87451 0.00450292 0.022594 0 0.000101739 0.00450292 0.107501 0.00101805 -3.87939 0.00449131 0.0231365 0 0.000103913 0.00449131 0.110224 0.0010325 -3.88428 0.00447971 0.023679 0 0.000106075 0.00447971 0.112955 0.00104677 -3.88916 0.00446812 0.0242216 0 0.000108225 0.00446812 0.115693 0.00106087 -3.89404 0.00445655 0.0247641 0 0.000110363 0.00445655 0.118437 0.00107479 -3.89893 0.00444499 0.0253067 0 0.000112488 0.00444499 0.121189 0.00108855 -3.90381 0.00443345 0.0258492 0 0.000114601 0.00443345 0.123949 0.00110214 -3.90869 0.00442193 0.0263917 0 0.000116702 0.00442193 0.126715 0.00111557 -3.91357 0.00441041 0.0269343 0 0.000118791 0.00441041 0.129488 0.00112884 -3.91846 0.00439892 0.0274768 0 0.000120868 0.00439892 0.132269 0.00114196 -3.92334 0.00438744 0.0280193 0 0.000122933 0.00438744 0.135057 0.00115493 -3.92822 0.00437597 0.0285619 0 0.000124986 0.00437597 0.137853 0.00116775 -3.93311 0.00436452 0.0291044 0 0.000127027 0.00436452 0.140655 0.00118043 -3.93799 0.00435308 0.0296469 0 0.000129056 0.00435308 0.143465 0.00119297 -3.94287 0.00434166 0.0301895 0 0.000131072 0.00434166 0.146283 0.00120536 -3.94775 0.00433026 0.030732 0 0.000133077 0.00433026 0.149107 0.00121762 -3.95264 0.00431887 0.0312745 0 0.000135071 0.00431887 0.15194 0.00122975 -3.95752 0.00430749 0.0318171 0 0.000137052 0.00430749 0.15478 0.00124175 -3.9624 0.00429613 0.0323596 0 0.000139021 0.00429613 0.157627 0.00125361 -3.96729 0.00428478 0.0329021 0 0.000140979 0.00428478 0.160482 0.00126535 -3.97217 0.00427345 0.0334447 0 0.000142924 0.00427345 0.163344 0.00127697 -3.97705 0.00426214 0.0339872 0 0.000144858 0.00426214 0.166214 0.00128846 -3.98193 0.00425084 0.0345297 0 0.00014678 0.00425084 0.169091 0.00129984 -3.98682 0.00423955 0.0350723 0 0.000148691 0.00423955 0.171977 0.00131109 -3.9917 0.00422828 0.0356148 0 0.000150589 0.00422828 0.17487 0.00132223 -3.99658 0.00421702 0.0361573 0 0.000152476 0.00421702 0.17777 0.00133325 -4.00146 0.00420578 0.0366999 0 0.000154352 0.00420578 0.180679 0.00134416 -4.00635 0.00419455 0.0372424 0 0.000156215 0.00419455 0.183595 0.00135496 -4.01123 0.00418334 0.0377849 0 0.000158067 0.00418334 0.186519 0.00136565 -4.01611 0.00417215 0.0383275 0 0.000159908 0.00417215 0.189451 0.00137623 -4.021 0.00416097 0.03887 0 0.000161737 0.00416097 0.19239 0.0013867 -4.02588 0.0041498 0.0394126 0 0.000163554 0.0041498 0.195338 0.00139707 -4.03076 0.00413865 0.0399551 0 0.00016536 0.00413865 0.198293 0.00140734 -4.03564 0.00412751 0.0404976 0 0.000167154 0.00412751 0.201257 0.0014175 -4.04053 0.00411639 0.0410402 0 0.000168937 0.00411639 0.204228 0.00142756 -4.04541 0.00410529 0.0415827 0 0.000170709 0.00410529 0.207208 0.00143752 -4.05029 0.0040942 0.0421252 0 0.000172469 0.0040942 0.210196 0.00144739 -4.05518 0.00408312 0.0426678 0 0.000174218 0.00408312 0.213191 0.00145715 -4.06006 0.00407206 0.0432103 0 0.000175955 0.00407206 0.216195 0.00146682 -4.06494 0.00406101 0.0437528 0 0.000177681 0.00406101 0.219207 0.0014764 -4.06982 0.00404998 0.0442954 0 0.000179395 0.00404998 0.222227 0.00148588 -4.07471 0.00403897 0.0448379 0 0.000181099 0.00403897 0.225256 0.00149527 -4.07959 0.00402796 0.0453804 0 0.000182791 0.00402796 0.228292 0.00150457 -4.08447 0.00401698 0.045923 0 0.000184472 0.00401698 0.231337 0.00151378 -4.08936 0.00400601 0.0464655 0 0.000186141 0.00400601 0.234391 0.0015229 -4.09424 0.00399505 0.047008 0 0.0001878 0.00399505 0.237452 0.00153193 -4.09912 0.00398411 0.0475506 0 0.000189447 0.00398411 0.240522 0.00154088 -4.104 0.00397318 0.0480931 0 0.000191083 0.00397318 0.243601 0.00154974 -4.10889 0.00396227 0.0486356 0 0.000192708 0.00396227 0.246688 0.00155851 -4.11377 0.00395138 0.0491782 0 0.000194322 0.00395138 0.249783 0.0015672 -4.11865 0.0039405 0.0497207 0 0.000195924 0.0039405 0.252887 0.00157581 -4.12354 0.00392963 0.0502632 0 0.000197516 0.00392963 0.256 0.00158433 -4.12842 0.00391878 0.0508058 0 0.000199097 0.00391878 0.259121 0.00159277 -4.1333 0.00390794 0.0513483 0 0.000200666 0.00390794 0.262251 0.00160113 -4.13818 0.00389712 0.0518909 0 0.000202225 0.00389712 0.26539 0.00160941 -4.14307 0.00388632 0.0524334 0 0.000203773 0.00388632 0.268537 0.00161761 -4.14795 0.00387553 0.0529759 0 0.00020531 0.00387553 0.271693 0.00162574 -4.15283 0.00386475 0.0535185 0 0.000206835 0.00386475 0.274858 0.00163378 -4.15771 0.00385399 0.054061 0 0.00020835 0.00385399 0.278032 0.00164175 -4.1626 0.00384324 0.0546035 0 0.000209855 0.00384324 0.281214 0.00164964 -4.16748 0.00383251 0.0551461 0 0.000211348 0.00383251 0.284406 0.00165746 -4.17236 0.0038218 0.0556886 0 0.00021283 0.0038218 0.287606 0.0016652 -4.17725 0.0038111 0.0562311 0 0.000214302 0.0038111 0.290815 0.00167287 -4.18213 0.00380041 0.0567737 0 0.000215763 0.00380041 0.294034 0.00168046 -4.18701 0.00378974 0.0573162 0 0.000217213 0.00378974 0.297261 0.00168798 -4.19189 0.00377908 0.0578587 0 0.000218653 0.00377908 0.300498 0.00169543 -4.19678 0.00376844 0.0584013 0 0.000220082 0.00376844 0.303743 0.00170281 -4.20166 0.00375782 0.0589438 0 0.0002215 0.00375782 0.306998 0.00171012 -4.20654 0.0037472 0.0594863 0 0.000222907 0.0037472 0.310262 0.00171736 -4.21143 0.00373661 0.0600289 0 0.000224304 0.00373661 0.313536 0.00172452 -4.21631 0.00372603 0.0605714 0 0.000225691 0.00372603 0.316818 0.00173162 -4.22119 0.00371546 0.0611139 0 0.000227067 0.00371546 0.32011 0.00173865 -4.22607 0.00370491 0.0616565 0 0.000228432 0.00370491 0.323411 0.00174561 -4.23096 0.00369437 0.062199 0 0.000229786 0.00369437 0.326722 0.00175251 -4.23584 0.00368385 0.0627415 0 0.000231131 0.00368385 0.330042 0.00175934 -4.24072 0.00367335 0.0632841 0 0.000232464 0.00367335 0.333372 0.0017661 -4.24561 0.00366286 0.0638266 0 0.000233788 0.00366286 0.336711 0.00177279 -4.25049 0.00365238 0.0643692 0 0.000235101 0.00365238 0.34006 0.00177942 -4.25537 0.00364192 0.0649117 0 0.000236403 0.00364192 0.343418 0.00178599 -4.26025 0.00363147 0.0654542 0 0.000237695 0.00363147 0.346786 0.00179249 -4.26514 0.00362104 0.0659968 0 0.000238977 0.00362104 0.350164 0.00179893 -4.27002 0.00361063 0.0665393 0 0.000240249 0.00361063 0.353551 0.00180531 -4.2749 0.00360023 0.0670818 0 0.00024151 0.00360023 0.356948 0.00181162 -4.27979 0.00358984 0.0676244 0 0.000242761 0.00358984 0.360355 0.00181787 -4.28467 0.00357947 0.0681669 0 0.000244001 0.00357947 0.363772 0.00182406 -4.28955 0.00356911 0.0687094 0 0.000245232 0.00356911 0.367199 0.00183018 -4.29443 0.00355877 0.069252 0 0.000246452 0.00355877 0.370636 0.00183625 -4.29932 0.00354845 0.0697945 0 0.000247662 0.00354845 0.374083 0.00184226 -4.3042 0.00353814 0.070337 0 0.000248862 0.00353814 0.377539 0.0018482 -4.30908 0.00352784 0.0708796 0 0.000250052 0.00352784 0.381006 0.00185409 -4.31396 0.00351756 0.0714221 0 0.000251231 0.00351756 0.384483 0.00185991 -4.31885 0.00350729 0.0719646 0 0.000252401 0.00350729 0.38797 0.00186568 -4.32373 0.00349704 0.0725072 0 0.000253561 0.00349704 0.391468 0.00187139 -4.32861 0.00348681 0.0730497 0 0.00025471 0.00348681 0.394975 0.00187704 -4.3335 0.00347659 0.0735922 0 0.00025585 0.00347659 0.398493 0.00188264 -4.33838 0.00346638 0.0741348 0 0.000256979 0.00346638 0.402022 0.00188818 -4.34326 0.00345619 0.0746773 0 0.000258099 0.00345619 0.40556 0.00189366 -4.34814 0.00344601 0.0752198 0 0.000259209 0.00344601 0.409109 0.00189908 -4.35303 0.00343585 0.0757624 0 0.000260308 0.00343585 0.412669 0.00190445 -4.35791 0.00342571 0.0763049 0 0.000261398 0.00342571 0.416239 0.00190976 -4.36279 0.00341558 0.0768474 0 0.000262478 0.00341558 0.41982 0.00191502 -4.36768 0.00340546 0.07739 0 0.000263548 0.00340546 0.423411 0.00192022 -4.37256 0.00339536 0.0779325 0 0.000264609 0.00339536 0.427013 0.00192537 -4.37744 0.00338527 0.0784751 0 0.000265659 0.00338527 0.430626 0.00193046 -4.38232 0.0033752 0.0790176 0 0.0002667 0.0033752 0.43425 0.0019355 -4.38721 0.00336515 0.0795601 0 0.000267731 0.00336515 0.437884 0.00194048 -4.39209 0.00335511 0.0801027 0 0.000268753 0.00335511 0.441529 0.00194542 -4.39697 0.00334508 0.0806452 0 0.000269765 0.00334508 0.445185 0.00195029 -4.40186 0.00333507 0.0811877 0 0.000270767 0.00333507 0.448852 0.00195512 -4.40674 0.00332507 0.0817303 0 0.000271759 0.00332507 0.45253 0.0019599 -4.41162 0.00331509 0.0822728 0 0.000272742 0.00331509 0.45622 0.00196462 -4.4165 0.00330513 0.0828153 0 0.000273715 0.00330513 0.45992 0.00196929 -4.42139 0.00329518 0.0833579 0 0.000274679 0.00329518 0.463631 0.00197391 -4.42627 0.00328524 0.0839004 0 0.000275633 0.00328524 0.467354 0.00197848 -4.43115 0.00327532 0.0844429 0 0.000276578 0.00327532 0.471088 0.00198299 -4.43604 0.00326541 0.0849855 0 0.000277513 0.00326541 0.474833 0.00198746 -4.44092 0.00325552 0.085528 0 0.000278438 0.00325552 0.47859 0.00199188 -4.4458 0.00324565 0.0860705 0 0.000279355 0.00324565 0.482358 0.00199625 -4.45068 0.00323579 0.0866131 0 0.000280261 0.00323579 0.486137 0.00200056 -4.45557 0.00322594 0.0871556 0 0.000281159 0.00322594 0.489929 0.00200483 -4.46045 0.00321611 0.0876981 0 0.000282047 0.00321611 0.493731 0.00200905 -4.46533 0.00320629 0.0882407 0 0.000282925 0.00320629 0.497545 0.00201322 -4.47021 0.00319649 0.0887832 0 0.000283795 0.00319649 0.501371 0.00201734 -4.4751 0.00318671 0.0893257 0 0.000284655 0.00318671 0.505209 0.00202142 -4.47998 0.00317694 0.0898683 0 0.000285506 0.00317694 0.509059 0.00202544 -4.48486 0.00316718 0.0904108 0 0.000286347 0.00316718 0.51292 0.00202942 -4.48975 0.00315744 0.0909534 0 0.00028718 0.00315744 0.516793 0.00203335 -4.49463 0.00314771 0.0914959 0 0.000288003 0.00314771 0.520678 0.00203724 -4.49951 0.003138 0.0920384 0 0.000288817 0.003138 0.524576 0.00204107 -4.50439 0.00312831 0.092581 0 0.000289622 0.00312831 0.528485 0.00204486 -4.50928 0.00311863 0.0931235 0 0.000290417 0.00311863 0.532406 0.00204861 -4.51416 0.00310896 0.093666 0 0.000291204 0.00310896 0.53634 0.0020523 -4.51904 0.00309931 0.0942086 0 0.000291981 0.00309931 0.540286 0.00205596 -4.52393 0.00308967 0.0947511 0 0.00029275 0.00308967 0.544244 0.00205956 -4.52881 0.00308005 0.0952936 0 0.000293509 0.00308005 0.548214 0.00206312 -4.53369 0.00307045 0.0958362 0 0.00029426 0.00307045 0.552197 0.00206664 -4.53857 0.00306086 0.0963787 0 0.000295001 0.00306086 0.556192 0.00207011 -4.54346 0.00305128 0.0969212 0 0.000295734 0.00305128 0.5602 0.00207353 -4.54834 0.00304172 0.0974638 0 0.000296457 0.00304172 0.564221 0.00207691 -4.55322 0.00303217 0.0980063 0 0.000297172 0.00303217 0.568254 0.00208025 -4.55811 0.00302264 0.0985488 0 0.000297878 0.00302264 0.5723 0.00208354 -4.56299 0.00301313 0.0990914 0 0.000298575 0.00301313 0.576358 0.00208679 -4.56787 0.00300363 0.0996339 0 0.000299263 0.00300363 0.58043 0.00209 -4.57275 0.00299414 0.100176 0 0.000299942 0.00299414 0.584514 0.00209316 -4.57764 0.00298467 0.100719 0 0.000300613 0.00298467 0.588611 0.00209628 -4.58252 0.00297521 0.101262 0 0.000301275 0.00297521 0.592722 0.00209935 -4.5874 0.00296577 0.101804 0 0.000301928 0.00296577 0.596845 0.00210239 -4.59229 0.00295635 0.102347 0 0.000302572 0.00295635 0.600981 0.00210538 -4.59717 0.00294694 0.102889 0 0.000303208 0.00294694 0.605131 0.00210832 -4.60205 0.00293754 0.103432 0 0.000303835 0.00293754 0.609294 0.00211123 -4.60693 0.00292816 0.103974 0 0.000304453 0.00292816 0.61347 0.00211409 -4.61182 0.00291879 0.104517 0 0.000305063 0.00291879 0.61766 0.00211691 -4.6167 0.00290944 0.105059 0 0.000305664 0.00290944 0.621863 0.00211969 -4.62158 0.00290011 0.105602 0 0.000306257 0.00290011 0.62608 0.00212243 -4.62646 0.00289079 0.106144 0 0.000306841 0.00289079 0.63031 0.00212513 -4.63135 0.00288148 0.106687 0 0.000307416 0.00288148 0.634554 0.00212779 -4.63623 0.00287219 0.107229 0 0.000307983 0.00287219 0.638811 0.0021304 -4.64111 0.00286291 0.107772 0 0.000308542 0.00286291 0.643083 0.00213298 -4.646 0.00285365 0.108314 0 0.000309092 0.00285365 0.647368 0.00213551 -4.65088 0.00284441 0.108857 0 0.000309634 0.00284441 0.651667 0.00213801 -4.65576 0.00283518 0.1094 0 0.000310167 0.00283518 0.65598 0.00214046 -4.66064 0.00282596 0.109942 0 0.000310692 0.00282596 0.660308 0.00214288 -4.66553 0.00281676 0.110485 0 0.000311209 0.00281676 0.664649 0.00214525 -4.67041 0.00280758 0.111027 0 0.000311717 0.00280758 0.669004 0.00214759 -4.67529 0.0027984 0.11157 0 0.000312217 0.0027984 0.673374 0.00214988 -4.68018 0.00278925 0.112112 0 0.000312709 0.00278925 0.677758 0.00215214 -4.68506 0.00278011 0.112655 0 0.000313192 0.00278011 0.682157 0.00215436 -4.68994 0.00277098 0.113197 0 0.000313668 0.00277098 0.68657 0.00215654 -4.69482 0.00276187 0.11374 0 0.000314135 0.00276187 0.690997 0.00215868 -4.69971 0.00275278 0.114282 0 0.000314594 0.00275278 0.695439 0.00216078 -4.70459 0.0027437 0.114825 0 0.000315045 0.0027437 0.699896 0.00216284 -4.70947 0.00273463 0.115367 0 0.000315487 0.00273463 0.704368 0.00216487 -4.71436 0.00272558 0.11591 0 0.000315922 0.00272558 0.708854 0.00216686 -4.71924 0.00271654 0.116452 0 0.000316348 0.00271654 0.713355 0.00216881 -4.72412 0.00270752 0.116995 0 0.000316767 0.00270752 0.717872 0.00217072 -4.729 0.00269852 0.117538 0 0.000317177 0.00269852 0.722403 0.00217259 -4.73389 0.00268953 0.11808 0 0.000317579 0.00268953 0.72695 0.00217443 -4.73877 0.00268055 0.118623 0 0.000317974 0.00268055 0.731511 0.00217623 -4.74365 0.00267159 0.119165 0 0.00031836 0.00267159 0.736088 0.00217799 -4.74854 0.00266264 0.119708 0 0.000318739 0.00266264 0.740681 0.00217972 -4.75342 0.00265371 0.12025 0 0.00031911 0.00265371 0.745288 0.00218141 -4.7583 0.0026448 0.120793 0 0.000319472 0.0026448 0.749912 0.00218306 -4.76318 0.0026359 0.121335 0 0.000319827 0.0026359 0.754551 0.00218468 -4.76807 0.00262701 0.121878 0 0.000320174 0.00262701 0.759205 0.00218626 -4.77295 0.00261814 0.12242 0 0.000320514 0.00261814 0.763876 0.0021878 -4.77783 0.00260929 0.122963 0 0.000320845 0.00260929 0.768562 0.00218931 -4.78271 0.00260044 0.123505 0 0.000321169 0.00260044 0.773264 0.00219078 -4.7876 0.00259162 0.124048 0 0.000321485 0.00259162 0.777982 0.00219222 -4.79248 0.00258281 0.124591 0 0.000321793 0.00258281 0.782716 0.00219362 -4.79736 0.00257401 0.125133 0 0.000322094 0.00257401 0.787467 0.00219499 -4.80225 0.00256523 0.125676 0 0.000322387 0.00256523 0.792233 0.00219632 -4.80713 0.00255647 0.126218 0 0.000322672 0.00255647 0.797016 0.00219761 -4.81201 0.00254772 0.126761 0 0.00032295 0.00254772 0.801816 0.00219888 -4.81689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.82178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.82666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.92432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01221 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02197 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02686 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03174 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03662 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04639 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05615 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06592 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.07568 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08057 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08545 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09033 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10986 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11475 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11963 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12451 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13428 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13916 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14404 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14893 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15381 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15869 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16357 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16846 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17334 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17822 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18799 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19287 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19775 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20264 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20752 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22217 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22705 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23193 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23682 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24658 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25635 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26611 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27588 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28076 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28564 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29053 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30029 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30518 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31006 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31494 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31982 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32471 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33447 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33936 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34424 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34912 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35889 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36865 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37842 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38818 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39307 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39795 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40283 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41748 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42236 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42725 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43213 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43701 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44678 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45166 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45654 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46143 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46631 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47119 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47607 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48096 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49072 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50049 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50537 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51025 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51514 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53467 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53955 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54443 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54932 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55908 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56885 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57861 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58838 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59326 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59814 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60303 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61279 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61768 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62256 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62744 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63232 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63721 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64209 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64697 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65674 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66162 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67139 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67627 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68115 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69092 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70068 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70557 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71045 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71533 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72998 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73486 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73975 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74463 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74951 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75928 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76904 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77393 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77881 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78369 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78857 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79346 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80322 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80811 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81299 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81787 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82275 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82764 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84717 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85205 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85693 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86182 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87158 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88135 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88623 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89111 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90088 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90576 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91553 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92041 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92529 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93018 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93506 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93994 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94482 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94971 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95459 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95947 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96924 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97412 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98389 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99365 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00342 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01318 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01807 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02295 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02783 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04736 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05225 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05713 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06201 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.17432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.26221 0.001 0 0 0 0.001 0 0 -6.26709 0.001 0 0 0 0.001 0 0 -6.27197 0.001 0 0 0 0.001 0 0 -6.27686 0.001 0 0 0 0.001 0 0 -6.28174 0.001 0 0 0 0.001 0 0 -6.28662 0.001 0 0 0 0.001 0 0 -6.2915 0.001 0 0 0 0.001 0 0 -6.29639 0.001 0 0 0 0.001 0 0 -6.30127 0.001 0 0 0 0.001 0 0 -6.30615 0.001 0 0 0 0.001 0 0 -6.31104 0.001 0 0 0 0.001 0 0 -6.31592 0.001 0 0 0 0.001 0 0 -6.3208 0.001 0 0 0 0.001 0 0 -6.32568 0.001 0 0 0 0.001 0 0 -6.33057 0.001 0 0 0 0.001 0 0 -6.33545 0.001 0 0 0 0.001 0 0 -6.34033 0.001 0 0 0 0.001 0 0 -6.34521 0.001 0 0 0 0.001 0 0 -6.3501 0.001 0 0 0 0.001 0 0 -6.35498 0.001 0 0 0 0.001 0 0 -6.35986 0.001 0 0 0 0.001 0 0 -6.36475 0.001 0 0 0 0.001 0 0 -6.36963 0.001 0 0 0 0.001 0 0 -6.37451 0.001 0 0 0 0.001 0 0 -6.37939 0.001 0 0 0 0.001 0 0 -6.38428 0.001 0 0 0 0.001 0 0 -6.38916 0.001 0 0 0 0.001 0 0 -6.39404 0.001 0 0 0 0.001 0 0 -6.39893 0.001 0 0 0 0.001 0 0 -6.40381 0.001 0 0 0 0.001 0 0 -6.40869 0.001 0 0 0 0.001 0 0 -6.41357 0.001 0 0 0 0.001 0 0 -6.41846 0.001 0 0 0 0.001 0 0 -6.42334 0.001 0 0 0 0.001 0 0 -6.42822 0.001 0 0 0 0.001 0 0 -6.43311 0.001 0 0 0 0.001 0 0 -6.43799 0.001 0 0 0 0.001 0 0 -6.44287 0.001 0 0 0 0.001 0 0 -6.44775 0.001 0 0 0 0.001 0 0 -6.45264 0.001 0 0 0 0.001 0 0 -6.45752 0.001 0 0 0 0.001 0 0 -6.4624 0.001 0 0 0 0.001 0 0 -6.46729 0.001 0 0 0 0.001 0 0 -6.47217 0.001 0 0 0 0.001 0 0 -6.47705 0.001 0 0 0 0.001 0 0 -6.48193 0.001 0 0 0 0.001 0 0 -6.48682 0.001 0 0 0 0.001 0 0 -6.4917 0.001 0 0 0 0.001 0 0 -6.49658 0.001 0 0 0 0.001 0 0 -6.50146 0.001 0 0 0 0.001 0 0 -6.50635 0.001 0 0 0 0.001 0 0 -6.51123 0.001 0 0 0 0.001 0 0 -6.51611 0.001 0 0 0 0.001 0 0 -6.521 0.001 0 0 0 0.001 0 0 -6.52588 0.001 0 0 0 0.001 0 0 -6.53076 0.001 0 0 0 0.001 0 0 -6.53564 0.001 0 0 0 0.001 0 0 -6.54053 0.001 0 0 0 0.001 0 0 -6.54541 0.001 0 0 0 0.001 0 0 -6.55029 0.001 0 0 0 0.001 0 0 -6.55518 0.001 0 0 0 0.001 0 0 -6.56006 0.001 0 0 0 0.001 0 0 -6.56494 0.001 0 0 0 0.001 0 0 -6.56982 0.001 0 0 0 0.001 0 0 -6.57471 0.001 0 0 0 0.001 0 0 -6.57959 0.001 0 0 0 0.001 0 0 -6.58447 0.001 0 0 0 0.001 0 0 -6.58936 0.001 0 0 0 0.001 0 0 -6.59424 0.001 0 0 0 0.001 0 0 -6.59912 0.001 0 0 0 0.001 0 0 -6.604 0.001 0 0 0 0.001 0 0 -6.60889 0.001 0 0 0 0.001 0 0 -6.61377 0.001 0 0 0 0.001 0 0 -6.61865 0.001 0 0 0 0.001 0 0 -6.62354 0.001 0 0 0 0.001 0 0 -6.62842 0.001 0 0 0 0.001 0 0 -6.6333 0.001 0 0 0 0.001 0 0 -6.63818 0.001 0 0 0 0.001 0 0 -6.64307 0.001 0 0 0 0.001 0 0 -6.64795 0.001 0 0 0 0.001 0 0 -6.65283 0.001 0 0 0 0.001 0 0 -6.65771 0.001 0 0 0 0.001 0 0 -6.6626 0.001 0 0 0 0.001 0 0 -6.66748 0.001 0 0 0 0.001 0 0 -6.67236 0.001 0 0 0 0.001 0 0 -6.67725 0.001 0 0 0 0.001 0 0 -6.68213 0.001 0 0 0 0.001 0 0 -6.68701 0.001 0 0 0 0.001 0 0 -6.69189 0.001 0 0 0 0.001 0 0 -6.69678 0.001 0 0 0 0.001 0 0 -6.70166 0.001 0 0 0 0.001 0 0 -6.70654 0.001 0 0 0 0.001 0 0 -6.71143 0.001 0 0 0 0.001 0 0 -6.71631 0.001 0 0 0 0.001 0 0 -6.72119 0.001 0 0 0 0.001 0 0 -6.72607 0.001 0 0 0 0.001 0 0 -6.73096 0.001 0 0 0 0.001 0 0 -6.73584 0.001 0 0 0 0.001 0 0 -6.74072 0.001 0 0 0 0.001 0 0 -6.74561 0.001 0 0 0 0.001 0 0 -6.75049 0.001 0 0 0 0.001 0 0 -6.75537 0.001 0 0 0 0.001 0 0 -6.76025 0.001 0 0 0 0.001 0 0 -6.76514 0.001 0 0 0 0.001 0 0 -6.77002 0.001 0 0 0 0.001 0 0 -6.7749 0.001 0 0 0 0.001 0 0 -6.77979 0.001 0 0 0 0.001 0 0 -6.78467 0.001 0 0 0 0.001 0 0 -6.78955 0.001 0 0 0 0.001 0 0 -6.79443 0.001 0 0 0 0.001 0 0 -6.79932 0.001 0 0 0 0.001 0 0 -6.8042 0.001 0 0 0 0.001 0 0 -6.80908 0.001 0 0 0 0.001 0 0 -6.81396 0.001 0 0 0 0.001 0 0 -6.81885 0.001 0 0 0 0.001 0 0 -6.82373 0.001 0 0 0 0.001 0 0 -6.82861 0.001 0 0 0 0.001 0 0 -6.8335 0.001 0 0 0 0.001 0 0 -6.83838 0.001 0 0 0 0.001 0 0 -6.84326 0.001 0 0 0 0.001 0 0 -6.84814 0.001 0 0 0 0.001 0 0 -6.85303 0.001 0 0 0 0.001 0 0 -6.85791 0.001 0 0 0 0.001 0 0 -6.86279 0.001 0 0 0 0.001 0 0 -6.86768 0.001 0 0 0 0.001 0 0 -6.87256 0.001 0 0 0 0.001 0 0 -6.87744 0.001 0 0 0 0.001 0 0 -6.88232 0.001 0 0 0 0.001 0 0 -6.88721 0.001 0 0 0 0.001 0 0 -6.89209 0.001 0 0 0 0.001 0 0 -6.89697 0.001 0 0 0 0.001 0 0 -6.90186 0.001 0 0 0 0.001 0 0 -6.90674 0.001 0 0 0 0.001 0 0 -6.91162 0.001 0 0 0 0.001 0 0 -6.9165 0.001 0 0 0 0.001 0 0 -6.92139 0.001 0 0 0 0.001 0 0 -6.92627 0.001 0 0 0 0.001 0 0 -6.93115 0.001 0 0 0 0.001 0 0 -6.93604 0.001 0 0 0 0.001 0 0 -6.94092 0.001 0 0 0 0.001 0 0 -6.9458 0.001 0 0 0 0.001 0 0 -6.95068 0.001 0 0 0 0.001 0 0 -6.95557 0.001 0 0 0 0.001 0 0 -6.96045 0.001 0 0 0 0.001 0 0 -6.96533 0.001 0 0 0 0.001 0 0 -6.97021 0.001 0 0 0 0.001 0 0 -6.9751 0.001 0 0 0 0.001 0 0 -6.97998 0.001 0 0 0 0.001 0 0 -6.98486 0.001 0 0 0 0.001 0 0 -6.98975 0.001 0 0 0 0.001 0 0 -6.99463 0.001 0 0 0 0.001 0 0 -6.99951 0.001 0 0 0 0.001 0 0 -7.00439 0.001 0 0 0 0.001 0 0 -7.00928 0.001 0 0 0 0.001 0 0 -7.01416 0.001 0 0 0 0.001 0 0 -7.01904 0.001 0 0 0 0.001 0 0 -7.02393 0.001 0 0 0 0.001 0 0 -7.02881 0.001 0 0 0 0.001 0 0 -7.03369 0.001 0 0 0 0.001 0 0 -7.03857 0.001 0 0 0 0.001 0 0 -7.04346 0.001 0 0 0 0.001 0 0 -7.04834 0.001 0 0 0 0.001 0 0 -7.05322 0.001 0 0 0 0.001 0 0 -7.05811 0.001 0 0 0 0.001 0 0 -7.06299 0.001 0 0 0 0.001 0 0 -7.06787 0.001 0 0 0 0.001 0 0 -7.07275 0.001 0 0 0 0.001 0 0 -7.07764 0.001 0 0 0 0.001 0 0 -7.08252 0.001 0 0 0 0.001 0 0 -7.0874 0.001 0 0 0 0.001 0 0 -7.09229 0.001 0 0 0 0.001 0 0 -7.09717 0.001 0 0 0 0.001 0 0 -7.10205 0.001 0 0 0 0.001 0 0 -7.10693 0.001 0 0 0 0.001 0 0 -7.11182 0.001 0 0 0 0.001 0 0 -7.1167 0.001 0 0 0 0.001 0 0 -7.12158 0.001 0 0 0 0.001 0 0 -7.12646 0.001 0 0 0 0.001 0 0 -7.13135 0.001 0 0 0 0.001 0 0 -7.13623 0.001 0 0 0 0.001 0 0 -7.14111 0.001 0 0 0 0.001 0 0 -7.146 0.001 0 0 0 0.001 0 0 -7.15088 0.001 0 0 0 0.001 0 0 -7.15576 0.001 0 0 0 0.001 0 0 -7.16064 0.001 0 0 0 0.001 0 0 -7.16553 0.001 0 0 0 0.001 0 0 -7.17041 0.001 0 0 0 0.001 0 0 -7.17529 0.001 0 0 0 0.001 0 0 -7.18018 0.001 0 0 0 0.001 0 0 -7.18506 0.001 0 0 0 0.001 0 0 -7.18994 0.001 0 0 0 0.001 0 0 -7.19482 0.001 0 0 0 0.001 0 0 -7.19971 0.001 0 0 0 0.001 0 0 -7.20459 0.001 0 0 0 0.001 0 0 -7.20947 0.001 0 0 0 0.001 0 0 -7.21436 0.001 0 0 0 0.001 0 0 -7.21924 0.001 0 0 0 0.001 0 0 -7.22412 0.001 0 0 0 0.001 0 0 -7.229 0.001 0 0 0 0.001 0 0 -7.23389 0.001 0 0 0 0.001 0 0 -7.23877 0.001 0 0 0 0.001 0 0 -7.24365 0.001 0 0 0 0.001 0 0 -7.24854 0.001 0 0 0 0.001 0 0 -7.25342 0.001 0 0 0 0.001 0 0 -7.2583 0.001 0 0 0 0.001 0 0 -7.26318 0.001 0 0 0 0.001 0 0 -7.26807 0.001 0 0 0 0.001 0 0 -7.27295 0.001 0 0 0 0.001 0 0 -7.27783 0.001 0 0 0 0.001 0 0 -7.28271 0.001 0 0 0 0.001 0 0 -7.2876 0.001 0 0 0 0.001 0 0 -7.29248 0.001 0 0 0 0.001 0 0 -7.29736 0.001 0 0 0 0.001 0 0 -7.30225 0.001 0 0 0 0.001 0 0 -7.30713 0.001 0 0 0 0.001 0 0 -7.31201 0.001 0 0 0 0.001 0 0 -7.31689 0.001 0 0 0 0.001 0 0 -7.32178 0.001 0 0 0 0.001 0 0 -7.32666 0.001 0 0 0 0.001 0 0 -7.33154 0.001 0 0 0 0.001 0 0 -7.33643 0.001 0 0 0 0.001 0 0 -7.34131 0.001 0 0 0 0.001 0 0 -7.34619 0.001 0 0 0 0.001 0 0 -7.35107 0.001 0 0 0 0.001 0 0 -7.35596 0.001 0 0 0 0.001 0 0 -7.36084 0.001 0 0 0 0.001 0 0 -7.36572 0.001 0 0 0 0.001 0 0 -7.37061 0.001 0 0 0 0.001 0 0 -7.37549 0.001 0 0 0 0.001 0 0 -7.38037 0.001 0 0 0 0.001 0 0 -7.38525 0.001 0 0 0 0.001 0 0 -7.39014 0.001 0 0 0 0.001 0 0 -7.39502 0.001 0 0 0 0.001 0 0 -7.3999 0.001 0 0 0 0.001 0 0 -7.40479 0.001 0 0 0 0.001 0 0 -7.40967 0.001 0 0 0 0.001 0 0 -7.41455 0.001 0 0 0 0.001 0 0 -7.41943 0.001 0 0 0 0.001 0 0 -7.42432 0.001 0 0 0 0.001 0 0 -7.4292 0.001 0 0 0 0.001 0 0 -7.43408 0.001 0 0 0 0.001 0 0 -7.43896 0.001 0 0 0 0.001 0 0 -7.44385 0.001 0 0 0 0.001 0 0 -7.44873 0.001 0 0 0 0.001 0 0 -7.45361 0.001 0 0 0 0.001 0 0 -7.4585 0.001 0 0 0 0.001 0 0 -7.46338 0.001 0 0 0 0.001 0 0 -7.46826 0.001 0 0 0 0.001 0 0 -7.47314 0.001 0 0 0 0.001 0 0 -7.47803 0.001 0 0 0 0.001 0 0 -7.48291 0.001 0 0 0 0.001 0 0 -7.48779 0.001 0 0 0 0.001 0 0 -7.49268 0.001 0 0 0 0.001 0 0 -7.49756 0.001 0 0 0 0.001 0 0 -7.50244 0.001 0 0 0 0.001 0 0 -7.50732 0.001 0 0 0 0.001 0 0 -7.51221 0.001 0 0 0 0.001 0 0 -7.51709 0.001 0 0 0 0.001 0 0 -7.52197 0.001 0 0 0 0.001 0 0 -7.52686 0.001 0 0 0 0.001 0 0 -7.53174 0.001 0 0 0 0.001 0 0 -7.53662 0.001 0 0 0 0.001 0 0 -7.5415 0.001 0 0 0 0.001 0 0 -7.54639 0.001 0 0 0 0.001 0 0 -7.55127 0.001 0 0 0 0.001 0 0 -7.55615 0.001 0 0 0 0.001 0 0 -7.56104 0.001 0 0 0 0.001 0 0 -7.56592 0.001 0 0 0 0.001 0 0 -7.5708 0.001 0 0 0 0.001 0 0 -7.57568 0.001 0 0 0 0.001 0 0 -7.58057 0.001 0 0 0 0.001 0 0 -7.58545 0.001 0 0 0 0.001 0 0 -7.59033 0.001 0 0 0 0.001 0 0 -7.59521 0.001 0 0 0 0.001 0 0 -7.6001 0.001 0 0 0 0.001 0 0 -7.60498 0.001 0 0 0 0.001 0 0 -7.60986 0.001 0 0 0 0.001 0 0 -7.61475 0.001 0 0 0 0.001 0 0 -7.61963 0.001 0 0 0 0.001 0 0 -7.62451 0.001 0 0 0 0.001 0 0 -7.62939 0.001 0 0 0 0.001 0 0 -7.63428 0.001 0 0 0 0.001 0 0 -7.63916 0.001 0 0 0 0.001 0 0 -7.64404 0.001 0 0 0 0.001 0 0 -7.64893 0.001 0 0 0 0.001 0 0 -7.65381 0.001 0 0 0 0.001 0 0 -7.65869 0.001 0 0 0 0.001 0 0 -7.66357 0.001 0 0 0 0.001 0 0 -7.66846 0.001 0 0 0 0.001 0 0 -7.67334 0.001 0 0 0 0.001 0 0 -7.67822 0.001 0 0 0 0.001 0 0 -7.68311 0.001 0 0 0 0.001 0 0 -7.68799 0.001 0 0 0 0.001 0 0 -7.69287 0.001 0 0 0 0.001 0 0 -7.69775 0.001 0 0 0 0.001 0 0 -7.70264 0.001 0 0 0 0.001 0 0 -7.70752 0.001 0 0 0 0.001 0 0 -7.7124 0.001 0 0 0 0.001 0 0 -7.71729 0.001 0 0 0 0.001 0 0 -7.72217 0.001 0 0 0 0.001 0 0 -7.72705 0.001 0 0 0 0.001 0 0 -7.73193 0.001 0 0 0 0.001 0 0 -7.73682 0.001 0 0 0 0.001 0 0 -7.7417 0.001 0 0 0 0.001 0 0 -7.74658 0.001 0 0 0 0.001 0 0 -7.75146 0.001 0 0 0 0.001 0 0 -7.75635 0.001 0 0 0 0.001 0 0 -7.76123 0.001 0 0 0 0.001 0 0 -7.76611 0.001 0 0 0 0.001 0 0 -7.771 0.001 0 0 0 0.001 0 0 -7.77588 0.001 0 0 0 0.001 0 0 -7.78076 0.001 0 0 0 0.001 0 0 -7.78564 0.001 0 0 0 0.001 0 0 -7.79053 0.001 0 0 0 0.001 0 0 -7.79541 0.001 0 0 0 0.001 0 0 -7.80029 0.001 0 0 0 0.001 0 0 -7.80518 0.001 0 0 0 0.001 0 0 -7.81006 0.001 0 0 0 0.001 0 0 -7.81494 0.001 0 0 0 0.001 0 0 -7.81982 0.001 0 0 0 0.001 0 0 -7.82471 0.001 0 0 0 0.001 0 0 -7.82959 0.001 0 0 0 0.001 0 0 -7.83447 0.001 0 0 0 0.001 0 0 -7.83936 0.001 0 0 0 0.001 0 0 -7.84424 0.001 0 0 0 0.001 0 0 -7.84912 0.001 0 0 0 0.001 0 0 -7.854 0.001 0 0 0 0.001 0 0 -7.85889 0.001 0 0 0 0.001 0 0 -7.86377 0.001 0 0 0 0.001 0 0 -7.86865 0.001 0 0 0 0.001 0 0 -7.87354 0.001 0 0 0 0.001 0 0 -7.87842 0.001 0 0 0 0.001 0 0 -7.8833 0.001 0 0 0 0.001 0 0 -7.88818 0.001 0 0 0 0.001 0 0 -7.89307 0.001 0 0 0 0.001 0 0 -7.89795 0.001 0 0 0 0.001 0 0 -7.90283 0.001 0 0 0 0.001 0 0 -7.90771 0.001 0 0 0 0.001 0 0 -7.9126 0.001 0 0 0 0.001 0 0 -7.91748 0.001 0 0 0 0.001 0 0 -7.92236 0.001 0 0 0 0.001 0 0 -7.92725 0.001 0 0 0 0.001 0 0 -7.93213 0.001 0 0 0 0.001 0 0 -7.93701 0.001 0 0 0 0.001 0 0 -7.94189 0.001 0 0 0 0.001 0 0 -7.94678 0.001 0 0 0 0.001 0 0 -7.95166 0.001 0 0 0 0.001 0 0 -7.95654 0.001 0 0 0 0.001 0 0 -7.96143 0.001 0 0 0 0.001 0 0 -7.96631 0.001 0 0 0 0.001 0 0 -7.97119 0.001 0 0 0 0.001 0 0 -7.97607 0.001 0 0 0 0.001 0 0 -7.98096 0.001 0 0 0 0.001 0 0 -7.98584 0.001 0 0 0 0.001 0 0 -7.99072 0.001 0 0 0 0.001 0 0 -7.99561 0.001 0 0 0 0.001 0 0 -8.00049 0.001 0 0 0 0.001 0 0 -8.00537 0.001 0 0 0 0.001 0 0 -8.01025 0.001 0 0 0 0.001 0 0 -8.01514 0.001 0 0 0 0.001 0 0 -8.02002 0.001 0 0 0 0.001 0 0 -8.0249 0.001 0 0 0 0.001 0 0 -8.02979 0.001 0 0 0 0.001 0 0 -8.03467 0.001 0 0 0 0.001 0 0 -8.03955 0.001 0 0 0 0.001 0 0 -8.04443 0.001 0 0 0 0.001 0 0 -8.04932 0.001 0 0 0 0.001 0 0 -8.0542 0.001 0 0 0 0.001 0 0 -8.05908 0.001 0 0 0 0.001 0 0 -8.06396 0.001 0 0 0 0.001 0 0 -8.06885 0.001 0 0 0 0.001 0 0 -8.07373 0.001 0 0 0 0.001 0 0 -8.07861 0.001 0 0 0 0.001 0 0 -8.0835 0.001 0 0 0 0.001 0 0 -8.08838 0.001 0 0 0 0.001 0 0 -8.09326 0.001 0 0 0 0.001 0 0 -8.09814 0.001 0 0 0 0.001 0 0 -8.10303 0.001 0 0 0 0.001 0 0 -8.10791 0.001 0 0 0 0.001 0 0 -8.11279 0.001 0 0 0 0.001 0 0 -8.11768 0.001 0 0 0 0.001 0 0 -8.12256 0.001 0 0 0 0.001 0 0 -8.12744 0.001 0 0 0 0.001 0 0 -8.13232 0.001 0 0 0 0.001 0 0 -8.13721 0.001 0 0 0 0.001 0 0 -8.14209 0.001 0 0 0 0.001 0 0 -8.14697 0.001 0 0 0 0.001 0 0 -8.15186 0.001 0 0 0 0.001 0 0 -8.15674 0.001 0 0 0 0.001 0 0 -8.16162 0.001 0 0 0 0.001 0 0 -8.1665 0.001 0 0 0 0.001 0 0 -8.17139 0.001 0 0 0 0.001 0 0 -8.17627 0.001 0 0 0 0.001 0 0 -8.18115 0.001 0 0 0 0.001 0 0 -8.18604 0.001 0 0 0 0.001 0 0 -8.19092 0.001 0 0 0 0.001 0 0 -8.1958 0.001 0 0 0 0.001 0 0 -8.20068 0.001 0 0 0 0.001 0 0 -8.20557 0.001 0 0 0 0.001 0 0 -8.21045 0.001 0 0 0 0.001 0 0 -8.21533 0.001 0 0 0 0.001 0 0 -8.22021 0.001 0 0 0 0.001 0 0 -8.2251 0.001 0 0 0 0.001 0 0 -8.22998 0.001 0 0 0 0.001 0 0 -8.23486 0.001 0 0 0 0.001 0 0 -8.23975 0.001 0 0 0 0.001 0 0 -8.24463 0.001 0 0 0 0.001 0 0 -8.24951 0.001 0 0 0 0.001 0 0 -8.25439 0.001 0 0 0 0.001 0 0 -8.25928 0.001 0 0 0 0.001 0 0 -8.26416 0.001 0 0 0 0.001 0 0 -8.26904 0.001 0 0 0 0.001 0 0 -8.27393 0.001 0 0 0 0.001 0 0 -8.27881 0.001 0 0 0 0.001 0 0 -8.28369 0.001 0 0 0 0.001 0 0 -8.28857 0.001 0 0 0 0.001 0 0 -8.29346 0.001 0 0 0 0.001 0 0 -8.29834 0.001 0 0 0 0.001 0 0 -8.30322 0.001 0 0 0 0.001 0 0 -8.30811 0.001 0 0 0 0.001 0 0 -8.31299 0.001 0 0 0 0.001 0 0 -8.31787 0.001 0 0 0 0.001 0 0 -8.32275 0.001 0 0 0 0.001 0 0 -8.32764 0.001 0 0 0 0.001 0 0 -8.33252 0.001 0 0 0 0.001 0 0 -8.3374 0.001 0 0 0 0.001 0 0 -8.34229 0.001 0 0 0 0.001 0 0 -8.34717 0.001 0 0 0 0.001 0 0 -8.35205 0.001 0 0 0 0.001 0 0 -8.35693 0.001 0 0 0 0.001 0 0 -8.36182 0.001 0 0 0 0.001 0 0 -8.3667 0.001 0 0 0 0.001 0 0 -8.37158 0.001 0 0 0 0.001 0 0 -8.37646 0.001 0 0 0 0.001 0 0 -8.38135 0.001 0 0 0 0.001 0 0 -8.38623 0.001 0 0 0 0.001 0 0 -8.39111 0.001 0 0 0 0.001 0 0 -8.396 0.001 0 0 0 0.001 0 0 -8.40088 0.001 0 0 0 0.001 0 0 -8.40576 0.001 0 0 0 0.001 0 0 -8.41064 0.001 0 0 0 0.001 0 0 -8.41553 0.001 0 0 0 0.001 0 0 -8.42041 0.001 0 0 0 0.001 0 0 -8.42529 0.001 0 0 0 0.001 0 0 -8.43018 0.001 0 0 0 0.001 0 0 -8.43506 0.001 0 0 0 0.001 0 0 -8.43994 0.001 0 0 0 0.001 0 0 -8.44482 0.001 0 0 0 0.001 0 0 -8.44971 0.001 0 0 0 0.001 0 0 -8.45459 0.001 0 0 0 0.001 0 0 -8.45947 0.001 0 0 0 0.001 0 0 -8.46436 0.001 0 0 0 0.001 0 0 -8.46924 0.001 0 0 0 0.001 0 0 -8.47412 0.001 0 0 0 0.001 0 0 -8.479 0.001 0 0 0 0.001 0 0 -8.48389 0.001 0 0 0 0.001 0 0 -8.48877 0.001 0 0 0 0.001 0 0 -8.49365 0.001 0 0 0 0.001 0 0 -8.49854 0.001 0 0 0 0.001 0 0 -8.50342 0.001 0 0 0 0.001 0 0 -8.5083 0.001 0 0 0 0.001 0 0 -8.51318 0.001 0 0 0 0.001 0 0 -8.51807 0.001 0 0 0 0.001 0 0 -8.52295 0.001 0 0 0 0.001 0 0 -8.52783 0.001 0 0 0 0.001 0 0 -8.53271 0.001 0 0 0 0.001 0 0 -8.5376 0.001 0 0 0 0.001 0 0 -8.54248 0.001 0 0 0 0.001 0 0 -8.54736 0.001 0 0 0 0.001 0 0 -8.55225 0.001 0 0 0 0.001 0 0 -8.55713 0.001 0 0 0 0.001 0 0 -8.56201 0.001 0 0 0 0.001 0 0 -8.56689 0.001 0 0 0 0.001 0 0 -8.57178 0.001 0 0 0 0.001 0 0 -8.57666 0.001 0 0 0 0.001 0 0 -8.58154 0.001 0 0 0 0.001 0 0 -8.58643 0.001 0 0 0 0.001 0 0 -8.59131 0.001 0 0 0 0.001 0 0 -8.59619 0.001 0 0 0 0.001 0 0 -8.60107 0.001 0 0 0 0.001 0 0 -8.60596 0.001 0 0 0 0.001 0 0 -8.61084 0.001 0 0 0 0.001 0 0 -8.61572 0.001 0 0 0 0.001 0 0 -8.62061 0.001 0 0 0 0.001 0 0 -8.62549 0.001 0 0 0 0.001 0 0 -8.63037 0.001 0 0 0 0.001 0 0 -8.63525 0.001 0 0 0 0.001 0 0 -8.64014 0.001 0 0 0 0.001 0 0 -8.64502 0.001 0 0 0 0.001 0 0 -8.6499 0.001 0 0 0 0.001 0 0 -8.65479 0.001 0 0 0 0.001 0 0 -8.65967 0.001 0 0 0 0.001 0 0 -8.66455 0.001 0 0 0 0.001 0 0 -8.66943 0.001 0 0 0 0.001 0 0 -8.67432 0.001 0 0 0 0.001 0 0 -8.6792 0.001 0 0 0 0.001 0 0 -8.68408 0.001 0 0 0 0.001 0 0 -8.68896 0.001 0 0 0 0.001 0 0 -8.69385 0.001 0 0 0 0.001 0 0 -8.69873 0.001 0 0 0 0.001 0 0 -8.70361 0.001 0 0 0 0.001 0 0 -8.7085 0.001 0 0 0 0.001 0 0 -8.71338 0.001 0 0 0 0.001 0 0 -8.71826 0.001 0 0 0 0.001 0 0 -8.72314 0.001 0 0 0 0.001 0 0 -8.72803 0.001 0 0 0 0.001 0 0 -8.73291 0.001 0 0 0 0.001 0 0 -8.73779 0.001 0 0 0 0.001 0 0 -8.74268 0.001 0 0 0 0.001 0 0 -8.74756 0.001 0 0 0 0.001 0 0 -8.75244 0.001 0 0 0 0.001 0 0 -8.75732 0.001 0 0 0 0.001 0 0 -8.76221 0.001 0 0 0 0.001 0 0 -8.76709 0.001 0 0 0 0.001 0 0 -8.77197 0.001 0 0 0 0.001 0 0 -8.77686 0.001 0 0 0 0.001 0 0 -8.78174 0.001 0 0 0 0.001 0 0 -8.78662 0.001 0 0 0 0.001 0 0 -8.7915 0.001 0 0 0 0.001 0 0 -8.79639 0.001 0 0 0 0.001 0 0 -8.80127 0.001 0 0 0 0.001 0 0 -8.80615 0.001 0 0 0 0.001 0 0 -8.81104 0.001 0 0 0 0.001 0 0 -8.81592 0.001 0 0 0 0.001 0 0 -8.8208 0.001 0 0 0 0.001 0 0 -8.82568 0.001 0 0 0 0.001 0 0 -8.83057 0.001 0 0 0 0.001 0 0 -8.83545 0.001 0 0 0 0.001 0 0 -8.84033 0.001 0 0 0 0.001 0 0 -8.84521 0.001 0 0 0 0.001 0 0 -8.8501 0.001 0 0 0 0.001 0 0 -8.85498 0.001 0 0 0 0.001 0 0 -8.85986 0.001 0 0 0 0.001 0 0 -8.86475 0.001 0 0 0 0.001 0 0 -8.86963 0.001 0 0 0 0.001 0 0 -8.87451 0.001 0 0 0 0.001 0 0 -8.87939 0.001 0 0 0 0.001 0 0 -8.88428 0.001 0 0 0 0.001 0 0 -8.88916 0.001 0 0 0 0.001 0 0 -8.89404 0.001 0 0 0 0.001 0 0 -8.89893 0.001 0 0 0 0.001 0 0 -8.90381 0.001 0 0 0 0.001 0 0 -8.90869 0.001 0 0 0 0.001 0 0 -8.91357 0.001 0 0 0 0.001 0 0 -8.91846 0.001 0 0 0 0.001 0 0 -8.92334 0.001 0 0 0 0.001 0 0 -8.92822 0.001 0 0 0 0.001 0 0 -8.93311 0.001 0 0 0 0.001 0 0 -8.93799 0.001 0 0 0 0.001 0 0 -8.94287 0.001 0 0 0 0.001 0 0 -8.94775 0.001 0 0 0 0.001 0 0 -8.95264 0.001 0 0 0 0.001 0 0 -8.95752 0.001 0 0 0 0.001 0 0 -8.9624 0.001 0 0 0 0.001 0 0 -8.96729 0.001 0 0 0 0.001 0 0 -8.97217 0.001 0 0 0 0.001 0 0 -8.97705 0.001 0 0 0 0.001 0 0 -8.98193 0.001 0 0 0 0.001 0 0 -8.98682 0.001 0 0 0 0.001 0 0 -8.9917 0.001 0 0 0 0.001 0 0 -8.99658 0.001 0 0 0 0.001 0 0 -9.00146 0.001 0 0 0 0.001 0 0 -9.00635 0.001 0 0 0 0.001 0 0 -9.01123 0.001 0 0 0 0.001 0 0 -9.01611 0.001 0 0 0 0.001 0 0 -9.021 0.001 0 0 0 0.001 0 0 -9.02588 0.001 0 0 0 0.001 0 0 -9.03076 0.001 0 0 0 0.001 0 0 -9.03564 0.001 0 0 0 0.001 0 0 -9.04053 0.001 0 0 0 0.001 0 0 -9.04541 0.001 0 0 0 0.001 0 0 -9.05029 0.001 0 0 0 0.001 0 0 -9.05518 0.001 0 0 0 0.001 0 0 -9.06006 0.001 0 0 0 0.001 0 0 -9.06494 0.001 0 0 0 0.001 0 0 -9.06982 0.001 0 0 0 0.001 0 0 -9.07471 0.001 0 0 0 0.001 0 0 -9.07959 0.001 0 0 0 0.001 0 0 -9.08447 0.001 0 0 0 0.001 0 0 -9.08936 0.001 0 0 0 0.001 0 0 -9.09424 0.001 0 0 0 0.001 0 0 -9.09912 0.001 0 0 0 0.001 0 0 -9.104 0.001 0 0 0 0.001 0 0 -9.10889 0.001 0 0 0 0.001 0 0 -9.11377 0.001 0 0 0 0.001 0 0 -9.11865 0.001 0 0 0 0.001 0 0 -9.12354 0.001 0 0 0 0.001 0 0 -9.12842 0.001 0 0 0 0.001 0 0 -9.1333 0.001 0 0 0 0.001 0 0 -9.13818 0.001 0 0 0 0.001 0 0 -9.14307 0.001 0 0 0 0.001 0 0 -9.14795 0.001 0 0 0 0.001 0 0 -9.15283 0.001 0 0 0 0.001 0 0 -9.15771 0.001 0 0 0 0.001 0 0 -9.1626 0.001 0 0 0 0.001 0 0 -9.16748 0.001 0 0 0 0.001 0 0 -9.17236 0.001 0 0 0 0.001 0 0 -9.17725 0.001 0 0 0 0.001 0 0 -9.18213 0.001 0 0 0 0.001 0 0 -9.18701 0.001 0 0 0 0.001 0 0 -9.19189 0.001 0 0 0 0.001 0 0 -9.19678 0.001 0 0 0 0.001 0 0 -9.20166 0.001 0 0 0 0.001 0 0 -9.20654 0.001 0 0 0 0.001 0 0 -9.21143 0.001 0 0 0 0.001 0 0 -9.21631 0.001 0 0 0 0.001 0 0 -9.22119 0.001 0 0 0 0.001 0 0 -9.22607 0.001 0 0 0 0.001 0 0 -9.23096 0.001 0 0 0 0.001 0 0 -9.23584 0.001 0 0 0 0.001 0 0 -9.24072 0.001 0 0 0 0.001 0 0 -9.24561 0.001 0 0 0 0.001 0 0 -9.25049 0.001 0 0 0 0.001 0 0 -9.25537 0.001 0 0 0 0.001 0 0 -9.26025 0.001 0 0 0 0.001 0 0 -9.26514 0.001 0 0 0 0.001 0 0 -9.27002 0.001 0 0 0 0.001 0 0 -9.2749 0.001 0 0 0 0.001 0 0 -9.27979 0.001 0 0 0 0.001 0 0 -9.28467 0.001 0 0 0 0.001 0 0 -9.28955 0.001 0 0 0 0.001 0 0 -9.29443 0.001 0 0 0 0.001 0 0 -9.29932 0.001 0 0 0 0.001 0 0 -9.3042 0.001 0 0 0 0.001 0 0 -9.30908 0.001 0 0 0 0.001 0 0 -9.31396 0.001 0 0 0 0.001 0 0 -9.31885 0.001 0 0 0 0.001 0 0 -9.32373 0.001 0 0 0 0.001 0 0 -9.32861 0.001 0 0 0 0.001 0 0 -9.3335 0.001 0 0 0 0.001 0 0 -9.33838 0.001 0 0 0 0.001 0 0 -9.34326 0.001 0 0 0 0.001 0 0 -9.34814 0.001 0 0 0 0.001 0 0 -9.35303 0.001 0 0 0 0.001 0 0 -9.35791 0.001 0 0 0 0.001 0 0 -9.36279 0.001 0 0 0 0.001 0 0 -9.36768 0.001 0 0 0 0.001 0 0 -9.37256 0.001 0 0 0 0.001 0 0 -9.37744 0.001 0 0 0 0.001 0 0 -9.38232 0.001 0 0 0 0.001 0 0 -9.38721 0.001 0 0 0 0.001 0 0 -9.39209 0.001 0 0 0 0.001 0 0 -9.39697 0.001 0 0 0 0.001 0 0 -9.40186 0.001 0 0 0 0.001 0 0 -9.40674 0.001 0 0 0 0.001 0 0 -9.41162 0.001 0 0 0 0.001 0 0 -9.4165 0.001 0 0 0 0.001 0 0 -9.42139 0.001 0 0 0 0.001 0 0 -9.42627 0.001 0 0 0 0.001 0 0 -9.43115 0.001 0 0 0 0.001 0 0 -9.43604 0.001 0 0 0 0.001 0 0 -9.44092 0.001 0 0 0 0.001 0 0 -9.4458 0.001 0 0 0 0.001 0 0 -9.45068 0.001 0 0 0 0.001 0 0 -9.45557 0.001 0 0 0 0.001 0 0 -9.46045 0.001 0 0 0 0.001 0 0 -9.46533 0.001 0 0 0 0.001 0 0 -9.47021 0.001 0 0 0 0.001 0 0 -9.4751 0.001 0 0 0 0.001 0 0 -9.47998 0.001 0 0 0 0.001 0 0 -9.48486 0.001 0 0 0 0.001 0 0 -9.48975 0.001 0 0 0 0.001 0 0 -9.49463 0.001 0 0 0 0.001 0 0 -9.49951 0.001 0 0 0 0.001 0 0 -9.50439 0.001 0 0 0 0.001 0 0 -9.50928 0.001 0 0 0 0.001 0 0 -9.51416 0.001 0 0 0 0.001 0 0 -9.51904 0.001 0 0 0 0.001 0 0 -9.52393 0.001 0 0 0 0.001 0 0 -9.52881 0.001 0 0 0 0.001 0 0 -9.53369 0.001 0 0 0 0.001 0 0 -9.53857 0.001 0 0 0 0.001 0 0 -9.54346 0.001 0 0 0 0.001 0 0 -9.54834 0.001 0 0 0 0.001 0 0 -9.55322 0.001 0 0 0 0.001 0 0 -9.55811 0.001 0 0 0 0.001 0 0 -9.56299 0.001 0 0 0 0.001 0 0 -9.56787 0.001 0 0 0 0.001 0 0 -9.57275 0.001 0 0 0 0.001 0 0 -9.57764 0.001 0 0 0 0.001 0 0 -9.58252 0.001 0 0 0 0.001 0 0 -9.5874 0.001 0 0 0 0.001 0 0 -9.59229 0.001 0 0 0 0.001 0 0 -9.59717 0.001 0 0 0 0.001 0 0 -9.60205 0.001 0 0 0 0.001 0 0 -9.60693 0.001 0 0 0 0.001 0 0 -9.61182 0.001 0 0 0 0.001 0 0 -9.6167 0.001 0 0 0 0.001 0 0 -9.62158 0.001 0 0 0 0.001 0 0 -9.62646 0.001 0 0 0 0.001 0 0 -9.63135 0.001 0 0 0 0.001 0 0 -9.63623 0.001 0 0 0 0.001 0 0 -9.64111 0.001 0 0 0 0.001 0 0 -9.646 0.001 0 0 0 0.001 0 0 -9.65088 0.001 0 0 0 0.001 0 0 -9.65576 0.001 0 0 0 0.001 0 0 -9.66064 0.001 0 0 0 0.001 0 0 -9.66553 0.001 0 0 0 0.001 0 0 -9.67041 0.001 0 0 0 0.001 0 0 -9.67529 0.001 0 0 0 0.001 0 0 -9.68018 0.001 0 0 0 0.001 0 0 -9.68506 0.001 0 0 0 0.001 0 0 -9.68994 0.001 0 0 0 0.001 0 0 -9.69482 0.001 0 0 0 0.001 0 0 -9.69971 0.001 0 0 0 0.001 0 0 -9.70459 0.001 0 0 0 0.001 0 0 -9.70947 0.001 0 0 0 0.001 0 0 -9.71436 0.001 0 0 0 0.001 0 0 -9.71924 0.001 0 0 0 0.001 0 0 -9.72412 0.001 0 0 0 0.001 0 0 -9.729 0.001 0 0 0 0.001 0 0 -9.73389 0.001 0 0 0 0.001 0 0 -9.73877 0.001 0 0 0 0.001 0 0 -9.74365 0.001 0 0 0 0.001 0 0 -9.74854 0.001 0 0 0 0.001 0 0 -9.75342 0.001 0 0 0 0.001 0 0 -9.7583 0.001 0 0 0 0.001 0 0 -9.76318 0.001 0 0 0 0.001 0 0 -9.76807 0.001 0 0 0 0.001 0 0 -9.77295 0.001 0 0 0 0.001 0 0 -9.77783 0.001 0 0 0 0.001 0 0 -9.78271 0.001 0 0 0 0.001 0 0 -9.7876 0.001 0 0 0 0.001 0 0 -9.79248 0.001 0 0 0 0.001 0 0 -9.79736 0.001 0 0 0 0.001 0 0 -9.80225 0.001 0 0 0 0.001 0 0 -9.80713 0.001 0 0 0 0.001 0 0 -9.81201 0.001 0 0 0 0.001 0 0 -9.81689 0.001 0 0 0 0.001 0 0 -9.82178 0.001 0 0 0 0.001 0 0 -9.82666 0.001 0 0 0 0.001 0 0 -9.83154 0.001 0 0 0 0.001 0 0 -9.83643 0.001 0 0 0 0.001 0 0 -9.84131 0.001 0 0 0 0.001 0 0 -9.84619 0.001 0 0 0 0.001 0 0 -9.85107 0.001 0 0 0 0.001 0 0 -9.85596 0.001 0 0 0 0.001 0 0 -9.86084 0.001 0 0 0 0.001 0 0 -9.86572 0.001 0 0 0 0.001 0 0 -9.87061 0.001 0 0 0 0.001 0 0 -9.87549 0.001 0 0 0 0.001 0 0 -9.88037 0.001 0 0 0 0.001 0 0 -9.88525 0.001 0 0 0 0.001 0 0 -9.89014 0.001 0 0 0 0.001 0 0 -9.89502 0.001 0 0 0 0.001 0 0 -9.8999 0.001 0 0 0 0.001 0 0 -9.90479 0.001 0 0 0 0.001 0 0 -9.90967 0.001 0 0 0 0.001 0 0 -9.91455 0.001 0 0 0 0.001 0 0 -9.91943 0.001 0 0 0 0.001 0 0 -9.92432 0.001 0 0 0 0.001 0 0 -9.9292 0.001 0 0 0 0.001 0 0 -9.93408 0.001 0 0 0 0.001 0 0 -9.93896 0.001 0 0 0 0.001 0 0 -9.94385 0.001 0 0 0 0.001 0 0 -9.94873 0.001 0 0 0 0.001 0 0 -9.95361 0.001 0 0 0 0.001 0 0 -9.9585 0.001 0 0 0 0.001 0 0 -9.96338 0.001 0 0 0 0.001 0 0 -9.96826 0.001 0 0 0 0.001 0 0 -9.97314 0.001 0 0 0 0.001 0 0 -9.97803 0.001 0 0 0 0.001 0 0 -9.98291 0.001 0 0 0 0.001 0 0 -9.98779 0.001 0 0 0 0.001 0 0 -9.99268 0.001 0 0 0 0.001 0 0 -9.99756 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=256.csv b/reference/swashes_1_nx=256.csv deleted file mode 100644 index 65a55ba..0000000 --- a/reference/swashes_1_nx=256.csv +++ /dev/null @@ -1,274 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.0390625 meters -# Number of cells: 256 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.0195312 0.005 0 0 0 0.005 0 0 -0.0585938 0.005 0 0 0 0.005 0 0 -0.0976562 0.005 0 0 0 0.005 0 0 -0.136719 0.005 0 0 0 0.005 0 0 -0.175781 0.005 0 0 0 0.005 0 0 -0.214844 0.005 0 0 0 0.005 0 0 -0.253906 0.005 0 0 0 0.005 0 0 -0.292969 0.005 0 0 0 0.005 0 0 -0.332031 0.005 0 0 0 0.005 0 0 -0.371094 0.005 0 0 0 0.005 0 0 -0.410156 0.005 0 0 0 0.005 0 0 -0.449219 0.005 0 0 0 0.005 0 0 -0.488281 0.005 0 0 0 0.005 0 0 -0.527344 0.005 0 0 0 0.005 0 0 -0.566406 0.005 0 0 0 0.005 0 0 -0.605469 0.005 0 0 0 0.005 0 0 -0.644531 0.005 0 0 0 0.005 0 0 -0.683594 0.005 0 0 0 0.005 0 0 -0.722656 0.005 0 0 0 0.005 0 0 -0.761719 0.005 0 0 0 0.005 0 0 -0.800781 0.005 0 0 0 0.005 0 0 -0.839844 0.005 0 0 0 0.005 0 0 -0.878906 0.005 0 0 0 0.005 0 0 -0.917969 0.005 0 0 0 0.005 0 0 -0.957031 0.005 0 0 0 0.005 0 0 -0.996094 0.005 0 0 0 0.005 0 0 -1.03516 0.005 0 0 0 0.005 0 0 -1.07422 0.005 0 0 0 0.005 0 0 -1.11328 0.005 0 0 0 0.005 0 0 -1.15234 0.005 0 0 0 0.005 0 0 -1.19141 0.005 0 0 0 0.005 0 0 -1.23047 0.005 0 0 0 0.005 0 0 -1.26953 0.005 0 0 0 0.005 0 0 -1.30859 0.005 0 0 0 0.005 0 0 -1.34766 0.005 0 0 0 0.005 0 0 -1.38672 0.005 0 0 0 0.005 0 0 -1.42578 0.005 0 0 0 0.005 0 0 -1.46484 0.005 0 0 0 0.005 0 0 -1.50391 0.005 0 0 0 0.005 0 0 -1.54297 0.005 0 0 0 0.005 0 0 -1.58203 0.005 0 0 0 0.005 0 0 -1.62109 0.005 0 0 0 0.005 0 0 -1.66016 0.005 0 0 0 0.005 0 0 -1.69922 0.005 0 0 0 0.005 0 0 -1.73828 0.005 0 0 0 0.005 0 0 -1.77734 0.005 0 0 0 0.005 0 0 -1.81641 0.005 0 0 0 0.005 0 0 -1.85547 0.005 0 0 0 0.005 0 0 -1.89453 0.005 0 0 0 0.005 0 0 -1.93359 0.005 0 0 0 0.005 0 0 -1.97266 0.005 0 0 0 0.005 0 0 -2.01172 0.005 0 0 0 0.005 0 0 -2.05078 0.005 0 0 0 0.005 0 0 -2.08984 0.005 0 0 0 0.005 0 0 -2.12891 0.005 0 0 0 0.005 0 0 -2.16797 0.005 0 0 0 0.005 0 0 -2.20703 0.005 0 0 0 0.005 0 0 -2.24609 0.005 0 0 0 0.005 0 0 -2.28516 0.005 0 0 0 0.005 0 0 -2.32422 0.005 0 0 0 0.005 0 0 -2.36328 0.005 0 0 0 0.005 0 0 -2.40234 0.005 0 0 0 0.005 0 0 -2.44141 0.005 0 0 0 0.005 0 0 -2.48047 0.005 0 0 0 0.005 0 0 -2.51953 0.005 0 0 0 0.005 0 0 -2.55859 0.005 0 0 0 0.005 0 0 -2.59766 0.005 0 0 0 0.005 0 0 -2.63672 0.005 0 0 0 0.005 0 0 -2.67578 0.005 0 0 0 0.005 0 0 -2.71484 0.005 0 0 0 0.005 0 0 -2.75391 0.005 0 0 0 0.005 0 0 -2.79297 0.005 0 0 0 0.005 0 0 -2.83203 0.005 0 0 0 0.005 0 0 -2.87109 0.005 0 0 0 0.005 0 0 -2.91016 0.005 0 0 0 0.005 0 0 -2.94922 0.005 0 0 0 0.005 0 0 -2.98828 0.005 0 0 0 0.005 0 0 -3.02734 0.005 0 0 0 0.005 0 0 -3.06641 0.005 0 0 0 0.005 0 0 -3.10547 0.005 0 0 0 0.005 0 0 -3.14453 0.005 0 0 0 0.005 0 0 -3.18359 0.005 0 0 0 0.005 0 0 -3.22266 0.005 0 0 0 0.005 0 0 -3.26172 0.005 0 0 0 0.005 0 0 -3.30078 0.005 0 0 0 0.005 0 0 -3.33984 0.005 0 0 0 0.005 0 0 -3.37891 0.005 0 0 0 0.005 0 0 -3.41797 0.005 0 0 0 0.005 0 0 -3.45703 0.005 0 0 0 0.005 0 0 -3.49609 0.005 0 0 0 0.005 0 0 -3.53516 0.005 0 0 0 0.005 0 0 -3.57422 0.005 0 0 0 0.005 0 0 -3.61328 0.005 0 0 0 0.005 0 0 -3.65234 0.005 0 0 0 0.005 0 0 -3.69141 0.00494936 0.00224893 0 1.11307e-005 0.00494936 0.0102062 0.000232877 -3.73047 0.00485235 0.0065892 0 3.19731e-005 0.00485235 0.0302011 0.00047058 -3.76953 0.0047563 0.0109295 0 5.19839e-005 0.0047563 0.0505977 0.000650663 -3.80859 0.00466121 0.0152698 0 7.11755e-005 0.00466121 0.0714082 0.000802289 -3.84766 0.00456708 0.01961 0 8.95606e-005 0.00456708 0.0926456 0.000935093 -3.88672 0.00447391 0.0239503 0 0.000107152 0.00447391 0.114323 0.00105384 -3.92578 0.0043817 0.0282906 0 0.000123961 0.0043817 0.136454 0.00116136 -3.96484 0.00429045 0.0326309 0 0.000140001 0.00429045 0.159053 0.0012595 -4.00391 0.00420017 0.0369711 0 0.000155285 0.00420017 0.182136 0.00134957 -4.04297 0.00411084 0.0413114 0 0.000169825 0.00411084 0.205717 0.00143255 -4.08203 0.00402247 0.0456517 0 0.000183633 0.00402247 0.229814 0.00150919 -4.12109 0.00393506 0.049992 0 0.000196722 0.00393506 0.254443 0.00158008 -4.16016 0.00384861 0.0543323 0 0.000209104 0.00384861 0.279622 0.0016457 -4.19922 0.00376313 0.0586725 0 0.000220792 0.00376313 0.30537 0.00170647 -4.23828 0.0036786 0.0630128 0 0.000231799 0.0036786 0.331706 0.00176272 -4.27734 0.00359503 0.0673531 0 0.000242137 0.00359503 0.358651 0.00181475 -4.31641 0.00351242 0.0716934 0 0.000251818 0.00351242 0.386226 0.00186281 -4.35547 0.00343078 0.0760336 0 0.000260855 0.00343078 0.414453 0.00190711 -4.39453 0.00335009 0.0803739 0 0.00026926 0.00335009 0.443356 0.00194786 -4.43359 0.00327036 0.0847142 0 0.000277046 0.00327036 0.472959 0.00198523 -4.47266 0.0031916 0.0890545 0 0.000284226 0.0031916 0.503289 0.00201939 -4.51172 0.00311379 0.0933948 0 0.000290812 0.00311379 0.534371 0.00205046 -4.55078 0.00303694 0.097735 0 0.000296816 0.00303694 0.566236 0.00207859 -4.58984 0.00296106 0.102075 0 0.000302251 0.00296106 0.598912 0.00210389 -4.62891 0.00288613 0.106416 0 0.000307129 0.00288613 0.63243 0.00212646 -4.66797 0.00281217 0.110756 0 0.000311464 0.00281217 0.666825 0.00214642 -4.70703 0.00273916 0.115096 0 0.000315267 0.00273916 0.70213 0.00216386 -4.74609 0.00266712 0.119436 0 0.000318551 0.00266712 0.738383 0.00217886 -4.78516 0.00259603 0.123777 0 0.000321328 0.00259603 0.775621 0.00219151 -4.82422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01953 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05859 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09766 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13672 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17578 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21484 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25391 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29297 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33203 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37109 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41016 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44922 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48828 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52734 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56641 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60547 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64453 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68359 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72266 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76172 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80078 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83984 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87891 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91797 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95703 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99609 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03516 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.26953 0.001 0 0 0 0.001 0 0 -6.30859 0.001 0 0 0 0.001 0 0 -6.34766 0.001 0 0 0 0.001 0 0 -6.38672 0.001 0 0 0 0.001 0 0 -6.42578 0.001 0 0 0 0.001 0 0 -6.46484 0.001 0 0 0 0.001 0 0 -6.50391 0.001 0 0 0 0.001 0 0 -6.54297 0.001 0 0 0 0.001 0 0 -6.58203 0.001 0 0 0 0.001 0 0 -6.62109 0.001 0 0 0 0.001 0 0 -6.66016 0.001 0 0 0 0.001 0 0 -6.69922 0.001 0 0 0 0.001 0 0 -6.73828 0.001 0 0 0 0.001 0 0 -6.77734 0.001 0 0 0 0.001 0 0 -6.81641 0.001 0 0 0 0.001 0 0 -6.85547 0.001 0 0 0 0.001 0 0 -6.89453 0.001 0 0 0 0.001 0 0 -6.93359 0.001 0 0 0 0.001 0 0 -6.97266 0.001 0 0 0 0.001 0 0 -7.01172 0.001 0 0 0 0.001 0 0 -7.05078 0.001 0 0 0 0.001 0 0 -7.08984 0.001 0 0 0 0.001 0 0 -7.12891 0.001 0 0 0 0.001 0 0 -7.16797 0.001 0 0 0 0.001 0 0 -7.20703 0.001 0 0 0 0.001 0 0 -7.24609 0.001 0 0 0 0.001 0 0 -7.28516 0.001 0 0 0 0.001 0 0 -7.32422 0.001 0 0 0 0.001 0 0 -7.36328 0.001 0 0 0 0.001 0 0 -7.40234 0.001 0 0 0 0.001 0 0 -7.44141 0.001 0 0 0 0.001 0 0 -7.48047 0.001 0 0 0 0.001 0 0 -7.51953 0.001 0 0 0 0.001 0 0 -7.55859 0.001 0 0 0 0.001 0 0 -7.59766 0.001 0 0 0 0.001 0 0 -7.63672 0.001 0 0 0 0.001 0 0 -7.67578 0.001 0 0 0 0.001 0 0 -7.71484 0.001 0 0 0 0.001 0 0 -7.75391 0.001 0 0 0 0.001 0 0 -7.79297 0.001 0 0 0 0.001 0 0 -7.83203 0.001 0 0 0 0.001 0 0 -7.87109 0.001 0 0 0 0.001 0 0 -7.91016 0.001 0 0 0 0.001 0 0 -7.94922 0.001 0 0 0 0.001 0 0 -7.98828 0.001 0 0 0 0.001 0 0 -8.02734 0.001 0 0 0 0.001 0 0 -8.06641 0.001 0 0 0 0.001 0 0 -8.10547 0.001 0 0 0 0.001 0 0 -8.14453 0.001 0 0 0 0.001 0 0 -8.18359 0.001 0 0 0 0.001 0 0 -8.22266 0.001 0 0 0 0.001 0 0 -8.26172 0.001 0 0 0 0.001 0 0 -8.30078 0.001 0 0 0 0.001 0 0 -8.33984 0.001 0 0 0 0.001 0 0 -8.37891 0.001 0 0 0 0.001 0 0 -8.41797 0.001 0 0 0 0.001 0 0 -8.45703 0.001 0 0 0 0.001 0 0 -8.49609 0.001 0 0 0 0.001 0 0 -8.53516 0.001 0 0 0 0.001 0 0 -8.57422 0.001 0 0 0 0.001 0 0 -8.61328 0.001 0 0 0 0.001 0 0 -8.65234 0.001 0 0 0 0.001 0 0 -8.69141 0.001 0 0 0 0.001 0 0 -8.73047 0.001 0 0 0 0.001 0 0 -8.76953 0.001 0 0 0 0.001 0 0 -8.80859 0.001 0 0 0 0.001 0 0 -8.84766 0.001 0 0 0 0.001 0 0 -8.88672 0.001 0 0 0 0.001 0 0 -8.92578 0.001 0 0 0 0.001 0 0 -8.96484 0.001 0 0 0 0.001 0 0 -9.00391 0.001 0 0 0 0.001 0 0 -9.04297 0.001 0 0 0 0.001 0 0 -9.08203 0.001 0 0 0 0.001 0 0 -9.12109 0.001 0 0 0 0.001 0 0 -9.16016 0.001 0 0 0 0.001 0 0 -9.19922 0.001 0 0 0 0.001 0 0 -9.23828 0.001 0 0 0 0.001 0 0 -9.27734 0.001 0 0 0 0.001 0 0 -9.31641 0.001 0 0 0 0.001 0 0 -9.35547 0.001 0 0 0 0.001 0 0 -9.39453 0.001 0 0 0 0.001 0 0 -9.43359 0.001 0 0 0 0.001 0 0 -9.47266 0.001 0 0 0 0.001 0 0 -9.51172 0.001 0 0 0 0.001 0 0 -9.55078 0.001 0 0 0 0.001 0 0 -9.58984 0.001 0 0 0 0.001 0 0 -9.62891 0.001 0 0 0 0.001 0 0 -9.66797 0.001 0 0 0 0.001 0 0 -9.70703 0.001 0 0 0 0.001 0 0 -9.74609 0.001 0 0 0 0.001 0 0 -9.78516 0.001 0 0 0 0.001 0 0 -9.82422 0.001 0 0 0 0.001 0 0 -9.86328 0.001 0 0 0 0.001 0 0 -9.90234 0.001 0 0 0 0.001 0 0 -9.94141 0.001 0 0 0 0.001 0 0 -9.98047 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=4096.csv b/reference/swashes_1_nx=4096.csv deleted file mode 100644 index be77bf7..0000000 --- a/reference/swashes_1_nx=4096.csv +++ /dev/null @@ -1,4114 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.00244141 meters -# Number of cells: 4096 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.0012207 0.005 0 0 0 0.005 0 0 -0.00366211 0.005 0 0 0 0.005 0 0 -0.00610352 0.005 0 0 0 0.005 0 0 -0.00854492 0.005 0 0 0 0.005 0 0 -0.0109863 0.005 0 0 0 0.005 0 0 -0.0134277 0.005 0 0 0 0.005 0 0 -0.0158691 0.005 0 0 0 0.005 0 0 -0.0183105 0.005 0 0 0 0.005 0 0 -0.020752 0.005 0 0 0 0.005 0 0 -0.0231934 0.005 0 0 0 0.005 0 0 -0.0256348 0.005 0 0 0 0.005 0 0 -0.0280762 0.005 0 0 0 0.005 0 0 -0.0305176 0.005 0 0 0 0.005 0 0 -0.032959 0.005 0 0 0 0.005 0 0 -0.0354004 0.005 0 0 0 0.005 0 0 -0.0378418 0.005 0 0 0 0.005 0 0 -0.0402832 0.005 0 0 0 0.005 0 0 -0.0427246 0.005 0 0 0 0.005 0 0 -0.045166 0.005 0 0 0 0.005 0 0 -0.0476074 0.005 0 0 0 0.005 0 0 -0.0500488 0.005 0 0 0 0.005 0 0 -0.0524902 0.005 0 0 0 0.005 0 0 -0.0549316 0.005 0 0 0 0.005 0 0 -0.057373 0.005 0 0 0 0.005 0 0 -0.0598145 0.005 0 0 0 0.005 0 0 -0.0622559 0.005 0 0 0 0.005 0 0 -0.0646973 0.005 0 0 0 0.005 0 0 -0.0671387 0.005 0 0 0 0.005 0 0 -0.0695801 0.005 0 0 0 0.005 0 0 -0.0720215 0.005 0 0 0 0.005 0 0 -0.0744629 0.005 0 0 0 0.005 0 0 -0.0769043 0.005 0 0 0 0.005 0 0 -0.0793457 0.005 0 0 0 0.005 0 0 -0.0817871 0.005 0 0 0 0.005 0 0 -0.0842285 0.005 0 0 0 0.005 0 0 -0.0866699 0.005 0 0 0 0.005 0 0 -0.0891113 0.005 0 0 0 0.005 0 0 -0.0915527 0.005 0 0 0 0.005 0 0 -0.0939941 0.005 0 0 0 0.005 0 0 -0.0964355 0.005 0 0 0 0.005 0 0 -0.098877 0.005 0 0 0 0.005 0 0 -0.101318 0.005 0 0 0 0.005 0 0 -0.10376 0.005 0 0 0 0.005 0 0 -0.106201 0.005 0 0 0 0.005 0 0 -0.108643 0.005 0 0 0 0.005 0 0 -0.111084 0.005 0 0 0 0.005 0 0 -0.113525 0.005 0 0 0 0.005 0 0 -0.115967 0.005 0 0 0 0.005 0 0 -0.118408 0.005 0 0 0 0.005 0 0 -0.12085 0.005 0 0 0 0.005 0 0 -0.123291 0.005 0 0 0 0.005 0 0 -0.125732 0.005 0 0 0 0.005 0 0 -0.128174 0.005 0 0 0 0.005 0 0 -0.130615 0.005 0 0 0 0.005 0 0 -0.133057 0.005 0 0 0 0.005 0 0 -0.135498 0.005 0 0 0 0.005 0 0 -0.137939 0.005 0 0 0 0.005 0 0 -0.140381 0.005 0 0 0 0.005 0 0 -0.142822 0.005 0 0 0 0.005 0 0 -0.145264 0.005 0 0 0 0.005 0 0 -0.147705 0.005 0 0 0 0.005 0 0 -0.150146 0.005 0 0 0 0.005 0 0 -0.152588 0.005 0 0 0 0.005 0 0 -0.155029 0.005 0 0 0 0.005 0 0 -0.157471 0.005 0 0 0 0.005 0 0 -0.159912 0.005 0 0 0 0.005 0 0 -0.162354 0.005 0 0 0 0.005 0 0 -0.164795 0.005 0 0 0 0.005 0 0 -0.167236 0.005 0 0 0 0.005 0 0 -0.169678 0.005 0 0 0 0.005 0 0 -0.172119 0.005 0 0 0 0.005 0 0 -0.174561 0.005 0 0 0 0.005 0 0 -0.177002 0.005 0 0 0 0.005 0 0 -0.179443 0.005 0 0 0 0.005 0 0 -0.181885 0.005 0 0 0 0.005 0 0 -0.184326 0.005 0 0 0 0.005 0 0 -0.186768 0.005 0 0 0 0.005 0 0 -0.189209 0.005 0 0 0 0.005 0 0 -0.19165 0.005 0 0 0 0.005 0 0 -0.194092 0.005 0 0 0 0.005 0 0 -0.196533 0.005 0 0 0 0.005 0 0 -0.198975 0.005 0 0 0 0.005 0 0 -0.201416 0.005 0 0 0 0.005 0 0 -0.203857 0.005 0 0 0 0.005 0 0 -0.206299 0.005 0 0 0 0.005 0 0 -0.20874 0.005 0 0 0 0.005 0 0 -0.211182 0.005 0 0 0 0.005 0 0 -0.213623 0.005 0 0 0 0.005 0 0 -0.216064 0.005 0 0 0 0.005 0 0 -0.218506 0.005 0 0 0 0.005 0 0 -0.220947 0.005 0 0 0 0.005 0 0 -0.223389 0.005 0 0 0 0.005 0 0 -0.22583 0.005 0 0 0 0.005 0 0 -0.228271 0.005 0 0 0 0.005 0 0 -0.230713 0.005 0 0 0 0.005 0 0 -0.233154 0.005 0 0 0 0.005 0 0 -0.235596 0.005 0 0 0 0.005 0 0 -0.238037 0.005 0 0 0 0.005 0 0 -0.240479 0.005 0 0 0 0.005 0 0 -0.24292 0.005 0 0 0 0.005 0 0 -0.245361 0.005 0 0 0 0.005 0 0 -0.247803 0.005 0 0 0 0.005 0 0 -0.250244 0.005 0 0 0 0.005 0 0 -0.252686 0.005 0 0 0 0.005 0 0 -0.255127 0.005 0 0 0 0.005 0 0 -0.257568 0.005 0 0 0 0.005 0 0 -0.26001 0.005 0 0 0 0.005 0 0 -0.262451 0.005 0 0 0 0.005 0 0 -0.264893 0.005 0 0 0 0.005 0 0 -0.267334 0.005 0 0 0 0.005 0 0 -0.269775 0.005 0 0 0 0.005 0 0 -0.272217 0.005 0 0 0 0.005 0 0 -0.274658 0.005 0 0 0 0.005 0 0 -0.2771 0.005 0 0 0 0.005 0 0 -0.279541 0.005 0 0 0 0.005 0 0 -0.281982 0.005 0 0 0 0.005 0 0 -0.284424 0.005 0 0 0 0.005 0 0 -0.286865 0.005 0 0 0 0.005 0 0 -0.289307 0.005 0 0 0 0.005 0 0 -0.291748 0.005 0 0 0 0.005 0 0 -0.294189 0.005 0 0 0 0.005 0 0 -0.296631 0.005 0 0 0 0.005 0 0 -0.299072 0.005 0 0 0 0.005 0 0 -0.301514 0.005 0 0 0 0.005 0 0 -0.303955 0.005 0 0 0 0.005 0 0 -0.306396 0.005 0 0 0 0.005 0 0 -0.308838 0.005 0 0 0 0.005 0 0 -0.311279 0.005 0 0 0 0.005 0 0 -0.313721 0.005 0 0 0 0.005 0 0 -0.316162 0.005 0 0 0 0.005 0 0 -0.318604 0.005 0 0 0 0.005 0 0 -0.321045 0.005 0 0 0 0.005 0 0 -0.323486 0.005 0 0 0 0.005 0 0 -0.325928 0.005 0 0 0 0.005 0 0 -0.328369 0.005 0 0 0 0.005 0 0 -0.330811 0.005 0 0 0 0.005 0 0 -0.333252 0.005 0 0 0 0.005 0 0 -0.335693 0.005 0 0 0 0.005 0 0 -0.338135 0.005 0 0 0 0.005 0 0 -0.340576 0.005 0 0 0 0.005 0 0 -0.343018 0.005 0 0 0 0.005 0 0 -0.345459 0.005 0 0 0 0.005 0 0 -0.3479 0.005 0 0 0 0.005 0 0 -0.350342 0.005 0 0 0 0.005 0 0 -0.352783 0.005 0 0 0 0.005 0 0 -0.355225 0.005 0 0 0 0.005 0 0 -0.357666 0.005 0 0 0 0.005 0 0 -0.360107 0.005 0 0 0 0.005 0 0 -0.362549 0.005 0 0 0 0.005 0 0 -0.36499 0.005 0 0 0 0.005 0 0 -0.367432 0.005 0 0 0 0.005 0 0 -0.369873 0.005 0 0 0 0.005 0 0 -0.372314 0.005 0 0 0 0.005 0 0 -0.374756 0.005 0 0 0 0.005 0 0 -0.377197 0.005 0 0 0 0.005 0 0 -0.379639 0.005 0 0 0 0.005 0 0 -0.38208 0.005 0 0 0 0.005 0 0 -0.384521 0.005 0 0 0 0.005 0 0 -0.386963 0.005 0 0 0 0.005 0 0 -0.389404 0.005 0 0 0 0.005 0 0 -0.391846 0.005 0 0 0 0.005 0 0 -0.394287 0.005 0 0 0 0.005 0 0 -0.396729 0.005 0 0 0 0.005 0 0 -0.39917 0.005 0 0 0 0.005 0 0 -0.401611 0.005 0 0 0 0.005 0 0 -0.404053 0.005 0 0 0 0.005 0 0 -0.406494 0.005 0 0 0 0.005 0 0 -0.408936 0.005 0 0 0 0.005 0 0 -0.411377 0.005 0 0 0 0.005 0 0 -0.413818 0.005 0 0 0 0.005 0 0 -0.41626 0.005 0 0 0 0.005 0 0 -0.418701 0.005 0 0 0 0.005 0 0 -0.421143 0.005 0 0 0 0.005 0 0 -0.423584 0.005 0 0 0 0.005 0 0 -0.426025 0.005 0 0 0 0.005 0 0 -0.428467 0.005 0 0 0 0.005 0 0 -0.430908 0.005 0 0 0 0.005 0 0 -0.43335 0.005 0 0 0 0.005 0 0 -0.435791 0.005 0 0 0 0.005 0 0 -0.438232 0.005 0 0 0 0.005 0 0 -0.440674 0.005 0 0 0 0.005 0 0 -0.443115 0.005 0 0 0 0.005 0 0 -0.445557 0.005 0 0 0 0.005 0 0 -0.447998 0.005 0 0 0 0.005 0 0 -0.450439 0.005 0 0 0 0.005 0 0 -0.452881 0.005 0 0 0 0.005 0 0 -0.455322 0.005 0 0 0 0.005 0 0 -0.457764 0.005 0 0 0 0.005 0 0 -0.460205 0.005 0 0 0 0.005 0 0 -0.462646 0.005 0 0 0 0.005 0 0 -0.465088 0.005 0 0 0 0.005 0 0 -0.467529 0.005 0 0 0 0.005 0 0 -0.469971 0.005 0 0 0 0.005 0 0 -0.472412 0.005 0 0 0 0.005 0 0 -0.474854 0.005 0 0 0 0.005 0 0 -0.477295 0.005 0 0 0 0.005 0 0 -0.479736 0.005 0 0 0 0.005 0 0 -0.482178 0.005 0 0 0 0.005 0 0 -0.484619 0.005 0 0 0 0.005 0 0 -0.487061 0.005 0 0 0 0.005 0 0 -0.489502 0.005 0 0 0 0.005 0 0 -0.491943 0.005 0 0 0 0.005 0 0 -0.494385 0.005 0 0 0 0.005 0 0 -0.496826 0.005 0 0 0 0.005 0 0 -0.499268 0.005 0 0 0 0.005 0 0 -0.501709 0.005 0 0 0 0.005 0 0 -0.50415 0.005 0 0 0 0.005 0 0 -0.506592 0.005 0 0 0 0.005 0 0 -0.509033 0.005 0 0 0 0.005 0 0 -0.511475 0.005 0 0 0 0.005 0 0 -0.513916 0.005 0 0 0 0.005 0 0 -0.516357 0.005 0 0 0 0.005 0 0 -0.518799 0.005 0 0 0 0.005 0 0 -0.52124 0.005 0 0 0 0.005 0 0 -0.523682 0.005 0 0 0 0.005 0 0 -0.526123 0.005 0 0 0 0.005 0 0 -0.528564 0.005 0 0 0 0.005 0 0 -0.531006 0.005 0 0 0 0.005 0 0 -0.533447 0.005 0 0 0 0.005 0 0 -0.535889 0.005 0 0 0 0.005 0 0 -0.53833 0.005 0 0 0 0.005 0 0 -0.540771 0.005 0 0 0 0.005 0 0 -0.543213 0.005 0 0 0 0.005 0 0 -0.545654 0.005 0 0 0 0.005 0 0 -0.548096 0.005 0 0 0 0.005 0 0 -0.550537 0.005 0 0 0 0.005 0 0 -0.552979 0.005 0 0 0 0.005 0 0 -0.55542 0.005 0 0 0 0.005 0 0 -0.557861 0.005 0 0 0 0.005 0 0 -0.560303 0.005 0 0 0 0.005 0 0 -0.562744 0.005 0 0 0 0.005 0 0 -0.565186 0.005 0 0 0 0.005 0 0 -0.567627 0.005 0 0 0 0.005 0 0 -0.570068 0.005 0 0 0 0.005 0 0 -0.57251 0.005 0 0 0 0.005 0 0 -0.574951 0.005 0 0 0 0.005 0 0 -0.577393 0.005 0 0 0 0.005 0 0 -0.579834 0.005 0 0 0 0.005 0 0 -0.582275 0.005 0 0 0 0.005 0 0 -0.584717 0.005 0 0 0 0.005 0 0 -0.587158 0.005 0 0 0 0.005 0 0 -0.5896 0.005 0 0 0 0.005 0 0 -0.592041 0.005 0 0 0 0.005 0 0 -0.594482 0.005 0 0 0 0.005 0 0 -0.596924 0.005 0 0 0 0.005 0 0 -0.599365 0.005 0 0 0 0.005 0 0 -0.601807 0.005 0 0 0 0.005 0 0 -0.604248 0.005 0 0 0 0.005 0 0 -0.606689 0.005 0 0 0 0.005 0 0 -0.609131 0.005 0 0 0 0.005 0 0 -0.611572 0.005 0 0 0 0.005 0 0 -0.614014 0.005 0 0 0 0.005 0 0 -0.616455 0.005 0 0 0 0.005 0 0 -0.618896 0.005 0 0 0 0.005 0 0 -0.621338 0.005 0 0 0 0.005 0 0 -0.623779 0.005 0 0 0 0.005 0 0 -0.626221 0.005 0 0 0 0.005 0 0 -0.628662 0.005 0 0 0 0.005 0 0 -0.631104 0.005 0 0 0 0.005 0 0 -0.633545 0.005 0 0 0 0.005 0 0 -0.635986 0.005 0 0 0 0.005 0 0 -0.638428 0.005 0 0 0 0.005 0 0 -0.640869 0.005 0 0 0 0.005 0 0 -0.643311 0.005 0 0 0 0.005 0 0 -0.645752 0.005 0 0 0 0.005 0 0 -0.648193 0.005 0 0 0 0.005 0 0 -0.650635 0.005 0 0 0 0.005 0 0 -0.653076 0.005 0 0 0 0.005 0 0 -0.655518 0.005 0 0 0 0.005 0 0 -0.657959 0.005 0 0 0 0.005 0 0 -0.6604 0.005 0 0 0 0.005 0 0 -0.662842 0.005 0 0 0 0.005 0 0 -0.665283 0.005 0 0 0 0.005 0 0 -0.667725 0.005 0 0 0 0.005 0 0 -0.670166 0.005 0 0 0 0.005 0 0 -0.672607 0.005 0 0 0 0.005 0 0 -0.675049 0.005 0 0 0 0.005 0 0 -0.67749 0.005 0 0 0 0.005 0 0 -0.679932 0.005 0 0 0 0.005 0 0 -0.682373 0.005 0 0 0 0.005 0 0 -0.684814 0.005 0 0 0 0.005 0 0 -0.687256 0.005 0 0 0 0.005 0 0 -0.689697 0.005 0 0 0 0.005 0 0 -0.692139 0.005 0 0 0 0.005 0 0 -0.69458 0.005 0 0 0 0.005 0 0 -0.697021 0.005 0 0 0 0.005 0 0 -0.699463 0.005 0 0 0 0.005 0 0 -0.701904 0.005 0 0 0 0.005 0 0 -0.704346 0.005 0 0 0 0.005 0 0 -0.706787 0.005 0 0 0 0.005 0 0 -0.709229 0.005 0 0 0 0.005 0 0 -0.71167 0.005 0 0 0 0.005 0 0 -0.714111 0.005 0 0 0 0.005 0 0 -0.716553 0.005 0 0 0 0.005 0 0 -0.718994 0.005 0 0 0 0.005 0 0 -0.721436 0.005 0 0 0 0.005 0 0 -0.723877 0.005 0 0 0 0.005 0 0 -0.726318 0.005 0 0 0 0.005 0 0 -0.72876 0.005 0 0 0 0.005 0 0 -0.731201 0.005 0 0 0 0.005 0 0 -0.733643 0.005 0 0 0 0.005 0 0 -0.736084 0.005 0 0 0 0.005 0 0 -0.738525 0.005 0 0 0 0.005 0 0 -0.740967 0.005 0 0 0 0.005 0 0 -0.743408 0.005 0 0 0 0.005 0 0 -0.74585 0.005 0 0 0 0.005 0 0 -0.748291 0.005 0 0 0 0.005 0 0 -0.750732 0.005 0 0 0 0.005 0 0 -0.753174 0.005 0 0 0 0.005 0 0 -0.755615 0.005 0 0 0 0.005 0 0 -0.758057 0.005 0 0 0 0.005 0 0 -0.760498 0.005 0 0 0 0.005 0 0 -0.762939 0.005 0 0 0 0.005 0 0 -0.765381 0.005 0 0 0 0.005 0 0 -0.767822 0.005 0 0 0 0.005 0 0 -0.770264 0.005 0 0 0 0.005 0 0 -0.772705 0.005 0 0 0 0.005 0 0 -0.775146 0.005 0 0 0 0.005 0 0 -0.777588 0.005 0 0 0 0.005 0 0 -0.780029 0.005 0 0 0 0.005 0 0 -0.782471 0.005 0 0 0 0.005 0 0 -0.784912 0.005 0 0 0 0.005 0 0 -0.787354 0.005 0 0 0 0.005 0 0 -0.789795 0.005 0 0 0 0.005 0 0 -0.792236 0.005 0 0 0 0.005 0 0 -0.794678 0.005 0 0 0 0.005 0 0 -0.797119 0.005 0 0 0 0.005 0 0 -0.799561 0.005 0 0 0 0.005 0 0 -0.802002 0.005 0 0 0 0.005 0 0 -0.804443 0.005 0 0 0 0.005 0 0 -0.806885 0.005 0 0 0 0.005 0 0 -0.809326 0.005 0 0 0 0.005 0 0 -0.811768 0.005 0 0 0 0.005 0 0 -0.814209 0.005 0 0 0 0.005 0 0 -0.81665 0.005 0 0 0 0.005 0 0 -0.819092 0.005 0 0 0 0.005 0 0 -0.821533 0.005 0 0 0 0.005 0 0 -0.823975 0.005 0 0 0 0.005 0 0 -0.826416 0.005 0 0 0 0.005 0 0 -0.828857 0.005 0 0 0 0.005 0 0 -0.831299 0.005 0 0 0 0.005 0 0 -0.83374 0.005 0 0 0 0.005 0 0 -0.836182 0.005 0 0 0 0.005 0 0 -0.838623 0.005 0 0 0 0.005 0 0 -0.841064 0.005 0 0 0 0.005 0 0 -0.843506 0.005 0 0 0 0.005 0 0 -0.845947 0.005 0 0 0 0.005 0 0 -0.848389 0.005 0 0 0 0.005 0 0 -0.85083 0.005 0 0 0 0.005 0 0 -0.853271 0.005 0 0 0 0.005 0 0 -0.855713 0.005 0 0 0 0.005 0 0 -0.858154 0.005 0 0 0 0.005 0 0 -0.860596 0.005 0 0 0 0.005 0 0 -0.863037 0.005 0 0 0 0.005 0 0 -0.865479 0.005 0 0 0 0.005 0 0 -0.86792 0.005 0 0 0 0.005 0 0 -0.870361 0.005 0 0 0 0.005 0 0 -0.872803 0.005 0 0 0 0.005 0 0 -0.875244 0.005 0 0 0 0.005 0 0 -0.877686 0.005 0 0 0 0.005 0 0 -0.880127 0.005 0 0 0 0.005 0 0 -0.882568 0.005 0 0 0 0.005 0 0 -0.88501 0.005 0 0 0 0.005 0 0 -0.887451 0.005 0 0 0 0.005 0 0 -0.889893 0.005 0 0 0 0.005 0 0 -0.892334 0.005 0 0 0 0.005 0 0 -0.894775 0.005 0 0 0 0.005 0 0 -0.897217 0.005 0 0 0 0.005 0 0 -0.899658 0.005 0 0 0 0.005 0 0 -0.9021 0.005 0 0 0 0.005 0 0 -0.904541 0.005 0 0 0 0.005 0 0 -0.906982 0.005 0 0 0 0.005 0 0 -0.909424 0.005 0 0 0 0.005 0 0 -0.911865 0.005 0 0 0 0.005 0 0 -0.914307 0.005 0 0 0 0.005 0 0 -0.916748 0.005 0 0 0 0.005 0 0 -0.919189 0.005 0 0 0 0.005 0 0 -0.921631 0.005 0 0 0 0.005 0 0 -0.924072 0.005 0 0 0 0.005 0 0 -0.926514 0.005 0 0 0 0.005 0 0 -0.928955 0.005 0 0 0 0.005 0 0 -0.931396 0.005 0 0 0 0.005 0 0 -0.933838 0.005 0 0 0 0.005 0 0 -0.936279 0.005 0 0 0 0.005 0 0 -0.938721 0.005 0 0 0 0.005 0 0 -0.941162 0.005 0 0 0 0.005 0 0 -0.943604 0.005 0 0 0 0.005 0 0 -0.946045 0.005 0 0 0 0.005 0 0 -0.948486 0.005 0 0 0 0.005 0 0 -0.950928 0.005 0 0 0 0.005 0 0 -0.953369 0.005 0 0 0 0.005 0 0 -0.955811 0.005 0 0 0 0.005 0 0 -0.958252 0.005 0 0 0 0.005 0 0 -0.960693 0.005 0 0 0 0.005 0 0 -0.963135 0.005 0 0 0 0.005 0 0 -0.965576 0.005 0 0 0 0.005 0 0 -0.968018 0.005 0 0 0 0.005 0 0 -0.970459 0.005 0 0 0 0.005 0 0 -0.9729 0.005 0 0 0 0.005 0 0 -0.975342 0.005 0 0 0 0.005 0 0 -0.977783 0.005 0 0 0 0.005 0 0 -0.980225 0.005 0 0 0 0.005 0 0 -0.982666 0.005 0 0 0 0.005 0 0 -0.985107 0.005 0 0 0 0.005 0 0 -0.987549 0.005 0 0 0 0.005 0 0 -0.98999 0.005 0 0 0 0.005 0 0 -0.992432 0.005 0 0 0 0.005 0 0 -0.994873 0.005 0 0 0 0.005 0 0 -0.997314 0.005 0 0 0 0.005 0 0 -0.999756 0.005 0 0 0 0.005 0 0 -1.0022 0.005 0 0 0 0.005 0 0 -1.00464 0.005 0 0 0 0.005 0 0 -1.00708 0.005 0 0 0 0.005 0 0 -1.00952 0.005 0 0 0 0.005 0 0 -1.01196 0.005 0 0 0 0.005 0 0 -1.0144 0.005 0 0 0 0.005 0 0 -1.01685 0.005 0 0 0 0.005 0 0 -1.01929 0.005 0 0 0 0.005 0 0 -1.02173 0.005 0 0 0 0.005 0 0 -1.02417 0.005 0 0 0 0.005 0 0 -1.02661 0.005 0 0 0 0.005 0 0 -1.02905 0.005 0 0 0 0.005 0 0 -1.03149 0.005 0 0 0 0.005 0 0 -1.03394 0.005 0 0 0 0.005 0 0 -1.03638 0.005 0 0 0 0.005 0 0 -1.03882 0.005 0 0 0 0.005 0 0 -1.04126 0.005 0 0 0 0.005 0 0 -1.0437 0.005 0 0 0 0.005 0 0 -1.04614 0.005 0 0 0 0.005 0 0 -1.04858 0.005 0 0 0 0.005 0 0 -1.05103 0.005 0 0 0 0.005 0 0 -1.05347 0.005 0 0 0 0.005 0 0 -1.05591 0.005 0 0 0 0.005 0 0 -1.05835 0.005 0 0 0 0.005 0 0 -1.06079 0.005 0 0 0 0.005 0 0 -1.06323 0.005 0 0 0 0.005 0 0 -1.06567 0.005 0 0 0 0.005 0 0 -1.06812 0.005 0 0 0 0.005 0 0 -1.07056 0.005 0 0 0 0.005 0 0 -1.073 0.005 0 0 0 0.005 0 0 -1.07544 0.005 0 0 0 0.005 0 0 -1.07788 0.005 0 0 0 0.005 0 0 -1.08032 0.005 0 0 0 0.005 0 0 -1.08276 0.005 0 0 0 0.005 0 0 -1.08521 0.005 0 0 0 0.005 0 0 -1.08765 0.005 0 0 0 0.005 0 0 -1.09009 0.005 0 0 0 0.005 0 0 -1.09253 0.005 0 0 0 0.005 0 0 -1.09497 0.005 0 0 0 0.005 0 0 -1.09741 0.005 0 0 0 0.005 0 0 -1.09985 0.005 0 0 0 0.005 0 0 -1.10229 0.005 0 0 0 0.005 0 0 -1.10474 0.005 0 0 0 0.005 0 0 -1.10718 0.005 0 0 0 0.005 0 0 -1.10962 0.005 0 0 0 0.005 0 0 -1.11206 0.005 0 0 0 0.005 0 0 -1.1145 0.005 0 0 0 0.005 0 0 -1.11694 0.005 0 0 0 0.005 0 0 -1.11938 0.005 0 0 0 0.005 0 0 -1.12183 0.005 0 0 0 0.005 0 0 -1.12427 0.005 0 0 0 0.005 0 0 -1.12671 0.005 0 0 0 0.005 0 0 -1.12915 0.005 0 0 0 0.005 0 0 -1.13159 0.005 0 0 0 0.005 0 0 -1.13403 0.005 0 0 0 0.005 0 0 -1.13647 0.005 0 0 0 0.005 0 0 -1.13892 0.005 0 0 0 0.005 0 0 -1.14136 0.005 0 0 0 0.005 0 0 -1.1438 0.005 0 0 0 0.005 0 0 -1.14624 0.005 0 0 0 0.005 0 0 -1.14868 0.005 0 0 0 0.005 0 0 -1.15112 0.005 0 0 0 0.005 0 0 -1.15356 0.005 0 0 0 0.005 0 0 -1.15601 0.005 0 0 0 0.005 0 0 -1.15845 0.005 0 0 0 0.005 0 0 -1.16089 0.005 0 0 0 0.005 0 0 -1.16333 0.005 0 0 0 0.005 0 0 -1.16577 0.005 0 0 0 0.005 0 0 -1.16821 0.005 0 0 0 0.005 0 0 -1.17065 0.005 0 0 0 0.005 0 0 -1.1731 0.005 0 0 0 0.005 0 0 -1.17554 0.005 0 0 0 0.005 0 0 -1.17798 0.005 0 0 0 0.005 0 0 -1.18042 0.005 0 0 0 0.005 0 0 -1.18286 0.005 0 0 0 0.005 0 0 -1.1853 0.005 0 0 0 0.005 0 0 -1.18774 0.005 0 0 0 0.005 0 0 -1.19019 0.005 0 0 0 0.005 0 0 -1.19263 0.005 0 0 0 0.005 0 0 -1.19507 0.005 0 0 0 0.005 0 0 -1.19751 0.005 0 0 0 0.005 0 0 -1.19995 0.005 0 0 0 0.005 0 0 -1.20239 0.005 0 0 0 0.005 0 0 -1.20483 0.005 0 0 0 0.005 0 0 -1.20728 0.005 0 0 0 0.005 0 0 -1.20972 0.005 0 0 0 0.005 0 0 -1.21216 0.005 0 0 0 0.005 0 0 -1.2146 0.005 0 0 0 0.005 0 0 -1.21704 0.005 0 0 0 0.005 0 0 -1.21948 0.005 0 0 0 0.005 0 0 -1.22192 0.005 0 0 0 0.005 0 0 -1.22437 0.005 0 0 0 0.005 0 0 -1.22681 0.005 0 0 0 0.005 0 0 -1.22925 0.005 0 0 0 0.005 0 0 -1.23169 0.005 0 0 0 0.005 0 0 -1.23413 0.005 0 0 0 0.005 0 0 -1.23657 0.005 0 0 0 0.005 0 0 -1.23901 0.005 0 0 0 0.005 0 0 -1.24146 0.005 0 0 0 0.005 0 0 -1.2439 0.005 0 0 0 0.005 0 0 -1.24634 0.005 0 0 0 0.005 0 0 -1.24878 0.005 0 0 0 0.005 0 0 -1.25122 0.005 0 0 0 0.005 0 0 -1.25366 0.005 0 0 0 0.005 0 0 -1.2561 0.005 0 0 0 0.005 0 0 -1.25854 0.005 0 0 0 0.005 0 0 -1.26099 0.005 0 0 0 0.005 0 0 -1.26343 0.005 0 0 0 0.005 0 0 -1.26587 0.005 0 0 0 0.005 0 0 -1.26831 0.005 0 0 0 0.005 0 0 -1.27075 0.005 0 0 0 0.005 0 0 -1.27319 0.005 0 0 0 0.005 0 0 -1.27563 0.005 0 0 0 0.005 0 0 -1.27808 0.005 0 0 0 0.005 0 0 -1.28052 0.005 0 0 0 0.005 0 0 -1.28296 0.005 0 0 0 0.005 0 0 -1.2854 0.005 0 0 0 0.005 0 0 -1.28784 0.005 0 0 0 0.005 0 0 -1.29028 0.005 0 0 0 0.005 0 0 -1.29272 0.005 0 0 0 0.005 0 0 -1.29517 0.005 0 0 0 0.005 0 0 -1.29761 0.005 0 0 0 0.005 0 0 -1.30005 0.005 0 0 0 0.005 0 0 -1.30249 0.005 0 0 0 0.005 0 0 -1.30493 0.005 0 0 0 0.005 0 0 -1.30737 0.005 0 0 0 0.005 0 0 -1.30981 0.005 0 0 0 0.005 0 0 -1.31226 0.005 0 0 0 0.005 0 0 -1.3147 0.005 0 0 0 0.005 0 0 -1.31714 0.005 0 0 0 0.005 0 0 -1.31958 0.005 0 0 0 0.005 0 0 -1.32202 0.005 0 0 0 0.005 0 0 -1.32446 0.005 0 0 0 0.005 0 0 -1.3269 0.005 0 0 0 0.005 0 0 -1.32935 0.005 0 0 0 0.005 0 0 -1.33179 0.005 0 0 0 0.005 0 0 -1.33423 0.005 0 0 0 0.005 0 0 -1.33667 0.005 0 0 0 0.005 0 0 -1.33911 0.005 0 0 0 0.005 0 0 -1.34155 0.005 0 0 0 0.005 0 0 -1.34399 0.005 0 0 0 0.005 0 0 -1.34644 0.005 0 0 0 0.005 0 0 -1.34888 0.005 0 0 0 0.005 0 0 -1.35132 0.005 0 0 0 0.005 0 0 -1.35376 0.005 0 0 0 0.005 0 0 -1.3562 0.005 0 0 0 0.005 0 0 -1.35864 0.005 0 0 0 0.005 0 0 -1.36108 0.005 0 0 0 0.005 0 0 -1.36353 0.005 0 0 0 0.005 0 0 -1.36597 0.005 0 0 0 0.005 0 0 -1.36841 0.005 0 0 0 0.005 0 0 -1.37085 0.005 0 0 0 0.005 0 0 -1.37329 0.005 0 0 0 0.005 0 0 -1.37573 0.005 0 0 0 0.005 0 0 -1.37817 0.005 0 0 0 0.005 0 0 -1.38062 0.005 0 0 0 0.005 0 0 -1.38306 0.005 0 0 0 0.005 0 0 -1.3855 0.005 0 0 0 0.005 0 0 -1.38794 0.005 0 0 0 0.005 0 0 -1.39038 0.005 0 0 0 0.005 0 0 -1.39282 0.005 0 0 0 0.005 0 0 -1.39526 0.005 0 0 0 0.005 0 0 -1.39771 0.005 0 0 0 0.005 0 0 -1.40015 0.005 0 0 0 0.005 0 0 -1.40259 0.005 0 0 0 0.005 0 0 -1.40503 0.005 0 0 0 0.005 0 0 -1.40747 0.005 0 0 0 0.005 0 0 -1.40991 0.005 0 0 0 0.005 0 0 -1.41235 0.005 0 0 0 0.005 0 0 -1.41479 0.005 0 0 0 0.005 0 0 -1.41724 0.005 0 0 0 0.005 0 0 -1.41968 0.005 0 0 0 0.005 0 0 -1.42212 0.005 0 0 0 0.005 0 0 -1.42456 0.005 0 0 0 0.005 0 0 -1.427 0.005 0 0 0 0.005 0 0 -1.42944 0.005 0 0 0 0.005 0 0 -1.43188 0.005 0 0 0 0.005 0 0 -1.43433 0.005 0 0 0 0.005 0 0 -1.43677 0.005 0 0 0 0.005 0 0 -1.43921 0.005 0 0 0 0.005 0 0 -1.44165 0.005 0 0 0 0.005 0 0 -1.44409 0.005 0 0 0 0.005 0 0 -1.44653 0.005 0 0 0 0.005 0 0 -1.44897 0.005 0 0 0 0.005 0 0 -1.45142 0.005 0 0 0 0.005 0 0 -1.45386 0.005 0 0 0 0.005 0 0 -1.4563 0.005 0 0 0 0.005 0 0 -1.45874 0.005 0 0 0 0.005 0 0 -1.46118 0.005 0 0 0 0.005 0 0 -1.46362 0.005 0 0 0 0.005 0 0 -1.46606 0.005 0 0 0 0.005 0 0 -1.46851 0.005 0 0 0 0.005 0 0 -1.47095 0.005 0 0 0 0.005 0 0 -1.47339 0.005 0 0 0 0.005 0 0 -1.47583 0.005 0 0 0 0.005 0 0 -1.47827 0.005 0 0 0 0.005 0 0 -1.48071 0.005 0 0 0 0.005 0 0 -1.48315 0.005 0 0 0 0.005 0 0 -1.4856 0.005 0 0 0 0.005 0 0 -1.48804 0.005 0 0 0 0.005 0 0 -1.49048 0.005 0 0 0 0.005 0 0 -1.49292 0.005 0 0 0 0.005 0 0 -1.49536 0.005 0 0 0 0.005 0 0 -1.4978 0.005 0 0 0 0.005 0 0 -1.50024 0.005 0 0 0 0.005 0 0 -1.50269 0.005 0 0 0 0.005 0 0 -1.50513 0.005 0 0 0 0.005 0 0 -1.50757 0.005 0 0 0 0.005 0 0 -1.51001 0.005 0 0 0 0.005 0 0 -1.51245 0.005 0 0 0 0.005 0 0 -1.51489 0.005 0 0 0 0.005 0 0 -1.51733 0.005 0 0 0 0.005 0 0 -1.51978 0.005 0 0 0 0.005 0 0 -1.52222 0.005 0 0 0 0.005 0 0 -1.52466 0.005 0 0 0 0.005 0 0 -1.5271 0.005 0 0 0 0.005 0 0 -1.52954 0.005 0 0 0 0.005 0 0 -1.53198 0.005 0 0 0 0.005 0 0 -1.53442 0.005 0 0 0 0.005 0 0 -1.53687 0.005 0 0 0 0.005 0 0 -1.53931 0.005 0 0 0 0.005 0 0 -1.54175 0.005 0 0 0 0.005 0 0 -1.54419 0.005 0 0 0 0.005 0 0 -1.54663 0.005 0 0 0 0.005 0 0 -1.54907 0.005 0 0 0 0.005 0 0 -1.55151 0.005 0 0 0 0.005 0 0 -1.55396 0.005 0 0 0 0.005 0 0 -1.5564 0.005 0 0 0 0.005 0 0 -1.55884 0.005 0 0 0 0.005 0 0 -1.56128 0.005 0 0 0 0.005 0 0 -1.56372 0.005 0 0 0 0.005 0 0 -1.56616 0.005 0 0 0 0.005 0 0 -1.5686 0.005 0 0 0 0.005 0 0 -1.57104 0.005 0 0 0 0.005 0 0 -1.57349 0.005 0 0 0 0.005 0 0 -1.57593 0.005 0 0 0 0.005 0 0 -1.57837 0.005 0 0 0 0.005 0 0 -1.58081 0.005 0 0 0 0.005 0 0 -1.58325 0.005 0 0 0 0.005 0 0 -1.58569 0.005 0 0 0 0.005 0 0 -1.58813 0.005 0 0 0 0.005 0 0 -1.59058 0.005 0 0 0 0.005 0 0 -1.59302 0.005 0 0 0 0.005 0 0 -1.59546 0.005 0 0 0 0.005 0 0 -1.5979 0.005 0 0 0 0.005 0 0 -1.60034 0.005 0 0 0 0.005 0 0 -1.60278 0.005 0 0 0 0.005 0 0 -1.60522 0.005 0 0 0 0.005 0 0 -1.60767 0.005 0 0 0 0.005 0 0 -1.61011 0.005 0 0 0 0.005 0 0 -1.61255 0.005 0 0 0 0.005 0 0 -1.61499 0.005 0 0 0 0.005 0 0 -1.61743 0.005 0 0 0 0.005 0 0 -1.61987 0.005 0 0 0 0.005 0 0 -1.62231 0.005 0 0 0 0.005 0 0 -1.62476 0.005 0 0 0 0.005 0 0 -1.6272 0.005 0 0 0 0.005 0 0 -1.62964 0.005 0 0 0 0.005 0 0 -1.63208 0.005 0 0 0 0.005 0 0 -1.63452 0.005 0 0 0 0.005 0 0 -1.63696 0.005 0 0 0 0.005 0 0 -1.6394 0.005 0 0 0 0.005 0 0 -1.64185 0.005 0 0 0 0.005 0 0 -1.64429 0.005 0 0 0 0.005 0 0 -1.64673 0.005 0 0 0 0.005 0 0 -1.64917 0.005 0 0 0 0.005 0 0 -1.65161 0.005 0 0 0 0.005 0 0 -1.65405 0.005 0 0 0 0.005 0 0 -1.65649 0.005 0 0 0 0.005 0 0 -1.65894 0.005 0 0 0 0.005 0 0 -1.66138 0.005 0 0 0 0.005 0 0 -1.66382 0.005 0 0 0 0.005 0 0 -1.66626 0.005 0 0 0 0.005 0 0 -1.6687 0.005 0 0 0 0.005 0 0 -1.67114 0.005 0 0 0 0.005 0 0 -1.67358 0.005 0 0 0 0.005 0 0 -1.67603 0.005 0 0 0 0.005 0 0 -1.67847 0.005 0 0 0 0.005 0 0 -1.68091 0.005 0 0 0 0.005 0 0 -1.68335 0.005 0 0 0 0.005 0 0 -1.68579 0.005 0 0 0 0.005 0 0 -1.68823 0.005 0 0 0 0.005 0 0 -1.69067 0.005 0 0 0 0.005 0 0 -1.69312 0.005 0 0 0 0.005 0 0 -1.69556 0.005 0 0 0 0.005 0 0 -1.698 0.005 0 0 0 0.005 0 0 -1.70044 0.005 0 0 0 0.005 0 0 -1.70288 0.005 0 0 0 0.005 0 0 -1.70532 0.005 0 0 0 0.005 0 0 -1.70776 0.005 0 0 0 0.005 0 0 -1.71021 0.005 0 0 0 0.005 0 0 -1.71265 0.005 0 0 0 0.005 0 0 -1.71509 0.005 0 0 0 0.005 0 0 -1.71753 0.005 0 0 0 0.005 0 0 -1.71997 0.005 0 0 0 0.005 0 0 -1.72241 0.005 0 0 0 0.005 0 0 -1.72485 0.005 0 0 0 0.005 0 0 -1.72729 0.005 0 0 0 0.005 0 0 -1.72974 0.005 0 0 0 0.005 0 0 -1.73218 0.005 0 0 0 0.005 0 0 -1.73462 0.005 0 0 0 0.005 0 0 -1.73706 0.005 0 0 0 0.005 0 0 -1.7395 0.005 0 0 0 0.005 0 0 -1.74194 0.005 0 0 0 0.005 0 0 -1.74438 0.005 0 0 0 0.005 0 0 -1.74683 0.005 0 0 0 0.005 0 0 -1.74927 0.005 0 0 0 0.005 0 0 -1.75171 0.005 0 0 0 0.005 0 0 -1.75415 0.005 0 0 0 0.005 0 0 -1.75659 0.005 0 0 0 0.005 0 0 -1.75903 0.005 0 0 0 0.005 0 0 -1.76147 0.005 0 0 0 0.005 0 0 -1.76392 0.005 0 0 0 0.005 0 0 -1.76636 0.005 0 0 0 0.005 0 0 -1.7688 0.005 0 0 0 0.005 0 0 -1.77124 0.005 0 0 0 0.005 0 0 -1.77368 0.005 0 0 0 0.005 0 0 -1.77612 0.005 0 0 0 0.005 0 0 -1.77856 0.005 0 0 0 0.005 0 0 -1.78101 0.005 0 0 0 0.005 0 0 -1.78345 0.005 0 0 0 0.005 0 0 -1.78589 0.005 0 0 0 0.005 0 0 -1.78833 0.005 0 0 0 0.005 0 0 -1.79077 0.005 0 0 0 0.005 0 0 -1.79321 0.005 0 0 0 0.005 0 0 -1.79565 0.005 0 0 0 0.005 0 0 -1.7981 0.005 0 0 0 0.005 0 0 -1.80054 0.005 0 0 0 0.005 0 0 -1.80298 0.005 0 0 0 0.005 0 0 -1.80542 0.005 0 0 0 0.005 0 0 -1.80786 0.005 0 0 0 0.005 0 0 -1.8103 0.005 0 0 0 0.005 0 0 -1.81274 0.005 0 0 0 0.005 0 0 -1.81519 0.005 0 0 0 0.005 0 0 -1.81763 0.005 0 0 0 0.005 0 0 -1.82007 0.005 0 0 0 0.005 0 0 -1.82251 0.005 0 0 0 0.005 0 0 -1.82495 0.005 0 0 0 0.005 0 0 -1.82739 0.005 0 0 0 0.005 0 0 -1.82983 0.005 0 0 0 0.005 0 0 -1.83228 0.005 0 0 0 0.005 0 0 -1.83472 0.005 0 0 0 0.005 0 0 -1.83716 0.005 0 0 0 0.005 0 0 -1.8396 0.005 0 0 0 0.005 0 0 -1.84204 0.005 0 0 0 0.005 0 0 -1.84448 0.005 0 0 0 0.005 0 0 -1.84692 0.005 0 0 0 0.005 0 0 -1.84937 0.005 0 0 0 0.005 0 0 -1.85181 0.005 0 0 0 0.005 0 0 -1.85425 0.005 0 0 0 0.005 0 0 -1.85669 0.005 0 0 0 0.005 0 0 -1.85913 0.005 0 0 0 0.005 0 0 -1.86157 0.005 0 0 0 0.005 0 0 -1.86401 0.005 0 0 0 0.005 0 0 -1.86646 0.005 0 0 0 0.005 0 0 -1.8689 0.005 0 0 0 0.005 0 0 -1.87134 0.005 0 0 0 0.005 0 0 -1.87378 0.005 0 0 0 0.005 0 0 -1.87622 0.005 0 0 0 0.005 0 0 -1.87866 0.005 0 0 0 0.005 0 0 -1.8811 0.005 0 0 0 0.005 0 0 -1.88354 0.005 0 0 0 0.005 0 0 -1.88599 0.005 0 0 0 0.005 0 0 -1.88843 0.005 0 0 0 0.005 0 0 -1.89087 0.005 0 0 0 0.005 0 0 -1.89331 0.005 0 0 0 0.005 0 0 -1.89575 0.005 0 0 0 0.005 0 0 -1.89819 0.005 0 0 0 0.005 0 0 -1.90063 0.005 0 0 0 0.005 0 0 -1.90308 0.005 0 0 0 0.005 0 0 -1.90552 0.005 0 0 0 0.005 0 0 -1.90796 0.005 0 0 0 0.005 0 0 -1.9104 0.005 0 0 0 0.005 0 0 -1.91284 0.005 0 0 0 0.005 0 0 -1.91528 0.005 0 0 0 0.005 0 0 -1.91772 0.005 0 0 0 0.005 0 0 -1.92017 0.005 0 0 0 0.005 0 0 -1.92261 0.005 0 0 0 0.005 0 0 -1.92505 0.005 0 0 0 0.005 0 0 -1.92749 0.005 0 0 0 0.005 0 0 -1.92993 0.005 0 0 0 0.005 0 0 -1.93237 0.005 0 0 0 0.005 0 0 -1.93481 0.005 0 0 0 0.005 0 0 -1.93726 0.005 0 0 0 0.005 0 0 -1.9397 0.005 0 0 0 0.005 0 0 -1.94214 0.005 0 0 0 0.005 0 0 -1.94458 0.005 0 0 0 0.005 0 0 -1.94702 0.005 0 0 0 0.005 0 0 -1.94946 0.005 0 0 0 0.005 0 0 -1.9519 0.005 0 0 0 0.005 0 0 -1.95435 0.005 0 0 0 0.005 0 0 -1.95679 0.005 0 0 0 0.005 0 0 -1.95923 0.005 0 0 0 0.005 0 0 -1.96167 0.005 0 0 0 0.005 0 0 -1.96411 0.005 0 0 0 0.005 0 0 -1.96655 0.005 0 0 0 0.005 0 0 -1.96899 0.005 0 0 0 0.005 0 0 -1.97144 0.005 0 0 0 0.005 0 0 -1.97388 0.005 0 0 0 0.005 0 0 -1.97632 0.005 0 0 0 0.005 0 0 -1.97876 0.005 0 0 0 0.005 0 0 -1.9812 0.005 0 0 0 0.005 0 0 -1.98364 0.005 0 0 0 0.005 0 0 -1.98608 0.005 0 0 0 0.005 0 0 -1.98853 0.005 0 0 0 0.005 0 0 -1.99097 0.005 0 0 0 0.005 0 0 -1.99341 0.005 0 0 0 0.005 0 0 -1.99585 0.005 0 0 0 0.005 0 0 -1.99829 0.005 0 0 0 0.005 0 0 -2.00073 0.005 0 0 0 0.005 0 0 -2.00317 0.005 0 0 0 0.005 0 0 -2.00562 0.005 0 0 0 0.005 0 0 -2.00806 0.005 0 0 0 0.005 0 0 -2.0105 0.005 0 0 0 0.005 0 0 -2.01294 0.005 0 0 0 0.005 0 0 -2.01538 0.005 0 0 0 0.005 0 0 -2.01782 0.005 0 0 0 0.005 0 0 -2.02026 0.005 0 0 0 0.005 0 0 -2.02271 0.005 0 0 0 0.005 0 0 -2.02515 0.005 0 0 0 0.005 0 0 -2.02759 0.005 0 0 0 0.005 0 0 -2.03003 0.005 0 0 0 0.005 0 0 -2.03247 0.005 0 0 0 0.005 0 0 -2.03491 0.005 0 0 0 0.005 0 0 -2.03735 0.005 0 0 0 0.005 0 0 -2.03979 0.005 0 0 0 0.005 0 0 -2.04224 0.005 0 0 0 0.005 0 0 -2.04468 0.005 0 0 0 0.005 0 0 -2.04712 0.005 0 0 0 0.005 0 0 -2.04956 0.005 0 0 0 0.005 0 0 -2.052 0.005 0 0 0 0.005 0 0 -2.05444 0.005 0 0 0 0.005 0 0 -2.05688 0.005 0 0 0 0.005 0 0 -2.05933 0.005 0 0 0 0.005 0 0 -2.06177 0.005 0 0 0 0.005 0 0 -2.06421 0.005 0 0 0 0.005 0 0 -2.06665 0.005 0 0 0 0.005 0 0 -2.06909 0.005 0 0 0 0.005 0 0 -2.07153 0.005 0 0 0 0.005 0 0 -2.07397 0.005 0 0 0 0.005 0 0 -2.07642 0.005 0 0 0 0.005 0 0 -2.07886 0.005 0 0 0 0.005 0 0 -2.0813 0.005 0 0 0 0.005 0 0 -2.08374 0.005 0 0 0 0.005 0 0 -2.08618 0.005 0 0 0 0.005 0 0 -2.08862 0.005 0 0 0 0.005 0 0 -2.09106 0.005 0 0 0 0.005 0 0 -2.09351 0.005 0 0 0 0.005 0 0 -2.09595 0.005 0 0 0 0.005 0 0 -2.09839 0.005 0 0 0 0.005 0 0 -2.10083 0.005 0 0 0 0.005 0 0 -2.10327 0.005 0 0 0 0.005 0 0 -2.10571 0.005 0 0 0 0.005 0 0 -2.10815 0.005 0 0 0 0.005 0 0 -2.1106 0.005 0 0 0 0.005 0 0 -2.11304 0.005 0 0 0 0.005 0 0 -2.11548 0.005 0 0 0 0.005 0 0 -2.11792 0.005 0 0 0 0.005 0 0 -2.12036 0.005 0 0 0 0.005 0 0 -2.1228 0.005 0 0 0 0.005 0 0 -2.12524 0.005 0 0 0 0.005 0 0 -2.12769 0.005 0 0 0 0.005 0 0 -2.13013 0.005 0 0 0 0.005 0 0 -2.13257 0.005 0 0 0 0.005 0 0 -2.13501 0.005 0 0 0 0.005 0 0 -2.13745 0.005 0 0 0 0.005 0 0 -2.13989 0.005 0 0 0 0.005 0 0 -2.14233 0.005 0 0 0 0.005 0 0 -2.14478 0.005 0 0 0 0.005 0 0 -2.14722 0.005 0 0 0 0.005 0 0 -2.14966 0.005 0 0 0 0.005 0 0 -2.1521 0.005 0 0 0 0.005 0 0 -2.15454 0.005 0 0 0 0.005 0 0 -2.15698 0.005 0 0 0 0.005 0 0 -2.15942 0.005 0 0 0 0.005 0 0 -2.16187 0.005 0 0 0 0.005 0 0 -2.16431 0.005 0 0 0 0.005 0 0 -2.16675 0.005 0 0 0 0.005 0 0 -2.16919 0.005 0 0 0 0.005 0 0 -2.17163 0.005 0 0 0 0.005 0 0 -2.17407 0.005 0 0 0 0.005 0 0 -2.17651 0.005 0 0 0 0.005 0 0 -2.17896 0.005 0 0 0 0.005 0 0 -2.1814 0.005 0 0 0 0.005 0 0 -2.18384 0.005 0 0 0 0.005 0 0 -2.18628 0.005 0 0 0 0.005 0 0 -2.18872 0.005 0 0 0 0.005 0 0 -2.19116 0.005 0 0 0 0.005 0 0 -2.1936 0.005 0 0 0 0.005 0 0 -2.19604 0.005 0 0 0 0.005 0 0 -2.19849 0.005 0 0 0 0.005 0 0 -2.20093 0.005 0 0 0 0.005 0 0 -2.20337 0.005 0 0 0 0.005 0 0 -2.20581 0.005 0 0 0 0.005 0 0 -2.20825 0.005 0 0 0 0.005 0 0 -2.21069 0.005 0 0 0 0.005 0 0 -2.21313 0.005 0 0 0 0.005 0 0 -2.21558 0.005 0 0 0 0.005 0 0 -2.21802 0.005 0 0 0 0.005 0 0 -2.22046 0.005 0 0 0 0.005 0 0 -2.2229 0.005 0 0 0 0.005 0 0 -2.22534 0.005 0 0 0 0.005 0 0 -2.22778 0.005 0 0 0 0.005 0 0 -2.23022 0.005 0 0 0 0.005 0 0 -2.23267 0.005 0 0 0 0.005 0 0 -2.23511 0.005 0 0 0 0.005 0 0 -2.23755 0.005 0 0 0 0.005 0 0 -2.23999 0.005 0 0 0 0.005 0 0 -2.24243 0.005 0 0 0 0.005 0 0 -2.24487 0.005 0 0 0 0.005 0 0 -2.24731 0.005 0 0 0 0.005 0 0 -2.24976 0.005 0 0 0 0.005 0 0 -2.2522 0.005 0 0 0 0.005 0 0 -2.25464 0.005 0 0 0 0.005 0 0 -2.25708 0.005 0 0 0 0.005 0 0 -2.25952 0.005 0 0 0 0.005 0 0 -2.26196 0.005 0 0 0 0.005 0 0 -2.2644 0.005 0 0 0 0.005 0 0 -2.26685 0.005 0 0 0 0.005 0 0 -2.26929 0.005 0 0 0 0.005 0 0 -2.27173 0.005 0 0 0 0.005 0 0 -2.27417 0.005 0 0 0 0.005 0 0 -2.27661 0.005 0 0 0 0.005 0 0 -2.27905 0.005 0 0 0 0.005 0 0 -2.28149 0.005 0 0 0 0.005 0 0 -2.28394 0.005 0 0 0 0.005 0 0 -2.28638 0.005 0 0 0 0.005 0 0 -2.28882 0.005 0 0 0 0.005 0 0 -2.29126 0.005 0 0 0 0.005 0 0 -2.2937 0.005 0 0 0 0.005 0 0 -2.29614 0.005 0 0 0 0.005 0 0 -2.29858 0.005 0 0 0 0.005 0 0 -2.30103 0.005 0 0 0 0.005 0 0 -2.30347 0.005 0 0 0 0.005 0 0 -2.30591 0.005 0 0 0 0.005 0 0 -2.30835 0.005 0 0 0 0.005 0 0 -2.31079 0.005 0 0 0 0.005 0 0 -2.31323 0.005 0 0 0 0.005 0 0 -2.31567 0.005 0 0 0 0.005 0 0 -2.31812 0.005 0 0 0 0.005 0 0 -2.32056 0.005 0 0 0 0.005 0 0 -2.323 0.005 0 0 0 0.005 0 0 -2.32544 0.005 0 0 0 0.005 0 0 -2.32788 0.005 0 0 0 0.005 0 0 -2.33032 0.005 0 0 0 0.005 0 0 -2.33276 0.005 0 0 0 0.005 0 0 -2.33521 0.005 0 0 0 0.005 0 0 -2.33765 0.005 0 0 0 0.005 0 0 -2.34009 0.005 0 0 0 0.005 0 0 -2.34253 0.005 0 0 0 0.005 0 0 -2.34497 0.005 0 0 0 0.005 0 0 -2.34741 0.005 0 0 0 0.005 0 0 -2.34985 0.005 0 0 0 0.005 0 0 -2.35229 0.005 0 0 0 0.005 0 0 -2.35474 0.005 0 0 0 0.005 0 0 -2.35718 0.005 0 0 0 0.005 0 0 -2.35962 0.005 0 0 0 0.005 0 0 -2.36206 0.005 0 0 0 0.005 0 0 -2.3645 0.005 0 0 0 0.005 0 0 -2.36694 0.005 0 0 0 0.005 0 0 -2.36938 0.005 0 0 0 0.005 0 0 -2.37183 0.005 0 0 0 0.005 0 0 -2.37427 0.005 0 0 0 0.005 0 0 -2.37671 0.005 0 0 0 0.005 0 0 -2.37915 0.005 0 0 0 0.005 0 0 -2.38159 0.005 0 0 0 0.005 0 0 -2.38403 0.005 0 0 0 0.005 0 0 -2.38647 0.005 0 0 0 0.005 0 0 -2.38892 0.005 0 0 0 0.005 0 0 -2.39136 0.005 0 0 0 0.005 0 0 -2.3938 0.005 0 0 0 0.005 0 0 -2.39624 0.005 0 0 0 0.005 0 0 -2.39868 0.005 0 0 0 0.005 0 0 -2.40112 0.005 0 0 0 0.005 0 0 -2.40356 0.005 0 0 0 0.005 0 0 -2.40601 0.005 0 0 0 0.005 0 0 -2.40845 0.005 0 0 0 0.005 0 0 -2.41089 0.005 0 0 0 0.005 0 0 -2.41333 0.005 0 0 0 0.005 0 0 -2.41577 0.005 0 0 0 0.005 0 0 -2.41821 0.005 0 0 0 0.005 0 0 -2.42065 0.005 0 0 0 0.005 0 0 -2.4231 0.005 0 0 0 0.005 0 0 -2.42554 0.005 0 0 0 0.005 0 0 -2.42798 0.005 0 0 0 0.005 0 0 -2.43042 0.005 0 0 0 0.005 0 0 -2.43286 0.005 0 0 0 0.005 0 0 -2.4353 0.005 0 0 0 0.005 0 0 -2.43774 0.005 0 0 0 0.005 0 0 -2.44019 0.005 0 0 0 0.005 0 0 -2.44263 0.005 0 0 0 0.005 0 0 -2.44507 0.005 0 0 0 0.005 0 0 -2.44751 0.005 0 0 0 0.005 0 0 -2.44995 0.005 0 0 0 0.005 0 0 -2.45239 0.005 0 0 0 0.005 0 0 -2.45483 0.005 0 0 0 0.005 0 0 -2.45728 0.005 0 0 0 0.005 0 0 -2.45972 0.005 0 0 0 0.005 0 0 -2.46216 0.005 0 0 0 0.005 0 0 -2.4646 0.005 0 0 0 0.005 0 0 -2.46704 0.005 0 0 0 0.005 0 0 -2.46948 0.005 0 0 0 0.005 0 0 -2.47192 0.005 0 0 0 0.005 0 0 -2.47437 0.005 0 0 0 0.005 0 0 -2.47681 0.005 0 0 0 0.005 0 0 -2.47925 0.005 0 0 0 0.005 0 0 -2.48169 0.005 0 0 0 0.005 0 0 -2.48413 0.005 0 0 0 0.005 0 0 -2.48657 0.005 0 0 0 0.005 0 0 -2.48901 0.005 0 0 0 0.005 0 0 -2.49146 0.005 0 0 0 0.005 0 0 -2.4939 0.005 0 0 0 0.005 0 0 -2.49634 0.005 0 0 0 0.005 0 0 -2.49878 0.005 0 0 0 0.005 0 0 -2.50122 0.005 0 0 0 0.005 0 0 -2.50366 0.005 0 0 0 0.005 0 0 -2.5061 0.005 0 0 0 0.005 0 0 -2.50854 0.005 0 0 0 0.005 0 0 -2.51099 0.005 0 0 0 0.005 0 0 -2.51343 0.005 0 0 0 0.005 0 0 -2.51587 0.005 0 0 0 0.005 0 0 -2.51831 0.005 0 0 0 0.005 0 0 -2.52075 0.005 0 0 0 0.005 0 0 -2.52319 0.005 0 0 0 0.005 0 0 -2.52563 0.005 0 0 0 0.005 0 0 -2.52808 0.005 0 0 0 0.005 0 0 -2.53052 0.005 0 0 0 0.005 0 0 -2.53296 0.005 0 0 0 0.005 0 0 -2.5354 0.005 0 0 0 0.005 0 0 -2.53784 0.005 0 0 0 0.005 0 0 -2.54028 0.005 0 0 0 0.005 0 0 -2.54272 0.005 0 0 0 0.005 0 0 -2.54517 0.005 0 0 0 0.005 0 0 -2.54761 0.005 0 0 0 0.005 0 0 -2.55005 0.005 0 0 0 0.005 0 0 -2.55249 0.005 0 0 0 0.005 0 0 -2.55493 0.005 0 0 0 0.005 0 0 -2.55737 0.005 0 0 0 0.005 0 0 -2.55981 0.005 0 0 0 0.005 0 0 -2.56226 0.005 0 0 0 0.005 0 0 -2.5647 0.005 0 0 0 0.005 0 0 -2.56714 0.005 0 0 0 0.005 0 0 -2.56958 0.005 0 0 0 0.005 0 0 -2.57202 0.005 0 0 0 0.005 0 0 -2.57446 0.005 0 0 0 0.005 0 0 -2.5769 0.005 0 0 0 0.005 0 0 -2.57935 0.005 0 0 0 0.005 0 0 -2.58179 0.005 0 0 0 0.005 0 0 -2.58423 0.005 0 0 0 0.005 0 0 -2.58667 0.005 0 0 0 0.005 0 0 -2.58911 0.005 0 0 0 0.005 0 0 -2.59155 0.005 0 0 0 0.005 0 0 -2.59399 0.005 0 0 0 0.005 0 0 -2.59644 0.005 0 0 0 0.005 0 0 -2.59888 0.005 0 0 0 0.005 0 0 -2.60132 0.005 0 0 0 0.005 0 0 -2.60376 0.005 0 0 0 0.005 0 0 -2.6062 0.005 0 0 0 0.005 0 0 -2.60864 0.005 0 0 0 0.005 0 0 -2.61108 0.005 0 0 0 0.005 0 0 -2.61353 0.005 0 0 0 0.005 0 0 -2.61597 0.005 0 0 0 0.005 0 0 -2.61841 0.005 0 0 0 0.005 0 0 -2.62085 0.005 0 0 0 0.005 0 0 -2.62329 0.005 0 0 0 0.005 0 0 -2.62573 0.005 0 0 0 0.005 0 0 -2.62817 0.005 0 0 0 0.005 0 0 -2.63062 0.005 0 0 0 0.005 0 0 -2.63306 0.005 0 0 0 0.005 0 0 -2.6355 0.005 0 0 0 0.005 0 0 -2.63794 0.005 0 0 0 0.005 0 0 -2.64038 0.005 0 0 0 0.005 0 0 -2.64282 0.005 0 0 0 0.005 0 0 -2.64526 0.005 0 0 0 0.005 0 0 -2.64771 0.005 0 0 0 0.005 0 0 -2.65015 0.005 0 0 0 0.005 0 0 -2.65259 0.005 0 0 0 0.005 0 0 -2.65503 0.005 0 0 0 0.005 0 0 -2.65747 0.005 0 0 0 0.005 0 0 -2.65991 0.005 0 0 0 0.005 0 0 -2.66235 0.005 0 0 0 0.005 0 0 -2.66479 0.005 0 0 0 0.005 0 0 -2.66724 0.005 0 0 0 0.005 0 0 -2.66968 0.005 0 0 0 0.005 0 0 -2.67212 0.005 0 0 0 0.005 0 0 -2.67456 0.005 0 0 0 0.005 0 0 -2.677 0.005 0 0 0 0.005 0 0 -2.67944 0.005 0 0 0 0.005 0 0 -2.68188 0.005 0 0 0 0.005 0 0 -2.68433 0.005 0 0 0 0.005 0 0 -2.68677 0.005 0 0 0 0.005 0 0 -2.68921 0.005 0 0 0 0.005 0 0 -2.69165 0.005 0 0 0 0.005 0 0 -2.69409 0.005 0 0 0 0.005 0 0 -2.69653 0.005 0 0 0 0.005 0 0 -2.69897 0.005 0 0 0 0.005 0 0 -2.70142 0.005 0 0 0 0.005 0 0 -2.70386 0.005 0 0 0 0.005 0 0 -2.7063 0.005 0 0 0 0.005 0 0 -2.70874 0.005 0 0 0 0.005 0 0 -2.71118 0.005 0 0 0 0.005 0 0 -2.71362 0.005 0 0 0 0.005 0 0 -2.71606 0.005 0 0 0 0.005 0 0 -2.71851 0.005 0 0 0 0.005 0 0 -2.72095 0.005 0 0 0 0.005 0 0 -2.72339 0.005 0 0 0 0.005 0 0 -2.72583 0.005 0 0 0 0.005 0 0 -2.72827 0.005 0 0 0 0.005 0 0 -2.73071 0.005 0 0 0 0.005 0 0 -2.73315 0.005 0 0 0 0.005 0 0 -2.7356 0.005 0 0 0 0.005 0 0 -2.73804 0.005 0 0 0 0.005 0 0 -2.74048 0.005 0 0 0 0.005 0 0 -2.74292 0.005 0 0 0 0.005 0 0 -2.74536 0.005 0 0 0 0.005 0 0 -2.7478 0.005 0 0 0 0.005 0 0 -2.75024 0.005 0 0 0 0.005 0 0 -2.75269 0.005 0 0 0 0.005 0 0 -2.75513 0.005 0 0 0 0.005 0 0 -2.75757 0.005 0 0 0 0.005 0 0 -2.76001 0.005 0 0 0 0.005 0 0 -2.76245 0.005 0 0 0 0.005 0 0 -2.76489 0.005 0 0 0 0.005 0 0 -2.76733 0.005 0 0 0 0.005 0 0 -2.76978 0.005 0 0 0 0.005 0 0 -2.77222 0.005 0 0 0 0.005 0 0 -2.77466 0.005 0 0 0 0.005 0 0 -2.7771 0.005 0 0 0 0.005 0 0 -2.77954 0.005 0 0 0 0.005 0 0 -2.78198 0.005 0 0 0 0.005 0 0 -2.78442 0.005 0 0 0 0.005 0 0 -2.78687 0.005 0 0 0 0.005 0 0 -2.78931 0.005 0 0 0 0.005 0 0 -2.79175 0.005 0 0 0 0.005 0 0 -2.79419 0.005 0 0 0 0.005 0 0 -2.79663 0.005 0 0 0 0.005 0 0 -2.79907 0.005 0 0 0 0.005 0 0 -2.80151 0.005 0 0 0 0.005 0 0 -2.80396 0.005 0 0 0 0.005 0 0 -2.8064 0.005 0 0 0 0.005 0 0 -2.80884 0.005 0 0 0 0.005 0 0 -2.81128 0.005 0 0 0 0.005 0 0 -2.81372 0.005 0 0 0 0.005 0 0 -2.81616 0.005 0 0 0 0.005 0 0 -2.8186 0.005 0 0 0 0.005 0 0 -2.82104 0.005 0 0 0 0.005 0 0 -2.82349 0.005 0 0 0 0.005 0 0 -2.82593 0.005 0 0 0 0.005 0 0 -2.82837 0.005 0 0 0 0.005 0 0 -2.83081 0.005 0 0 0 0.005 0 0 -2.83325 0.005 0 0 0 0.005 0 0 -2.83569 0.005 0 0 0 0.005 0 0 -2.83813 0.005 0 0 0 0.005 0 0 -2.84058 0.005 0 0 0 0.005 0 0 -2.84302 0.005 0 0 0 0.005 0 0 -2.84546 0.005 0 0 0 0.005 0 0 -2.8479 0.005 0 0 0 0.005 0 0 -2.85034 0.005 0 0 0 0.005 0 0 -2.85278 0.005 0 0 0 0.005 0 0 -2.85522 0.005 0 0 0 0.005 0 0 -2.85767 0.005 0 0 0 0.005 0 0 -2.86011 0.005 0 0 0 0.005 0 0 -2.86255 0.005 0 0 0 0.005 0 0 -2.86499 0.005 0 0 0 0.005 0 0 -2.86743 0.005 0 0 0 0.005 0 0 -2.86987 0.005 0 0 0 0.005 0 0 -2.87231 0.005 0 0 0 0.005 0 0 -2.87476 0.005 0 0 0 0.005 0 0 -2.8772 0.005 0 0 0 0.005 0 0 -2.87964 0.005 0 0 0 0.005 0 0 -2.88208 0.005 0 0 0 0.005 0 0 -2.88452 0.005 0 0 0 0.005 0 0 -2.88696 0.005 0 0 0 0.005 0 0 -2.8894 0.005 0 0 0 0.005 0 0 -2.89185 0.005 0 0 0 0.005 0 0 -2.89429 0.005 0 0 0 0.005 0 0 -2.89673 0.005 0 0 0 0.005 0 0 -2.89917 0.005 0 0 0 0.005 0 0 -2.90161 0.005 0 0 0 0.005 0 0 -2.90405 0.005 0 0 0 0.005 0 0 -2.90649 0.005 0 0 0 0.005 0 0 -2.90894 0.005 0 0 0 0.005 0 0 -2.91138 0.005 0 0 0 0.005 0 0 -2.91382 0.005 0 0 0 0.005 0 0 -2.91626 0.005 0 0 0 0.005 0 0 -2.9187 0.005 0 0 0 0.005 0 0 -2.92114 0.005 0 0 0 0.005 0 0 -2.92358 0.005 0 0 0 0.005 0 0 -2.92603 0.005 0 0 0 0.005 0 0 -2.92847 0.005 0 0 0 0.005 0 0 -2.93091 0.005 0 0 0 0.005 0 0 -2.93335 0.005 0 0 0 0.005 0 0 -2.93579 0.005 0 0 0 0.005 0 0 -2.93823 0.005 0 0 0 0.005 0 0 -2.94067 0.005 0 0 0 0.005 0 0 -2.94312 0.005 0 0 0 0.005 0 0 -2.94556 0.005 0 0 0 0.005 0 0 -2.948 0.005 0 0 0 0.005 0 0 -2.95044 0.005 0 0 0 0.005 0 0 -2.95288 0.005 0 0 0 0.005 0 0 -2.95532 0.005 0 0 0 0.005 0 0 -2.95776 0.005 0 0 0 0.005 0 0 -2.96021 0.005 0 0 0 0.005 0 0 -2.96265 0.005 0 0 0 0.005 0 0 -2.96509 0.005 0 0 0 0.005 0 0 -2.96753 0.005 0 0 0 0.005 0 0 -2.96997 0.005 0 0 0 0.005 0 0 -2.97241 0.005 0 0 0 0.005 0 0 -2.97485 0.005 0 0 0 0.005 0 0 -2.97729 0.005 0 0 0 0.005 0 0 -2.97974 0.005 0 0 0 0.005 0 0 -2.98218 0.005 0 0 0 0.005 0 0 -2.98462 0.005 0 0 0 0.005 0 0 -2.98706 0.005 0 0 0 0.005 0 0 -2.9895 0.005 0 0 0 0.005 0 0 -2.99194 0.005 0 0 0 0.005 0 0 -2.99438 0.005 0 0 0 0.005 0 0 -2.99683 0.005 0 0 0 0.005 0 0 -2.99927 0.005 0 0 0 0.005 0 0 -3.00171 0.005 0 0 0 0.005 0 0 -3.00415 0.005 0 0 0 0.005 0 0 -3.00659 0.005 0 0 0 0.005 0 0 -3.00903 0.005 0 0 0 0.005 0 0 -3.01147 0.005 0 0 0 0.005 0 0 -3.01392 0.005 0 0 0 0.005 0 0 -3.01636 0.005 0 0 0 0.005 0 0 -3.0188 0.005 0 0 0 0.005 0 0 -3.02124 0.005 0 0 0 0.005 0 0 -3.02368 0.005 0 0 0 0.005 0 0 -3.02612 0.005 0 0 0 0.005 0 0 -3.02856 0.005 0 0 0 0.005 0 0 -3.03101 0.005 0 0 0 0.005 0 0 -3.03345 0.005 0 0 0 0.005 0 0 -3.03589 0.005 0 0 0 0.005 0 0 -3.03833 0.005 0 0 0 0.005 0 0 -3.04077 0.005 0 0 0 0.005 0 0 -3.04321 0.005 0 0 0 0.005 0 0 -3.04565 0.005 0 0 0 0.005 0 0 -3.0481 0.005 0 0 0 0.005 0 0 -3.05054 0.005 0 0 0 0.005 0 0 -3.05298 0.005 0 0 0 0.005 0 0 -3.05542 0.005 0 0 0 0.005 0 0 -3.05786 0.005 0 0 0 0.005 0 0 -3.0603 0.005 0 0 0 0.005 0 0 -3.06274 0.005 0 0 0 0.005 0 0 -3.06519 0.005 0 0 0 0.005 0 0 -3.06763 0.005 0 0 0 0.005 0 0 -3.07007 0.005 0 0 0 0.005 0 0 -3.07251 0.005 0 0 0 0.005 0 0 -3.07495 0.005 0 0 0 0.005 0 0 -3.07739 0.005 0 0 0 0.005 0 0 -3.07983 0.005 0 0 0 0.005 0 0 -3.08228 0.005 0 0 0 0.005 0 0 -3.08472 0.005 0 0 0 0.005 0 0 -3.08716 0.005 0 0 0 0.005 0 0 -3.0896 0.005 0 0 0 0.005 0 0 -3.09204 0.005 0 0 0 0.005 0 0 -3.09448 0.005 0 0 0 0.005 0 0 -3.09692 0.005 0 0 0 0.005 0 0 -3.09937 0.005 0 0 0 0.005 0 0 -3.10181 0.005 0 0 0 0.005 0 0 -3.10425 0.005 0 0 0 0.005 0 0 -3.10669 0.005 0 0 0 0.005 0 0 -3.10913 0.005 0 0 0 0.005 0 0 -3.11157 0.005 0 0 0 0.005 0 0 -3.11401 0.005 0 0 0 0.005 0 0 -3.11646 0.005 0 0 0 0.005 0 0 -3.1189 0.005 0 0 0 0.005 0 0 -3.12134 0.005 0 0 0 0.005 0 0 -3.12378 0.005 0 0 0 0.005 0 0 -3.12622 0.005 0 0 0 0.005 0 0 -3.12866 0.005 0 0 0 0.005 0 0 -3.1311 0.005 0 0 0 0.005 0 0 -3.13354 0.005 0 0 0 0.005 0 0 -3.13599 0.005 0 0 0 0.005 0 0 -3.13843 0.005 0 0 0 0.005 0 0 -3.14087 0.005 0 0 0 0.005 0 0 -3.14331 0.005 0 0 0 0.005 0 0 -3.14575 0.005 0 0 0 0.005 0 0 -3.14819 0.005 0 0 0 0.005 0 0 -3.15063 0.005 0 0 0 0.005 0 0 -3.15308 0.005 0 0 0 0.005 0 0 -3.15552 0.005 0 0 0 0.005 0 0 -3.15796 0.005 0 0 0 0.005 0 0 -3.1604 0.005 0 0 0 0.005 0 0 -3.16284 0.005 0 0 0 0.005 0 0 -3.16528 0.005 0 0 0 0.005 0 0 -3.16772 0.005 0 0 0 0.005 0 0 -3.17017 0.005 0 0 0 0.005 0 0 -3.17261 0.005 0 0 0 0.005 0 0 -3.17505 0.005 0 0 0 0.005 0 0 -3.17749 0.005 0 0 0 0.005 0 0 -3.17993 0.005 0 0 0 0.005 0 0 -3.18237 0.005 0 0 0 0.005 0 0 -3.18481 0.005 0 0 0 0.005 0 0 -3.18726 0.005 0 0 0 0.005 0 0 -3.1897 0.005 0 0 0 0.005 0 0 -3.19214 0.005 0 0 0 0.005 0 0 -3.19458 0.005 0 0 0 0.005 0 0 -3.19702 0.005 0 0 0 0.005 0 0 -3.19946 0.005 0 0 0 0.005 0 0 -3.2019 0.005 0 0 0 0.005 0 0 -3.20435 0.005 0 0 0 0.005 0 0 -3.20679 0.005 0 0 0 0.005 0 0 -3.20923 0.005 0 0 0 0.005 0 0 -3.21167 0.005 0 0 0 0.005 0 0 -3.21411 0.005 0 0 0 0.005 0 0 -3.21655 0.005 0 0 0 0.005 0 0 -3.21899 0.005 0 0 0 0.005 0 0 -3.22144 0.005 0 0 0 0.005 0 0 -3.22388 0.005 0 0 0 0.005 0 0 -3.22632 0.005 0 0 0 0.005 0 0 -3.22876 0.005 0 0 0 0.005 0 0 -3.2312 0.005 0 0 0 0.005 0 0 -3.23364 0.005 0 0 0 0.005 0 0 -3.23608 0.005 0 0 0 0.005 0 0 -3.23853 0.005 0 0 0 0.005 0 0 -3.24097 0.005 0 0 0 0.005 0 0 -3.24341 0.005 0 0 0 0.005 0 0 -3.24585 0.005 0 0 0 0.005 0 0 -3.24829 0.005 0 0 0 0.005 0 0 -3.25073 0.005 0 0 0 0.005 0 0 -3.25317 0.005 0 0 0 0.005 0 0 -3.25562 0.005 0 0 0 0.005 0 0 -3.25806 0.005 0 0 0 0.005 0 0 -3.2605 0.005 0 0 0 0.005 0 0 -3.26294 0.005 0 0 0 0.005 0 0 -3.26538 0.005 0 0 0 0.005 0 0 -3.26782 0.005 0 0 0 0.005 0 0 -3.27026 0.005 0 0 0 0.005 0 0 -3.27271 0.005 0 0 0 0.005 0 0 -3.27515 0.005 0 0 0 0.005 0 0 -3.27759 0.005 0 0 0 0.005 0 0 -3.28003 0.005 0 0 0 0.005 0 0 -3.28247 0.005 0 0 0 0.005 0 0 -3.28491 0.005 0 0 0 0.005 0 0 -3.28735 0.005 0 0 0 0.005 0 0 -3.28979 0.005 0 0 0 0.005 0 0 -3.29224 0.005 0 0 0 0.005 0 0 -3.29468 0.005 0 0 0 0.005 0 0 -3.29712 0.005 0 0 0 0.005 0 0 -3.29956 0.005 0 0 0 0.005 0 0 -3.302 0.005 0 0 0 0.005 0 0 -3.30444 0.005 0 0 0 0.005 0 0 -3.30688 0.005 0 0 0 0.005 0 0 -3.30933 0.005 0 0 0 0.005 0 0 -3.31177 0.005 0 0 0 0.005 0 0 -3.31421 0.005 0 0 0 0.005 0 0 -3.31665 0.005 0 0 0 0.005 0 0 -3.31909 0.005 0 0 0 0.005 0 0 -3.32153 0.005 0 0 0 0.005 0 0 -3.32397 0.005 0 0 0 0.005 0 0 -3.32642 0.005 0 0 0 0.005 0 0 -3.32886 0.005 0 0 0 0.005 0 0 -3.3313 0.005 0 0 0 0.005 0 0 -3.33374 0.005 0 0 0 0.005 0 0 -3.33618 0.005 0 0 0 0.005 0 0 -3.33862 0.005 0 0 0 0.005 0 0 -3.34106 0.005 0 0 0 0.005 0 0 -3.34351 0.005 0 0 0 0.005 0 0 -3.34595 0.005 0 0 0 0.005 0 0 -3.34839 0.005 0 0 0 0.005 0 0 -3.35083 0.005 0 0 0 0.005 0 0 -3.35327 0.005 0 0 0 0.005 0 0 -3.35571 0.005 0 0 0 0.005 0 0 -3.35815 0.005 0 0 0 0.005 0 0 -3.3606 0.005 0 0 0 0.005 0 0 -3.36304 0.005 0 0 0 0.005 0 0 -3.36548 0.005 0 0 0 0.005 0 0 -3.36792 0.005 0 0 0 0.005 0 0 -3.37036 0.005 0 0 0 0.005 0 0 -3.3728 0.005 0 0 0 0.005 0 0 -3.37524 0.005 0 0 0 0.005 0 0 -3.37769 0.005 0 0 0 0.005 0 0 -3.38013 0.005 0 0 0 0.005 0 0 -3.38257 0.005 0 0 0 0.005 0 0 -3.38501 0.005 0 0 0 0.005 0 0 -3.38745 0.005 0 0 0 0.005 0 0 -3.38989 0.005 0 0 0 0.005 0 0 -3.39233 0.005 0 0 0 0.005 0 0 -3.39478 0.005 0 0 0 0.005 0 0 -3.39722 0.005 0 0 0 0.005 0 0 -3.39966 0.005 0 0 0 0.005 0 0 -3.4021 0.005 0 0 0 0.005 0 0 -3.40454 0.005 0 0 0 0.005 0 0 -3.40698 0.005 0 0 0 0.005 0 0 -3.40942 0.005 0 0 0 0.005 0 0 -3.41187 0.005 0 0 0 0.005 0 0 -3.41431 0.005 0 0 0 0.005 0 0 -3.41675 0.005 0 0 0 0.005 0 0 -3.41919 0.005 0 0 0 0.005 0 0 -3.42163 0.005 0 0 0 0.005 0 0 -3.42407 0.005 0 0 0 0.005 0 0 -3.42651 0.005 0 0 0 0.005 0 0 -3.42896 0.005 0 0 0 0.005 0 0 -3.4314 0.005 0 0 0 0.005 0 0 -3.43384 0.005 0 0 0 0.005 0 0 -3.43628 0.005 0 0 0 0.005 0 0 -3.43872 0.005 0 0 0 0.005 0 0 -3.44116 0.005 0 0 0 0.005 0 0 -3.4436 0.005 0 0 0 0.005 0 0 -3.44604 0.005 0 0 0 0.005 0 0 -3.44849 0.005 0 0 0 0.005 0 0 -3.45093 0.005 0 0 0 0.005 0 0 -3.45337 0.005 0 0 0 0.005 0 0 -3.45581 0.005 0 0 0 0.005 0 0 -3.45825 0.005 0 0 0 0.005 0 0 -3.46069 0.005 0 0 0 0.005 0 0 -3.46313 0.005 0 0 0 0.005 0 0 -3.46558 0.005 0 0 0 0.005 0 0 -3.46802 0.005 0 0 0 0.005 0 0 -3.47046 0.005 0 0 0 0.005 0 0 -3.4729 0.005 0 0 0 0.005 0 0 -3.47534 0.005 0 0 0 0.005 0 0 -3.47778 0.005 0 0 0 0.005 0 0 -3.48022 0.005 0 0 0 0.005 0 0 -3.48267 0.005 0 0 0 0.005 0 0 -3.48511 0.005 0 0 0 0.005 0 0 -3.48755 0.005 0 0 0 0.005 0 0 -3.48999 0.005 0 0 0 0.005 0 0 -3.49243 0.005 0 0 0 0.005 0 0 -3.49487 0.005 0 0 0 0.005 0 0 -3.49731 0.005 0 0 0 0.005 0 0 -3.49976 0.005 0 0 0 0.005 0 0 -3.5022 0.005 0 0 0 0.005 0 0 -3.50464 0.005 0 0 0 0.005 0 0 -3.50708 0.005 0 0 0 0.005 0 0 -3.50952 0.005 0 0 0 0.005 0 0 -3.51196 0.005 0 0 0 0.005 0 0 -3.5144 0.005 0 0 0 0.005 0 0 -3.51685 0.005 0 0 0 0.005 0 0 -3.51929 0.005 0 0 0 0.005 0 0 -3.52173 0.005 0 0 0 0.005 0 0 -3.52417 0.005 0 0 0 0.005 0 0 -3.52661 0.005 0 0 0 0.005 0 0 -3.52905 0.005 0 0 0 0.005 0 0 -3.53149 0.005 0 0 0 0.005 0 0 -3.53394 0.005 0 0 0 0.005 0 0 -3.53638 0.005 0 0 0 0.005 0 0 -3.53882 0.005 0 0 0 0.005 0 0 -3.54126 0.005 0 0 0 0.005 0 0 -3.5437 0.005 0 0 0 0.005 0 0 -3.54614 0.005 0 0 0 0.005 0 0 -3.54858 0.005 0 0 0 0.005 0 0 -3.55103 0.005 0 0 0 0.005 0 0 -3.55347 0.005 0 0 0 0.005 0 0 -3.55591 0.005 0 0 0 0.005 0 0 -3.55835 0.005 0 0 0 0.005 0 0 -3.56079 0.005 0 0 0 0.005 0 0 -3.56323 0.005 0 0 0 0.005 0 0 -3.56567 0.005 0 0 0 0.005 0 0 -3.56812 0.005 0 0 0 0.005 0 0 -3.57056 0.005 0 0 0 0.005 0 0 -3.573 0.005 0 0 0 0.005 0 0 -3.57544 0.005 0 0 0 0.005 0 0 -3.57788 0.005 0 0 0 0.005 0 0 -3.58032 0.005 0 0 0 0.005 0 0 -3.58276 0.005 0 0 0 0.005 0 0 -3.58521 0.005 0 0 0 0.005 0 0 -3.58765 0.005 0 0 0 0.005 0 0 -3.59009 0.005 0 0 0 0.005 0 0 -3.59253 0.005 0 0 0 0.005 0 0 -3.59497 0.005 0 0 0 0.005 0 0 -3.59741 0.005 0 0 0 0.005 0 0 -3.59985 0.005 0 0 0 0.005 0 0 -3.60229 0.005 0 0 0 0.005 0 0 -3.60474 0.005 0 0 0 0.005 0 0 -3.60718 0.005 0 0 0 0.005 0 0 -3.60962 0.005 0 0 0 0.005 0 0 -3.61206 0.005 0 0 0 0.005 0 0 -3.6145 0.005 0 0 0 0.005 0 0 -3.61694 0.005 0 0 0 0.005 0 0 -3.61938 0.005 0 0 0 0.005 0 0 -3.62183 0.005 0 0 0 0.005 0 0 -3.62427 0.005 0 0 0 0.005 0 0 -3.62671 0.005 0 0 0 0.005 0 0 -3.62915 0.005 0 0 0 0.005 0 0 -3.63159 0.005 0 0 0 0.005 0 0 -3.63403 0.005 0 0 0 0.005 0 0 -3.63647 0.005 0 0 0 0.005 0 0 -3.63892 0.005 0 0 0 0.005 0 0 -3.64136 0.005 0 0 0 0.005 0 0 -3.6438 0.005 0 0 0 0.005 0 0 -3.64624 0.005 0 0 0 0.005 0 0 -3.64868 0.005 0 0 0 0.005 0 0 -3.65112 0.005 0 0 0 0.005 0 0 -3.65356 0.005 0 0 0 0.005 0 0 -3.65601 0.005 0 0 0 0.005 0 0 -3.65845 0.005 0 0 0 0.005 0 0 -3.66089 0.005 0 0 0 0.005 0 0 -3.66333 0.005 0 0 0 0.005 0 0 -3.66577 0.005 0 0 0 0.005 0 0 -3.66821 0.005 0 0 0 0.005 0 0 -3.67065 0.005 0 0 0 0.005 0 0 -3.6731 0.00499516 0.00021442 0 1.07106e-006 0.00499516 0.000968625 4.89012e-005 -3.67554 0.00498904 0.000485687 0 2.42311e-006 0.00498904 0.0021954 8.42738e-005 -3.67798 0.00498293 0.000756955 0 3.77185e-006 0.00498293 0.00342368 0.000113191 -3.68042 0.00497681 0.00102822 0 5.11727e-006 0.00497681 0.00465347 0.000138719 -3.68286 0.00497071 0.00129949 0 6.45938e-006 0.00497071 0.00588477 0.000162021 -3.6853 0.0049646 0.00157076 0 7.79818e-006 0.0049646 0.00711758 0.000183699 -3.68774 0.0049585 0.00184202 0 9.13368e-006 0.0049585 0.00835191 0.000204115 -3.69019 0.0049524 0.00211329 0 1.04659e-005 0.0049524 0.00958775 0.000223508 -3.69263 0.00494631 0.00238456 0 1.17948e-005 0.00494631 0.0108251 0.000242049 -3.69507 0.00494022 0.00265583 0 1.31204e-005 0.00494022 0.012064 0.000259861 -3.69751 0.00493414 0.00292709 0 1.44427e-005 0.00493414 0.0133044 0.00027704 -3.69995 0.00492805 0.00319836 0 1.57617e-005 0.00492805 0.0145464 0.000293661 -3.70239 0.00492198 0.00346963 0 1.70774e-005 0.00492198 0.0157899 0.000309784 -3.70483 0.0049159 0.0037409 0 1.83899e-005 0.0049159 0.0170349 0.000325459 -3.70728 0.00490983 0.00401216 0 1.9699e-005 0.00490983 0.0182815 0.000340727 -3.70972 0.00490376 0.00428343 0 2.10049e-005 0.00490376 0.0195296 0.000355624 -3.71216 0.0048977 0.0045547 0 2.23075e-005 0.0048977 0.0207792 0.000370179 -3.7146 0.00489164 0.00482596 0 2.36069e-005 0.00489164 0.0220304 0.000384417 -3.71704 0.00488559 0.00509723 0 2.4903e-005 0.00488559 0.0232831 0.000398362 -3.71948 0.00487953 0.0053685 0 2.61958e-005 0.00487953 0.0245374 0.000412032 -3.72192 0.00487349 0.00563977 0 2.74853e-005 0.00487349 0.0257933 0.000425446 -3.72437 0.00486744 0.00591103 0 2.87716e-005 0.00486744 0.0270507 0.000438618 -3.72681 0.0048614 0.0061823 0 3.00546e-005 0.0048614 0.0283097 0.000451563 -3.72925 0.00485536 0.00645357 0 3.13344e-005 0.00485536 0.0295702 0.000464292 -3.73169 0.00484933 0.00672484 0 3.2611e-005 0.00484933 0.0308323 0.000476818 -3.73413 0.0048433 0.0069961 0 3.38842e-005 0.0048433 0.032096 0.00048915 -3.73657 0.00483728 0.00726737 0 3.51543e-005 0.00483728 0.0333613 0.000501298 -3.73901 0.00483125 0.00753864 0 3.64211e-005 0.00483125 0.0346281 0.00051327 -3.74146 0.00482524 0.00780991 0 3.76846e-005 0.00482524 0.0358965 0.000525073 -3.7439 0.00481922 0.00808117 0 3.8945e-005 0.00481922 0.0371665 0.000536716 -3.74634 0.00481321 0.00835244 0 4.02021e-005 0.00481321 0.0384381 0.000548204 -3.74878 0.0048072 0.00862371 0 4.14559e-005 0.0048072 0.0397112 0.000559545 -3.75122 0.0048012 0.00889498 0 4.27066e-005 0.0048012 0.040986 0.000570742 -3.75366 0.0047952 0.00916624 0 4.3954e-005 0.0047952 0.0422623 0.000581803 -3.7561 0.00478921 0.00943751 0 4.51982e-005 0.00478921 0.0435403 0.000592731 -3.75854 0.00478322 0.00970878 0 4.64392e-005 0.00478322 0.0448198 0.000603532 -3.76099 0.00477723 0.00998004 0 4.76769e-005 0.00477723 0.046101 0.000614209 -3.76343 0.00477124 0.0102513 0 4.89115e-005 0.00477124 0.0473837 0.000624766 -3.76587 0.00476526 0.0105226 0 5.01428e-005 0.00476526 0.0486681 0.000635209 -3.76831 0.00475929 0.0107938 0 5.1371e-005 0.00475929 0.0499541 0.000645539 -3.77075 0.00475331 0.0110651 0 5.25959e-005 0.00475331 0.0512417 0.00065576 -3.77319 0.00474734 0.0113364 0 5.38177e-005 0.00474734 0.0525309 0.000665877 -3.77563 0.00474138 0.0116076 0 5.50362e-005 0.00474138 0.0538217 0.00067589 -3.77808 0.00473542 0.0118789 0 5.62516e-005 0.00473542 0.0551142 0.000685804 -3.78052 0.00472946 0.0121502 0 5.74638e-005 0.00472946 0.0564083 0.000695622 -3.78296 0.0047235 0.0124215 0 5.86728e-005 0.0047235 0.057704 0.000705345 -3.7854 0.00471755 0.0126927 0 5.98786e-005 0.00471755 0.0590013 0.000714976 -3.78784 0.00471161 0.012964 0 6.10812e-005 0.00471161 0.0603003 0.000724517 -3.79028 0.00470566 0.0132353 0 6.22806e-005 0.00470566 0.0616009 0.000733971 -3.79272 0.00469972 0.0135065 0 6.34769e-005 0.00469972 0.0629032 0.00074334 -3.79517 0.00469379 0.0137778 0 6.467e-005 0.00469379 0.0642071 0.000752626 -3.79761 0.00468786 0.0140491 0 6.58599e-005 0.00468786 0.0655127 0.00076183 -3.80005 0.00468193 0.0143203 0 6.70467e-005 0.00468193 0.0668199 0.000770954 -3.80249 0.004676 0.0145916 0 6.82303e-005 0.004676 0.0681288 0.000780001 -3.80493 0.00467008 0.0148629 0 6.94108e-005 0.00467008 0.0694393 0.000788972 -3.80737 0.00466417 0.0151341 0 7.05881e-005 0.00466417 0.0707515 0.000797868 -3.80981 0.00465825 0.0154054 0 7.17622e-005 0.00465825 0.0720654 0.000806692 -3.81226 0.00465234 0.0156767 0 7.29332e-005 0.00465234 0.0733809 0.000815444 -3.8147 0.00464644 0.0159479 0 7.41011e-005 0.00464644 0.0746981 0.000824125 -3.81714 0.00464054 0.0162192 0 7.52658e-005 0.00464054 0.076017 0.000832738 -3.81958 0.00463464 0.0164905 0 7.64273e-005 0.00463464 0.0773375 0.000841284 -3.82202 0.00462874 0.0167617 0 7.75858e-005 0.00462874 0.0786598 0.000849764 -3.82446 0.00462285 0.017033 0 7.8741e-005 0.00462285 0.0799837 0.000858179 -3.8269 0.00461697 0.0173043 0 7.98932e-005 0.00461697 0.0813093 0.00086653 -3.82935 0.00461108 0.0175755 0 8.10422e-005 0.00461108 0.0826366 0.000874819 -3.83179 0.0046052 0.0178468 0 8.21882e-005 0.0046052 0.0839656 0.000883046 -3.83423 0.00459933 0.0181181 0 8.33309e-005 0.00459933 0.0852963 0.000891212 -3.83667 0.00459346 0.0183893 0 8.44706e-005 0.00459346 0.0866287 0.00089932 -3.83911 0.00458759 0.0186606 0 8.56072e-005 0.00458759 0.0879628 0.000907369 -3.84155 0.00458172 0.0189319 0 8.67406e-005 0.00458172 0.0892986 0.00091536 -3.84399 0.00457586 0.0192031 0 8.78709e-005 0.00457586 0.0906361 0.000923295 -3.84644 0.00457001 0.0194744 0 8.89982e-005 0.00457001 0.0919753 0.000931174 -3.84888 0.00456415 0.0197457 0 9.01223e-005 0.00456415 0.0933162 0.000938999 -3.85132 0.0045583 0.0200169 0 9.12433e-005 0.0045583 0.0946589 0.00094677 -3.85376 0.00455246 0.0202882 0 9.23612e-005 0.00455246 0.0960033 0.000954487 -3.8562 0.00454662 0.0205595 0 9.34761e-005 0.00454662 0.0973494 0.000962153 -3.85864 0.00454078 0.0208307 0 9.45878e-005 0.00454078 0.0986972 0.000969766 -3.86108 0.00453495 0.021102 0 9.56964e-005 0.00453495 0.100047 0.000977329 -3.86353 0.00452911 0.0213733 0 9.6802e-005 0.00452911 0.101398 0.000984842 -3.86597 0.00452329 0.0216445 0 9.79045e-005 0.00452329 0.102751 0.000992306 -3.86841 0.00451746 0.0219158 0 9.90039e-005 0.00451746 0.104106 0.00099972 -3.87085 0.00451165 0.0221871 0 0.0001001 0.00451165 0.105463 0.00100709 -3.87329 0.00450583 0.0224583 0 0.000101193 0.00450583 0.106821 0.00101441 -3.87573 0.00450002 0.0227296 0 0.000102284 0.00450002 0.108181 0.00102168 -3.87817 0.00449421 0.0230009 0 0.000103371 0.00449421 0.109543 0.00102891 -3.88062 0.00448841 0.0232721 0 0.000104455 0.00448841 0.110906 0.00103609 -3.88306 0.00448261 0.0235434 0 0.000105536 0.00448261 0.112272 0.00104322 -3.8855 0.00447681 0.0238147 0 0.000106614 0.00447681 0.113639 0.00105031 -3.88794 0.00447102 0.0240859 0 0.000107689 0.00447102 0.115007 0.00105736 -3.89038 0.00446523 0.0243572 0 0.00010876 0.00446523 0.116378 0.00106437 -3.89282 0.00445944 0.0246285 0 0.000109829 0.00445944 0.117751 0.00107133 -3.89526 0.00445366 0.0248997 0 0.000110895 0.00445366 0.119125 0.00107825 -3.89771 0.00444788 0.025171 0 0.000111958 0.00444788 0.120501 0.00108512 -3.90015 0.00444211 0.0254423 0 0.000113017 0.00444211 0.121879 0.00109196 -3.90259 0.00443634 0.0257136 0 0.000114074 0.00443634 0.123258 0.00109876 -3.90503 0.00443057 0.0259848 0 0.000115128 0.00443057 0.124639 0.00110551 -3.90747 0.00442481 0.0262561 0 0.000116178 0.00442481 0.126023 0.00111223 -3.90991 0.00441905 0.0265274 0 0.000117226 0.00441905 0.127408 0.0011189 -3.91235 0.00441329 0.0267986 0 0.00011827 0.00441329 0.128794 0.00112554 -3.91479 0.00440754 0.0270699 0 0.000119312 0.00440754 0.130183 0.00113214 -3.91724 0.00440179 0.0273412 0 0.00012035 0.00440179 0.131573 0.0011387 -3.91968 0.00439605 0.0276124 0 0.000121386 0.00439605 0.132965 0.00114522 -3.92212 0.00439031 0.0278837 0 0.000122418 0.00439031 0.134359 0.0011517 -3.92456 0.00438457 0.028155 0 0.000123447 0.00438457 0.135755 0.00115815 -3.927 0.00437884 0.0284262 0 0.000124474 0.00437884 0.137153 0.00116456 -3.92944 0.00437311 0.0286975 0 0.000125497 0.00437311 0.138553 0.00117094 -3.93188 0.00436738 0.0289688 0 0.000126518 0.00436738 0.139954 0.00117727 -3.93433 0.00436166 0.02924 0 0.000127535 0.00436166 0.141357 0.00118358 -3.93677 0.00435594 0.0295113 0 0.000128549 0.00435594 0.142762 0.00118985 -3.93921 0.00435023 0.0297826 0 0.000129561 0.00435023 0.144169 0.00119608 -3.94165 0.00434452 0.0300538 0 0.000130569 0.00434452 0.145578 0.00120228 -3.94409 0.00433881 0.0303251 0 0.000131575 0.00433881 0.146988 0.00120844 -3.94653 0.00433311 0.0305964 0 0.000132577 0.00433311 0.148401 0.00121457 -3.94897 0.00432741 0.0308676 0 0.000133577 0.00432741 0.149815 0.00122067 -3.95142 0.00432171 0.0311389 0 0.000134573 0.00432171 0.151231 0.00122673 -3.95386 0.00431602 0.0314102 0 0.000135567 0.00431602 0.152649 0.00123276 -3.9563 0.00431033 0.0316814 0 0.000136558 0.00431033 0.154069 0.00123876 -3.95874 0.00430465 0.0319527 0 0.000137545 0.00430465 0.155491 0.00124473 -3.96118 0.00429897 0.032224 0 0.00013853 0.00429897 0.156914 0.00125066 -3.96362 0.00429329 0.0324952 0 0.000139512 0.00429329 0.15834 0.00125656 -3.96606 0.00428762 0.0327665 0 0.00014049 0.00428762 0.159767 0.00126243 -3.96851 0.00428195 0.0330378 0 0.000141466 0.00428195 0.161196 0.00126827 -3.97095 0.00427628 0.033309 0 0.000142439 0.00427628 0.162628 0.00127408 -3.97339 0.00427062 0.0335803 0 0.000143409 0.00427062 0.164061 0.00127986 -3.97583 0.00426496 0.0338516 0 0.000144376 0.00426496 0.165496 0.0012856 -3.97827 0.00425931 0.0341228 0 0.00014534 0.00425931 0.166933 0.00129132 -3.98071 0.00425366 0.0343941 0 0.000146301 0.00425366 0.168371 0.001297 -3.98315 0.00424801 0.0346654 0 0.000147259 0.00424801 0.169812 0.00130266 -3.9856 0.00424237 0.0349366 0 0.000148214 0.00424237 0.171255 0.00130829 -3.98804 0.00423673 0.0352079 0 0.000149166 0.00423673 0.172699 0.00131389 -3.99048 0.00423109 0.0354792 0 0.000150116 0.00423109 0.174146 0.00131945 -3.99292 0.00422546 0.0357504 0 0.000151062 0.00422546 0.175594 0.00132499 -3.99536 0.00421983 0.0360217 0 0.000152006 0.00421983 0.177044 0.00133051 -3.9978 0.00421421 0.036293 0 0.000152946 0.00421421 0.178497 0.00133599 -4.00024 0.00420859 0.0365642 0 0.000153884 0.00420859 0.179951 0.00134144 -4.00269 0.00420297 0.0368355 0 0.000154819 0.00420297 0.181407 0.00134687 -4.00513 0.00419736 0.0371068 0 0.000155751 0.00419736 0.182865 0.00135227 -4.00757 0.00419175 0.037378 0 0.000156679 0.00419175 0.184325 0.00135764 -4.01001 0.00418615 0.0376493 0 0.000157606 0.00418615 0.185787 0.00136299 -4.01245 0.00418054 0.0379206 0 0.000158529 0.00418054 0.187251 0.0013683 -4.01489 0.00417495 0.0381919 0 0.000159449 0.00417495 0.188717 0.00137359 -4.01733 0.00416935 0.0384631 0 0.000160366 0.00416935 0.190185 0.00137886 -4.01978 0.00416376 0.0387344 0 0.000161281 0.00416376 0.191655 0.00138409 -4.02222 0.00415817 0.0390057 0 0.000162192 0.00415817 0.193126 0.0013893 -4.02466 0.00415259 0.0392769 0 0.000163101 0.00415259 0.1946 0.00139449 -4.0271 0.00414701 0.0395482 0 0.000164007 0.00414701 0.196076 0.00139965 -4.02954 0.00414144 0.0398195 0 0.00016491 0.00414144 0.197554 0.00140478 -4.03198 0.00413586 0.0400907 0 0.00016581 0.00413586 0.199034 0.00140989 -4.03442 0.0041303 0.040362 0 0.000166707 0.0041303 0.200515 0.00141497 -4.03687 0.00412473 0.0406333 0 0.000167601 0.00412473 0.201999 0.00142002 -4.03931 0.00411917 0.0409045 0 0.000168493 0.00411917 0.203485 0.00142505 -4.04175 0.00411361 0.0411758 0 0.000169381 0.00411361 0.204973 0.00143006 -4.04419 0.00410806 0.0414471 0 0.000170267 0.00410806 0.206462 0.00143504 -4.04663 0.00410251 0.0417183 0 0.00017115 0.00410251 0.207954 0.00144 -4.04907 0.00409697 0.0419896 0 0.00017203 0.00409697 0.209448 0.00144493 -4.05151 0.00409143 0.0422609 0 0.000172907 0.00409143 0.210944 0.00144984 -4.05396 0.00408589 0.0425321 0 0.000173781 0.00408589 0.212442 0.00145472 -4.0564 0.00408035 0.0428034 0 0.000174653 0.00408035 0.213941 0.00145958 -4.05884 0.00407482 0.0430747 0 0.000175522 0.00407482 0.215443 0.00146441 -4.06128 0.0040693 0.0433459 0 0.000176387 0.0040693 0.216947 0.00146923 -4.06372 0.00406377 0.0436172 0 0.00017725 0.00406377 0.218453 0.00147401 -4.06616 0.00405825 0.0438885 0 0.000178111 0.00405825 0.219961 0.00147878 -4.0686 0.00405274 0.0441597 0 0.000178968 0.00405274 0.221471 0.00148352 -4.07104 0.00404723 0.044431 0 0.000179822 0.00404723 0.222984 0.00148824 -4.07349 0.00404172 0.0447023 0 0.000180674 0.00404172 0.224498 0.00149293 -4.07593 0.00403621 0.0449735 0 0.000181523 0.00403621 0.226014 0.00149761 -4.07837 0.00403071 0.0452448 0 0.000182369 0.00403071 0.227532 0.00150226 -4.08081 0.00402522 0.0455161 0 0.000183212 0.00402522 0.229053 0.00150688 -4.08325 0.00401972 0.0457873 0 0.000184052 0.00401972 0.230575 0.00151149 -4.08569 0.00401423 0.0460586 0 0.00018489 0.00401423 0.2321 0.00151607 -4.08813 0.00400875 0.0463299 0 0.000185725 0.00400875 0.233627 0.00152063 -4.09058 0.00400327 0.0466011 0 0.000186557 0.00400327 0.235155 0.00152517 -4.09302 0.00399779 0.0468724 0 0.000187386 0.00399779 0.236686 0.00152968 -4.09546 0.00399232 0.0471437 0 0.000188212 0.00399232 0.238219 0.00153418 -4.0979 0.00398684 0.0474149 0 0.000189036 0.00398684 0.239754 0.00153865 -4.10034 0.00398138 0.0476862 0 0.000189857 0.00398138 0.241291 0.0015431 -4.10278 0.00397591 0.0479575 0 0.000190675 0.00397591 0.242831 0.00154753 -4.10522 0.00397046 0.0482287 0 0.00019149 0.00397046 0.244372 0.00155194 -4.10767 0.003965 0.0485 0 0.000192303 0.003965 0.245915 0.00155633 -4.11011 0.00395955 0.0487713 0 0.000193112 0.00395955 0.247461 0.00156069 -4.11255 0.0039541 0.0490425 0 0.000193919 0.0039541 0.249009 0.00156504 -4.11499 0.00394866 0.0493138 0 0.000194723 0.00394866 0.250559 0.00156936 -4.11743 0.00394322 0.0495851 0 0.000195525 0.00394322 0.252111 0.00157366 -4.11987 0.00393778 0.0498563 0 0.000196323 0.00393778 0.253665 0.00157794 -4.12231 0.00393235 0.0501276 0 0.000197119 0.00393235 0.255221 0.00158221 -4.12476 0.00392692 0.0503989 0 0.000197912 0.00392692 0.25678 0.00158645 -4.1272 0.00392149 0.0506701 0 0.000198703 0.00392149 0.25834 0.00159067 -4.12964 0.00391607 0.0509414 0 0.00019949 0.00391607 0.259903 0.00159487 -4.13208 0.00391065 0.0512127 0 0.000200275 0.00391065 0.261468 0.00159905 -4.13452 0.00390524 0.051484 0 0.000201057 0.00390524 0.263035 0.00160321 -4.13696 0.00389983 0.0517552 0 0.000201836 0.00389983 0.264604 0.00160735 -4.1394 0.00389442 0.0520265 0 0.000202613 0.00389442 0.266176 0.00161147 -4.14185 0.00388902 0.0522978 0 0.000203387 0.00388902 0.267749 0.00161557 -4.14429 0.00388362 0.052569 0 0.000204158 0.00388362 0.269325 0.00161965 -4.14673 0.00387822 0.0528403 0 0.000204926 0.00387822 0.270903 0.00162371 -4.14917 0.00387283 0.0531116 0 0.000205692 0.00387283 0.272484 0.00162775 -4.15161 0.00386744 0.0533828 0 0.000206455 0.00386744 0.274066 0.00163178 -4.15405 0.00386206 0.0536541 0 0.000207215 0.00386206 0.275651 0.00163578 -4.15649 0.00385668 0.0539254 0 0.000207973 0.00385668 0.277237 0.00163976 -4.15894 0.0038513 0.0541966 0 0.000208728 0.0038513 0.278826 0.00164373 -4.16138 0.00384593 0.0544679 0 0.00020948 0.00384593 0.280418 0.00164767 -4.16382 0.00384056 0.0547392 0 0.000210229 0.00384056 0.282011 0.0016516 -4.16626 0.00383519 0.0550104 0 0.000210976 0.00383519 0.283607 0.00165551 -4.1687 0.00382983 0.0552817 0 0.00021172 0.00382983 0.285205 0.0016594 -4.17114 0.00382447 0.055553 0 0.000212461 0.00382447 0.286805 0.00166327 -4.17358 0.00381912 0.0558242 0 0.000213199 0.00381912 0.288408 0.00166712 -4.17603 0.00381377 0.0560955 0 0.000213935 0.00381377 0.290012 0.00167096 -4.17847 0.00380842 0.0563668 0 0.000214668 0.00380842 0.291619 0.00167477 -4.18091 0.00380308 0.056638 0 0.000215399 0.00380308 0.293228 0.00167857 -4.18335 0.00379774 0.0569093 0 0.000216127 0.00379774 0.29484 0.00168235 -4.18579 0.0037924 0.0571806 0 0.000216852 0.0037924 0.296454 0.00168611 -4.18823 0.00378707 0.0574518 0 0.000217574 0.00378707 0.29807 0.00168985 -4.19067 0.00378175 0.0577231 0 0.000218294 0.00378175 0.299688 0.00169358 -4.19312 0.00377642 0.0579944 0 0.000219011 0.00377642 0.301308 0.00169728 -4.19556 0.0037711 0.0582656 0 0.000219726 0.0037711 0.302931 0.00170097 -4.198 0.00376578 0.0585369 0 0.000220437 0.00376578 0.304556 0.00170464 -4.20044 0.00376047 0.0588082 0 0.000221146 0.00376047 0.306184 0.0017083 -4.20288 0.00375516 0.0590794 0 0.000221853 0.00375516 0.307813 0.00171193 -4.20532 0.00374986 0.0593507 0 0.000222557 0.00374986 0.309445 0.00171555 -4.20776 0.00374455 0.059622 0 0.000223258 0.00374455 0.31108 0.00171915 -4.21021 0.00373926 0.0598932 0 0.000223956 0.00373926 0.312716 0.00172274 -4.21265 0.00373396 0.0601645 0 0.000224652 0.00373396 0.314355 0.0017263 -4.21509 0.00372867 0.0604358 0 0.000225345 0.00372867 0.315997 0.00172985 -4.21753 0.00372338 0.060707 0 0.000226036 0.00372338 0.31764 0.00173338 -4.21997 0.0037181 0.0609783 0 0.000226724 0.0037181 0.319286 0.0017369 -4.22241 0.00371282 0.0612496 0 0.000227409 0.00371282 0.320935 0.0017404 -4.22485 0.00370755 0.0615208 0 0.000228091 0.00370755 0.322585 0.00174388 -4.22729 0.00370228 0.0617921 0 0.000228771 0.00370228 0.324238 0.00174734 -4.22974 0.00369701 0.0620634 0 0.000229449 0.00369701 0.325894 0.00175079 -4.23218 0.00369174 0.0623346 0 0.000230124 0.00369174 0.327551 0.00175422 -4.23462 0.00368648 0.0626059 0 0.000230796 0.00368648 0.329211 0.00175764 -4.23706 0.00368123 0.0628772 0 0.000231465 0.00368123 0.330874 0.00176103 -4.2395 0.00367597 0.0631484 0 0.000232132 0.00367597 0.332539 0.00176441 -4.24194 0.00367072 0.0634197 0 0.000232796 0.00367072 0.334206 0.00176778 -4.24438 0.00366548 0.063691 0 0.000233458 0.00366548 0.335875 0.00177113 -4.24683 0.00366024 0.0639622 0 0.000234117 0.00366024 0.337547 0.00177446 -4.24927 0.003655 0.0642335 0 0.000234773 0.003655 0.339222 0.00177777 -4.25171 0.00364976 0.0645048 0 0.000235427 0.00364976 0.340898 0.00178107 -4.25415 0.00364453 0.0647761 0 0.000236079 0.00364453 0.342578 0.00178436 -4.25659 0.00363931 0.0650473 0 0.000236727 0.00363931 0.344259 0.00178762 -4.25903 0.00363408 0.0653186 0 0.000237373 0.00363408 0.345943 0.00179087 -4.26147 0.00362887 0.0655899 0 0.000238017 0.00362887 0.34763 0.00179411 -4.26392 0.00362365 0.0658611 0 0.000238658 0.00362365 0.349318 0.00179733 -4.26636 0.00361844 0.0661324 0 0.000239296 0.00361844 0.35101 0.00180053 -4.2688 0.00361323 0.0664037 0 0.000239932 0.00361323 0.352703 0.00180372 -4.27124 0.00360803 0.0666749 0 0.000240565 0.00360803 0.3544 0.00180689 -4.27368 0.00360283 0.0669462 0 0.000241195 0.00360283 0.356098 0.00181005 -4.27612 0.00359763 0.0672175 0 0.000241823 0.00359763 0.357799 0.00181319 -4.27856 0.00359244 0.0674887 0 0.000242449 0.00359244 0.359503 0.00181631 -4.28101 0.00358725 0.06776 0 0.000243072 0.00358725 0.361209 0.00181942 -4.28345 0.00358206 0.0680313 0 0.000243692 0.00358206 0.362917 0.00182252 -4.28589 0.00357688 0.0683025 0 0.00024431 0.00357688 0.364628 0.00182559 -4.28833 0.0035717 0.0685738 0 0.000244925 0.0035717 0.366341 0.00182866 -4.29077 0.00356653 0.0688451 0 0.000245538 0.00356653 0.368057 0.00183171 -4.29321 0.00356136 0.0691163 0 0.000246148 0.00356136 0.369776 0.00183474 -4.29565 0.00355619 0.0693876 0 0.000246755 0.00355619 0.371497 0.00183776 -4.2981 0.00355103 0.0696589 0 0.000247361 0.00355103 0.37322 0.00184076 -4.30054 0.00354587 0.0699301 0 0.000247963 0.00354587 0.374946 0.00184375 -4.30298 0.00354071 0.0702014 0 0.000248563 0.00354071 0.376674 0.00184672 -4.30542 0.00353556 0.0704727 0 0.00024916 0.00353556 0.378405 0.00184968 -4.30786 0.00353041 0.0707439 0 0.000249755 0.00353041 0.380139 0.00185262 -4.3103 0.00352527 0.0710152 0 0.000250348 0.00352527 0.381875 0.00185555 -4.31274 0.00352013 0.0712865 0 0.000250938 0.00352013 0.383613 0.00185846 -4.31519 0.00351499 0.0715577 0 0.000251525 0.00351499 0.385354 0.00186136 -4.31763 0.00350986 0.071829 0 0.00025211 0.00350986 0.387098 0.00186425 -4.32007 0.00350473 0.0721003 0 0.000252692 0.00350473 0.388844 0.00186712 -4.32251 0.0034996 0.0723715 0 0.000253272 0.0034996 0.390592 0.00186997 -4.32495 0.00349448 0.0726428 0 0.000253849 0.00349448 0.392344 0.00187281 -4.32739 0.00348936 0.0729141 0 0.000254424 0.00348936 0.394098 0.00187564 -4.32983 0.00348425 0.0731853 0 0.000254996 0.00348425 0.395854 0.00187845 -4.33228 0.00347914 0.0734566 0 0.000255566 0.00347914 0.397613 0.00188125 -4.33472 0.00347403 0.0737279 0 0.000256133 0.00347403 0.399374 0.00188403 -4.33716 0.00346893 0.0739991 0 0.000256698 0.00346893 0.401139 0.0018868 -4.3396 0.00346383 0.0742704 0 0.00025726 0.00346383 0.402905 0.00188955 -4.34204 0.00345874 0.0745417 0 0.00025782 0.00345874 0.404675 0.00189229 -4.34448 0.00345364 0.0748129 0 0.000258377 0.00345364 0.406447 0.00189502 -4.34692 0.00344856 0.0750842 0 0.000258932 0.00344856 0.408221 0.00189773 -4.34937 0.00344347 0.0753555 0 0.000259484 0.00344347 0.409998 0.00190043 -4.35181 0.00343839 0.0756267 0 0.000260034 0.00343839 0.411778 0.00190311 -4.35425 0.00343331 0.075898 0 0.000260582 0.00343331 0.413561 0.00190578 -4.35669 0.00342824 0.0761693 0 0.000261127 0.00342824 0.415346 0.00190844 -4.35913 0.00342317 0.0764405 0 0.000261669 0.00342317 0.417133 0.00191108 -4.36157 0.00341811 0.0767118 0 0.000262209 0.00341811 0.418924 0.00191371 -4.36401 0.00341305 0.0769831 0 0.000262747 0.00341305 0.420717 0.00191632 -4.36646 0.00340799 0.0772544 0 0.000263282 0.00340799 0.422512 0.00191892 -4.3689 0.00340293 0.0775256 0 0.000263815 0.00340293 0.424311 0.00192151 -4.37134 0.00339788 0.0777969 0 0.000264345 0.00339788 0.426112 0.00192408 -4.37378 0.00339284 0.0780682 0 0.000264872 0.00339284 0.427915 0.00192664 -4.37622 0.00338779 0.0783394 0 0.000265398 0.00338779 0.429722 0.00192919 -4.37866 0.00338275 0.0786107 0 0.000265921 0.00338275 0.431531 0.00193172 -4.3811 0.00337772 0.078882 0 0.000266441 0.00337772 0.433343 0.00193424 -4.38354 0.00337269 0.0791532 0 0.000266959 0.00337269 0.435157 0.00193675 -4.38599 0.00336766 0.0794245 0 0.000267475 0.00336766 0.436974 0.00193924 -4.38843 0.00336263 0.0796958 0 0.000267988 0.00336263 0.438794 0.00194172 -4.39087 0.00335761 0.079967 0 0.000268498 0.00335761 0.440617 0.00194419 -4.39331 0.0033526 0.0802383 0 0.000269007 0.0033526 0.442442 0.00194664 -4.39575 0.00334758 0.0805096 0 0.000269513 0.00334758 0.44427 0.00194908 -4.39819 0.00334258 0.0807808 0 0.000270016 0.00334258 0.446101 0.00195151 -4.40063 0.00333757 0.0810521 0 0.000270517 0.00333757 0.447935 0.00195392 -4.40308 0.00333257 0.0813234 0 0.000271016 0.00333257 0.449771 0.00195632 -4.40552 0.00332757 0.0815946 0 0.000271512 0.00332757 0.45161 0.00195871 -4.40796 0.00332258 0.0818659 0 0.000272006 0.00332258 0.453452 0.00196108 -4.4104 0.00331759 0.0821372 0 0.000272497 0.00331759 0.455296 0.00196344 -4.41284 0.0033126 0.0824084 0 0.000272986 0.0033126 0.457144 0.00196579 -4.41528 0.00330762 0.0826797 0 0.000273473 0.00330762 0.458994 0.00196813 -4.41772 0.00330264 0.082951 0 0.000273957 0.00330264 0.460847 0.00197045 -4.42017 0.00329766 0.0832222 0 0.000274439 0.00329766 0.462702 0.00197276 -4.42261 0.00329269 0.0834935 0 0.000274918 0.00329269 0.464561 0.00197506 -4.42505 0.00328772 0.0837648 0 0.000275395 0.00328772 0.466422 0.00197734 -4.42749 0.00328276 0.084036 0 0.00027587 0.00328276 0.468286 0.00197961 -4.42993 0.0032778 0.0843073 0 0.000276342 0.0032778 0.470153 0.00198187 -4.43237 0.00327284 0.0845786 0 0.000276812 0.00327284 0.472023 0.00198412 -4.43481 0.00326789 0.0848498 0 0.00027728 0.00326789 0.473896 0.00198635 -4.43726 0.00326294 0.0851211 0 0.000277745 0.00326294 0.475771 0.00198857 -4.4397 0.00325799 0.0853924 0 0.000278208 0.00325799 0.47765 0.00199078 -4.44214 0.00325305 0.0856636 0 0.000278668 0.00325305 0.479531 0.00199298 -4.44458 0.00324811 0.0859349 0 0.000279126 0.00324811 0.481415 0.00199516 -4.44702 0.00324318 0.0862062 0 0.000279582 0.00324318 0.483302 0.00199733 -4.44946 0.00323825 0.0864774 0 0.000280036 0.00323825 0.485192 0.00199949 -4.4519 0.00323332 0.0867487 0 0.000280487 0.00323332 0.487084 0.00200164 -4.45435 0.0032284 0.08702 0 0.000280935 0.0032284 0.48898 0.00200377 -4.45679 0.00322348 0.0872912 0 0.000281382 0.00322348 0.490878 0.00200589 -4.45923 0.00321857 0.0875625 0 0.000281826 0.00321857 0.492779 0.002008 -4.46167 0.00321365 0.0878338 0 0.000282267 0.00321365 0.494684 0.0020101 -4.46411 0.00320875 0.088105 0 0.000282707 0.00320875 0.496591 0.00201218 -4.46655 0.00320384 0.0883763 0 0.000283144 0.00320384 0.498501 0.00201426 -4.46899 0.00319894 0.0886476 0 0.000283578 0.00319894 0.500414 0.00201632 -4.47144 0.00319404 0.0889188 0 0.000284011 0.00319404 0.50233 0.00201837 -4.47388 0.00318915 0.0891901 0 0.000284441 0.00318915 0.504249 0.0020204 -4.47632 0.00318426 0.0894614 0 0.000284869 0.00318426 0.50617 0.00202243 -4.47876 0.00317938 0.0897326 0 0.000285294 0.00317938 0.508095 0.00202444 -4.4812 0.0031745 0.0900039 0 0.000285717 0.0031745 0.510023 0.00202644 -4.48364 0.00316962 0.0902752 0 0.000286138 0.00316962 0.511953 0.00202843 -4.48608 0.00316474 0.0905465 0 0.000286556 0.00316474 0.513887 0.00203041 -4.48853 0.00315987 0.0908177 0 0.000286972 0.00315987 0.515824 0.00203237 -4.49097 0.00315501 0.091089 0 0.000287386 0.00315501 0.517763 0.00203433 -4.49341 0.00315014 0.0913603 0 0.000287798 0.00315014 0.519706 0.00203627 -4.49585 0.00314528 0.0916315 0 0.000288207 0.00314528 0.521651 0.0020382 -4.49829 0.00314043 0.0919028 0 0.000288614 0.00314043 0.5236 0.00204012 -4.50073 0.00313558 0.0921741 0 0.000289019 0.00313558 0.525552 0.00204203 -4.50317 0.00313073 0.0924453 0 0.000289421 0.00313073 0.527506 0.00204392 -4.50562 0.00312588 0.0927166 0 0.000289821 0.00312588 0.529464 0.0020458 -4.50806 0.00312104 0.0929879 0 0.000290219 0.00312104 0.531425 0.00204768 -4.5105 0.00311621 0.0932591 0 0.000290615 0.00311621 0.533388 0.00204954 -4.51294 0.00311137 0.0935304 0 0.000291008 0.00311137 0.535355 0.00205138 -4.51538 0.00310655 0.0938017 0 0.000291399 0.00310655 0.537325 0.00205322 -4.51782 0.00310172 0.0940729 0 0.000291788 0.00310172 0.539298 0.00205505 -4.52026 0.0030969 0.0943442 0 0.000292174 0.0030969 0.541274 0.00205686 -4.52271 0.00309208 0.0946155 0 0.000292559 0.00309208 0.543253 0.00205866 -4.52515 0.00308727 0.0948867 0 0.000292941 0.00308727 0.545235 0.00206046 -4.52759 0.00308246 0.095158 0 0.00029332 0.00308246 0.54722 0.00206224 -4.53003 0.00307765 0.0954293 0 0.000293698 0.00307765 0.549209 0.00206401 -4.53247 0.00307285 0.0957005 0 0.000294073 0.00307285 0.5512 0.00206576 -4.53491 0.00306805 0.0959718 0 0.000294446 0.00306805 0.553195 0.00206751 -4.53735 0.00306325 0.0962431 0 0.000294817 0.00306325 0.555192 0.00206924 -4.53979 0.00305846 0.0965143 0 0.000295185 0.00305846 0.557193 0.00207097 -4.54224 0.00305367 0.0967856 0 0.000295552 0.00305367 0.559197 0.00207268 -4.54468 0.00304889 0.0970569 0 0.000295916 0.00304889 0.561204 0.00207438 -4.54712 0.00304411 0.0973281 0 0.000296277 0.00304411 0.563215 0.00207607 -4.54956 0.00303933 0.0975994 0 0.000296637 0.00303933 0.565228 0.00207775 -4.552 0.00303456 0.0978707 0 0.000296994 0.00303456 0.567244 0.00207942 -4.55444 0.00302979 0.0981419 0 0.000297349 0.00302979 0.569264 0.00208108 -4.55688 0.00302502 0.0984132 0 0.000297702 0.00302502 0.571287 0.00208272 -4.55933 0.00302026 0.0986845 0 0.000298053 0.00302026 0.573313 0.00208436 -4.56177 0.0030155 0.0989557 0 0.000298401 0.0030155 0.575343 0.00208598 -4.56421 0.00301075 0.099227 0 0.000298748 0.00301075 0.577375 0.0020876 -4.56665 0.003006 0.0994983 0 0.000299092 0.003006 0.579411 0.0020892 -4.56909 0.00300125 0.0997695 0 0.000299434 0.00300125 0.58145 0.00209079 -4.57153 0.00299651 0.100041 0 0.000299773 0.00299651 0.583492 0.00209237 -4.57397 0.00299177 0.100312 0 0.000300111 0.00299177 0.585537 0.00209394 -4.57642 0.00298704 0.100583 0 0.000300446 0.00298704 0.587586 0.0020955 -4.57886 0.0029823 0.100855 0 0.000300779 0.0029823 0.589638 0.00209705 -4.5813 0.00297758 0.101126 0 0.00030111 0.00297758 0.591693 0.00209859 -4.58374 0.00297285 0.101397 0 0.000301439 0.00297285 0.593751 0.00210011 -4.58618 0.00296813 0.101668 0 0.000301765 0.00296813 0.595813 0.00210163 -4.58862 0.00296342 0.10194 0 0.00030209 0.00296342 0.597878 0.00210314 -4.59106 0.0029587 0.102211 0 0.000302412 0.0029587 0.599946 0.00210463 -4.59351 0.00295399 0.102482 0 0.000302732 0.00295399 0.602018 0.00210612 -4.59595 0.00294929 0.102753 0 0.00030305 0.00294929 0.604093 0.00210759 -4.59839 0.00294459 0.103025 0 0.000303365 0.00294459 0.606171 0.00210905 -4.60083 0.00293989 0.103296 0 0.000303679 0.00293989 0.608252 0.00211051 -4.60327 0.00293519 0.103567 0 0.00030399 0.00293519 0.610337 0.00211195 -4.60571 0.0029305 0.103839 0 0.000304299 0.0029305 0.612425 0.00211338 -4.60815 0.00292582 0.10411 0 0.000304606 0.00292582 0.614517 0.0021148 -4.6106 0.00292113 0.104381 0 0.000304911 0.00292113 0.616611 0.00211621 -4.61304 0.00291646 0.104652 0 0.000305214 0.00291646 0.61871 0.00211761 -4.61548 0.00291178 0.104924 0 0.000305514 0.00291178 0.620811 0.002119 -4.61792 0.00290711 0.105195 0 0.000305813 0.00290711 0.622916 0.00212038 -4.62036 0.00290244 0.105466 0 0.000306109 0.00290244 0.625024 0.00212175 -4.6228 0.00289778 0.105737 0 0.000306403 0.00289778 0.627136 0.00212311 -4.62524 0.00289312 0.106009 0 0.000306695 0.00289312 0.629251 0.00212446 -4.62769 0.00288846 0.10628 0 0.000306985 0.00288846 0.63137 0.0021258 -4.63013 0.00288381 0.106551 0 0.000307273 0.00288381 0.633492 0.00212713 -4.63257 0.00287916 0.106822 0 0.000307559 0.00287916 0.635617 0.00212844 -4.63501 0.00287451 0.107094 0 0.000307842 0.00287451 0.637746 0.00212975 -4.63745 0.00286987 0.107365 0 0.000308124 0.00286987 0.639878 0.00213105 -4.63989 0.00286523 0.107636 0 0.000308403 0.00286523 0.642014 0.00213234 -4.64233 0.0028606 0.107908 0 0.00030868 0.0028606 0.644153 0.00213362 -4.64478 0.00285597 0.108179 0 0.000308955 0.00285597 0.646295 0.00213488 -4.64722 0.00285134 0.10845 0 0.000309228 0.00285134 0.648442 0.00213614 -4.64966 0.00284672 0.108721 0 0.000309499 0.00284672 0.650591 0.00213739 -4.6521 0.0028421 0.108993 0 0.000309768 0.0028421 0.652744 0.00213862 -4.65454 0.00283748 0.109264 0 0.000310035 0.00283748 0.654901 0.00213985 -4.65698 0.00283287 0.109535 0 0.000310299 0.00283287 0.657061 0.00214107 -4.65942 0.00282826 0.109806 0 0.000310562 0.00282826 0.659224 0.00214228 -4.66187 0.00282366 0.110078 0 0.000310822 0.00282366 0.661392 0.00214347 -4.66431 0.00281906 0.110349 0 0.00031108 0.00281906 0.663562 0.00214466 -4.66675 0.00281446 0.11062 0 0.000311337 0.00281446 0.665736 0.00214584 -4.66919 0.00280987 0.110892 0 0.000311591 0.00280987 0.667914 0.00214701 -4.67163 0.00280528 0.111163 0 0.000311843 0.00280528 0.670095 0.00214816 -4.67407 0.0028007 0.111434 0 0.000312093 0.0028007 0.67228 0.00214931 -4.67651 0.00279611 0.111705 0 0.000312341 0.00279611 0.674469 0.00215045 -4.67896 0.00279154 0.111977 0 0.000312587 0.00279154 0.676661 0.00215158 -4.6814 0.00278696 0.112248 0 0.000312831 0.00278696 0.678856 0.0021527 -4.68384 0.00278239 0.112519 0 0.000313072 0.00278239 0.681056 0.00215381 -4.68628 0.00277783 0.11279 0 0.000313312 0.00277783 0.683259 0.00215491 -4.68872 0.00277326 0.113062 0 0.00031355 0.00277326 0.685465 0.002156 -4.69116 0.0027687 0.113333 0 0.000313785 0.0027687 0.687675 0.00215708 -4.6936 0.00276415 0.113604 0 0.000314019 0.00276415 0.689889 0.00215815 -4.69604 0.0027596 0.113875 0 0.00031425 0.0027596 0.692106 0.00215921 -4.69849 0.00275505 0.114147 0 0.00031448 0.00275505 0.694327 0.00216026 -4.70093 0.0027505 0.114418 0 0.000314707 0.0027505 0.696552 0.0021613 -4.70337 0.00274596 0.114689 0 0.000314933 0.00274596 0.69878 0.00216233 -4.70581 0.00274143 0.114961 0 0.000315156 0.00274143 0.701013 0.00216335 -4.70825 0.00273689 0.115232 0 0.000315377 0.00273689 0.703248 0.00216437 -4.71069 0.00273237 0.115503 0 0.000315597 0.00273237 0.705488 0.00216537 -4.71313 0.00272784 0.115774 0 0.000315814 0.00272784 0.707731 0.00216636 -4.71558 0.00272332 0.116046 0 0.000316029 0.00272332 0.709978 0.00216735 -4.71802 0.0027188 0.116317 0 0.000316242 0.0027188 0.712229 0.00216832 -4.72046 0.00271429 0.116588 0 0.000316454 0.00271429 0.714483 0.00216929 -4.7229 0.00270978 0.116859 0 0.000316663 0.00270978 0.716741 0.00217024 -4.72534 0.00270527 0.117131 0 0.00031687 0.00270527 0.719003 0.00217119 -4.72778 0.00270077 0.117402 0 0.000317075 0.00270077 0.721269 0.00217213 -4.73022 0.00269627 0.117673 0 0.000317278 0.00269627 0.723538 0.00217305 -4.73267 0.00269177 0.117944 0 0.00031748 0.00269177 0.725811 0.00217397 -4.73511 0.00268728 0.118216 0 0.000317679 0.00268728 0.728089 0.00217488 -4.73755 0.00268279 0.118487 0 0.000317876 0.00268279 0.730369 0.00217578 -4.73999 0.00267831 0.118758 0 0.000318071 0.00267831 0.732654 0.00217667 -4.74243 0.00267383 0.11903 0 0.000318265 0.00267383 0.734943 0.00217756 -4.74487 0.00266935 0.119301 0 0.000318456 0.00266935 0.737235 0.00217843 -4.74731 0.00266488 0.119572 0 0.000318645 0.00266488 0.739531 0.00217929 -4.74976 0.00266041 0.119843 0 0.000318832 0.00266041 0.741831 0.00218014 -4.7522 0.00265594 0.120115 0 0.000319018 0.00265594 0.744135 0.00218099 -4.75464 0.00265148 0.120386 0 0.000319201 0.00265148 0.746443 0.00218182 -4.75708 0.00264703 0.120657 0 0.000319382 0.00264703 0.748754 0.00218265 -4.75952 0.00264257 0.120928 0 0.000319562 0.00264257 0.75107 0.00218347 -4.76196 0.00263812 0.1212 0 0.000319739 0.00263812 0.753389 0.00218428 -4.7644 0.00263367 0.121471 0 0.000319915 0.00263367 0.755713 0.00218508 -4.76685 0.00262923 0.121742 0 0.000320088 0.00262923 0.75804 0.00218587 -4.76929 0.00262479 0.122013 0 0.00032026 0.00262479 0.760371 0.00218665 -4.77173 0.00262036 0.122285 0 0.00032043 0.00262036 0.762707 0.00218742 -4.77417 0.00261593 0.122556 0 0.000320597 0.00261593 0.765046 0.00218818 -4.77661 0.0026115 0.122827 0 0.000320763 0.0026115 0.767389 0.00218894 -4.77905 0.00260707 0.123099 0 0.000320927 0.00260707 0.769736 0.00218968 -4.78149 0.00260265 0.12337 0 0.000321089 0.00260265 0.772087 0.00219042 -4.78394 0.00259824 0.123641 0 0.000321249 0.00259824 0.774442 0.00219115 -4.78638 0.00259382 0.123912 0 0.000321407 0.00259382 0.776801 0.00219186 -4.78882 0.00258941 0.124184 0 0.000321563 0.00258941 0.779164 0.00219257 -4.79126 0.00258501 0.124455 0 0.000321717 0.00258501 0.781531 0.00219327 -4.7937 0.00258061 0.124726 0 0.000321869 0.00258061 0.783902 0.00219397 -4.79614 0.00257621 0.124997 0 0.00032202 0.00257621 0.786278 0.00219465 -4.79858 0.00257182 0.125269 0 0.000322168 0.00257182 0.788657 0.00219532 -4.80103 0.00256743 0.12554 0 0.000322314 0.00256743 0.79104 0.00219599 -4.80347 0.00256304 0.125811 0 0.000322459 0.00256304 0.793427 0.00219665 -4.80591 0.00255866 0.126082 0 0.000322602 0.00255866 0.795819 0.00219729 -4.80835 0.00255428 0.126354 0 0.000322742 0.00255428 0.798214 0.00219793 -4.81079 0.0025499 0.126625 0 0.000322881 0.0025499 0.800614 0.00219856 -4.81323 0.00254553 0.126896 0 0.000323018 0.00254553 0.803018 0.00219918 -4.81567 0.00254116 0.127168 0 0.000323153 0.00254116 0.805426 0.0021998 -4.81812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.82056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.82544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.82788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.83765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.84985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.86938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.88892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.8938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.90845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.92065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.92554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.92798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.94995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.96948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.97925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.98901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01099 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01343 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01587 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.01831 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02075 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02319 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02563 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.02808 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03052 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03296 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.03784 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04028 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04272 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04517 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04761 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05005 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05493 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05737 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.05981 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06226 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06714 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.07202 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.07446 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.07935 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08179 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08423 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08911 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09155 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09399 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09644 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.09888 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10132 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10864 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11108 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11597 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.11841 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12329 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12573 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12817 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13306 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.1355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.13794 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14038 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14282 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14526 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15015 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15259 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15503 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15747 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.15991 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16235 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16724 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16968 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17212 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17456 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.17944 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18433 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18677 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18921 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19165 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19409 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19653 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.19897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20142 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20386 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20874 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21118 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21362 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.21851 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22095 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22339 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22583 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22827 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23071 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23315 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.23804 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24048 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24536 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25024 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25269 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25513 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.25757 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26245 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26489 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26733 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26978 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27222 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27466 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.27954 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28198 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28442 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28687 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.28931 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29175 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29419 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29663 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.29907 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30151 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30884 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31128 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31372 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.31616 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32349 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32593 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32837 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33081 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33325 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33569 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.33813 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34058 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34302 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.34546 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35034 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35278 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35522 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.35767 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36011 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36255 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36743 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36987 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37476 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.37964 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38208 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38452 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38696 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39185 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39429 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39673 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.39917 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40161 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40405 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40649 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41138 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41382 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41626 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42114 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42358 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.42847 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43091 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43335 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43579 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44067 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.44556 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45044 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45288 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45532 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45776 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46265 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46509 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46753 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.46997 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47241 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47485 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47974 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48218 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48462 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.48706 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.4895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49194 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49683 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49927 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50171 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50659 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.50903 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51147 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51392 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51636 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52368 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52612 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.52856 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53101 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53345 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53589 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54077 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54321 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.54565 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55054 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55298 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55786 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56274 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56519 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.56763 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57007 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57495 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57739 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57983 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58228 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58472 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.58716 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59204 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59692 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.59937 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60181 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60425 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60669 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.60913 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61157 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61401 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62134 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62378 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62622 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.62866 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63599 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63843 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64087 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64331 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64575 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.64819 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65308 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65552 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.65796 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66284 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66528 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.66772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67017 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67261 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67505 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67749 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67993 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68237 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.68726 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69214 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69458 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69702 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69946 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70435 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70679 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.70923 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71167 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71411 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71655 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71899 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72388 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72632 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.72876 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73364 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73608 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74097 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74341 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.74829 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75317 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75562 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75806 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.7605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76294 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76538 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.76782 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77026 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77515 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77759 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78003 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78247 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78491 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78735 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.78979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79224 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79468 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79712 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79956 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80444 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80688 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.80933 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81421 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81909 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82153 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82397 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82642 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.82886 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83618 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83862 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84106 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84351 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84595 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84839 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85327 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85571 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.85815 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86304 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86548 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87036 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.8728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87524 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.87769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88013 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88257 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88501 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88745 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88989 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89233 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89722 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.89966 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90454 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90698 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.90942 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91431 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91675 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.91919 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92163 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92407 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92651 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93384 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93628 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.93872 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94116 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94849 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95093 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95337 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95581 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.95825 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96069 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96558 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.96802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97046 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97534 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.97778 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98267 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98511 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98755 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99243 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99487 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.99976 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00464 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00952 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01196 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01685 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.01929 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02173 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02661 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02905 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03149 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03394 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03638 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.03882 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.0437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04614 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04858 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05103 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05347 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05591 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.05835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06079 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06323 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06567 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.07788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.09985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.11938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.13892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.15845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.17065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.17554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.17798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.1853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.19995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.21948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.22925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.23901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.26099 0.001 0 0 0 0.001 0 0 -6.26343 0.001 0 0 0 0.001 0 0 -6.26587 0.001 0 0 0 0.001 0 0 -6.26831 0.001 0 0 0 0.001 0 0 -6.27075 0.001 0 0 0 0.001 0 0 -6.27319 0.001 0 0 0 0.001 0 0 -6.27563 0.001 0 0 0 0.001 0 0 -6.27808 0.001 0 0 0 0.001 0 0 -6.28052 0.001 0 0 0 0.001 0 0 -6.28296 0.001 0 0 0 0.001 0 0 -6.2854 0.001 0 0 0 0.001 0 0 -6.28784 0.001 0 0 0 0.001 0 0 -6.29028 0.001 0 0 0 0.001 0 0 -6.29272 0.001 0 0 0 0.001 0 0 -6.29517 0.001 0 0 0 0.001 0 0 -6.29761 0.001 0 0 0 0.001 0 0 -6.30005 0.001 0 0 0 0.001 0 0 -6.30249 0.001 0 0 0 0.001 0 0 -6.30493 0.001 0 0 0 0.001 0 0 -6.30737 0.001 0 0 0 0.001 0 0 -6.30981 0.001 0 0 0 0.001 0 0 -6.31226 0.001 0 0 0 0.001 0 0 -6.3147 0.001 0 0 0 0.001 0 0 -6.31714 0.001 0 0 0 0.001 0 0 -6.31958 0.001 0 0 0 0.001 0 0 -6.32202 0.001 0 0 0 0.001 0 0 -6.32446 0.001 0 0 0 0.001 0 0 -6.3269 0.001 0 0 0 0.001 0 0 -6.32935 0.001 0 0 0 0.001 0 0 -6.33179 0.001 0 0 0 0.001 0 0 -6.33423 0.001 0 0 0 0.001 0 0 -6.33667 0.001 0 0 0 0.001 0 0 -6.33911 0.001 0 0 0 0.001 0 0 -6.34155 0.001 0 0 0 0.001 0 0 -6.34399 0.001 0 0 0 0.001 0 0 -6.34644 0.001 0 0 0 0.001 0 0 -6.34888 0.001 0 0 0 0.001 0 0 -6.35132 0.001 0 0 0 0.001 0 0 -6.35376 0.001 0 0 0 0.001 0 0 -6.3562 0.001 0 0 0 0.001 0 0 -6.35864 0.001 0 0 0 0.001 0 0 -6.36108 0.001 0 0 0 0.001 0 0 -6.36353 0.001 0 0 0 0.001 0 0 -6.36597 0.001 0 0 0 0.001 0 0 -6.36841 0.001 0 0 0 0.001 0 0 -6.37085 0.001 0 0 0 0.001 0 0 -6.37329 0.001 0 0 0 0.001 0 0 -6.37573 0.001 0 0 0 0.001 0 0 -6.37817 0.001 0 0 0 0.001 0 0 -6.38062 0.001 0 0 0 0.001 0 0 -6.38306 0.001 0 0 0 0.001 0 0 -6.3855 0.001 0 0 0 0.001 0 0 -6.38794 0.001 0 0 0 0.001 0 0 -6.39038 0.001 0 0 0 0.001 0 0 -6.39282 0.001 0 0 0 0.001 0 0 -6.39526 0.001 0 0 0 0.001 0 0 -6.39771 0.001 0 0 0 0.001 0 0 -6.40015 0.001 0 0 0 0.001 0 0 -6.40259 0.001 0 0 0 0.001 0 0 -6.40503 0.001 0 0 0 0.001 0 0 -6.40747 0.001 0 0 0 0.001 0 0 -6.40991 0.001 0 0 0 0.001 0 0 -6.41235 0.001 0 0 0 0.001 0 0 -6.41479 0.001 0 0 0 0.001 0 0 -6.41724 0.001 0 0 0 0.001 0 0 -6.41968 0.001 0 0 0 0.001 0 0 -6.42212 0.001 0 0 0 0.001 0 0 -6.42456 0.001 0 0 0 0.001 0 0 -6.427 0.001 0 0 0 0.001 0 0 -6.42944 0.001 0 0 0 0.001 0 0 -6.43188 0.001 0 0 0 0.001 0 0 -6.43433 0.001 0 0 0 0.001 0 0 -6.43677 0.001 0 0 0 0.001 0 0 -6.43921 0.001 0 0 0 0.001 0 0 -6.44165 0.001 0 0 0 0.001 0 0 -6.44409 0.001 0 0 0 0.001 0 0 -6.44653 0.001 0 0 0 0.001 0 0 -6.44897 0.001 0 0 0 0.001 0 0 -6.45142 0.001 0 0 0 0.001 0 0 -6.45386 0.001 0 0 0 0.001 0 0 -6.4563 0.001 0 0 0 0.001 0 0 -6.45874 0.001 0 0 0 0.001 0 0 -6.46118 0.001 0 0 0 0.001 0 0 -6.46362 0.001 0 0 0 0.001 0 0 -6.46606 0.001 0 0 0 0.001 0 0 -6.46851 0.001 0 0 0 0.001 0 0 -6.47095 0.001 0 0 0 0.001 0 0 -6.47339 0.001 0 0 0 0.001 0 0 -6.47583 0.001 0 0 0 0.001 0 0 -6.47827 0.001 0 0 0 0.001 0 0 -6.48071 0.001 0 0 0 0.001 0 0 -6.48315 0.001 0 0 0 0.001 0 0 -6.4856 0.001 0 0 0 0.001 0 0 -6.48804 0.001 0 0 0 0.001 0 0 -6.49048 0.001 0 0 0 0.001 0 0 -6.49292 0.001 0 0 0 0.001 0 0 -6.49536 0.001 0 0 0 0.001 0 0 -6.4978 0.001 0 0 0 0.001 0 0 -6.50024 0.001 0 0 0 0.001 0 0 -6.50269 0.001 0 0 0 0.001 0 0 -6.50513 0.001 0 0 0 0.001 0 0 -6.50757 0.001 0 0 0 0.001 0 0 -6.51001 0.001 0 0 0 0.001 0 0 -6.51245 0.001 0 0 0 0.001 0 0 -6.51489 0.001 0 0 0 0.001 0 0 -6.51733 0.001 0 0 0 0.001 0 0 -6.51978 0.001 0 0 0 0.001 0 0 -6.52222 0.001 0 0 0 0.001 0 0 -6.52466 0.001 0 0 0 0.001 0 0 -6.5271 0.001 0 0 0 0.001 0 0 -6.52954 0.001 0 0 0 0.001 0 0 -6.53198 0.001 0 0 0 0.001 0 0 -6.53442 0.001 0 0 0 0.001 0 0 -6.53687 0.001 0 0 0 0.001 0 0 -6.53931 0.001 0 0 0 0.001 0 0 -6.54175 0.001 0 0 0 0.001 0 0 -6.54419 0.001 0 0 0 0.001 0 0 -6.54663 0.001 0 0 0 0.001 0 0 -6.54907 0.001 0 0 0 0.001 0 0 -6.55151 0.001 0 0 0 0.001 0 0 -6.55396 0.001 0 0 0 0.001 0 0 -6.5564 0.001 0 0 0 0.001 0 0 -6.55884 0.001 0 0 0 0.001 0 0 -6.56128 0.001 0 0 0 0.001 0 0 -6.56372 0.001 0 0 0 0.001 0 0 -6.56616 0.001 0 0 0 0.001 0 0 -6.5686 0.001 0 0 0 0.001 0 0 -6.57104 0.001 0 0 0 0.001 0 0 -6.57349 0.001 0 0 0 0.001 0 0 -6.57593 0.001 0 0 0 0.001 0 0 -6.57837 0.001 0 0 0 0.001 0 0 -6.58081 0.001 0 0 0 0.001 0 0 -6.58325 0.001 0 0 0 0.001 0 0 -6.58569 0.001 0 0 0 0.001 0 0 -6.58813 0.001 0 0 0 0.001 0 0 -6.59058 0.001 0 0 0 0.001 0 0 -6.59302 0.001 0 0 0 0.001 0 0 -6.59546 0.001 0 0 0 0.001 0 0 -6.5979 0.001 0 0 0 0.001 0 0 -6.60034 0.001 0 0 0 0.001 0 0 -6.60278 0.001 0 0 0 0.001 0 0 -6.60522 0.001 0 0 0 0.001 0 0 -6.60767 0.001 0 0 0 0.001 0 0 -6.61011 0.001 0 0 0 0.001 0 0 -6.61255 0.001 0 0 0 0.001 0 0 -6.61499 0.001 0 0 0 0.001 0 0 -6.61743 0.001 0 0 0 0.001 0 0 -6.61987 0.001 0 0 0 0.001 0 0 -6.62231 0.001 0 0 0 0.001 0 0 -6.62476 0.001 0 0 0 0.001 0 0 -6.6272 0.001 0 0 0 0.001 0 0 -6.62964 0.001 0 0 0 0.001 0 0 -6.63208 0.001 0 0 0 0.001 0 0 -6.63452 0.001 0 0 0 0.001 0 0 -6.63696 0.001 0 0 0 0.001 0 0 -6.6394 0.001 0 0 0 0.001 0 0 -6.64185 0.001 0 0 0 0.001 0 0 -6.64429 0.001 0 0 0 0.001 0 0 -6.64673 0.001 0 0 0 0.001 0 0 -6.64917 0.001 0 0 0 0.001 0 0 -6.65161 0.001 0 0 0 0.001 0 0 -6.65405 0.001 0 0 0 0.001 0 0 -6.65649 0.001 0 0 0 0.001 0 0 -6.65894 0.001 0 0 0 0.001 0 0 -6.66138 0.001 0 0 0 0.001 0 0 -6.66382 0.001 0 0 0 0.001 0 0 -6.66626 0.001 0 0 0 0.001 0 0 -6.6687 0.001 0 0 0 0.001 0 0 -6.67114 0.001 0 0 0 0.001 0 0 -6.67358 0.001 0 0 0 0.001 0 0 -6.67603 0.001 0 0 0 0.001 0 0 -6.67847 0.001 0 0 0 0.001 0 0 -6.68091 0.001 0 0 0 0.001 0 0 -6.68335 0.001 0 0 0 0.001 0 0 -6.68579 0.001 0 0 0 0.001 0 0 -6.68823 0.001 0 0 0 0.001 0 0 -6.69067 0.001 0 0 0 0.001 0 0 -6.69312 0.001 0 0 0 0.001 0 0 -6.69556 0.001 0 0 0 0.001 0 0 -6.698 0.001 0 0 0 0.001 0 0 -6.70044 0.001 0 0 0 0.001 0 0 -6.70288 0.001 0 0 0 0.001 0 0 -6.70532 0.001 0 0 0 0.001 0 0 -6.70776 0.001 0 0 0 0.001 0 0 -6.71021 0.001 0 0 0 0.001 0 0 -6.71265 0.001 0 0 0 0.001 0 0 -6.71509 0.001 0 0 0 0.001 0 0 -6.71753 0.001 0 0 0 0.001 0 0 -6.71997 0.001 0 0 0 0.001 0 0 -6.72241 0.001 0 0 0 0.001 0 0 -6.72485 0.001 0 0 0 0.001 0 0 -6.72729 0.001 0 0 0 0.001 0 0 -6.72974 0.001 0 0 0 0.001 0 0 -6.73218 0.001 0 0 0 0.001 0 0 -6.73462 0.001 0 0 0 0.001 0 0 -6.73706 0.001 0 0 0 0.001 0 0 -6.7395 0.001 0 0 0 0.001 0 0 -6.74194 0.001 0 0 0 0.001 0 0 -6.74438 0.001 0 0 0 0.001 0 0 -6.74683 0.001 0 0 0 0.001 0 0 -6.74927 0.001 0 0 0 0.001 0 0 -6.75171 0.001 0 0 0 0.001 0 0 -6.75415 0.001 0 0 0 0.001 0 0 -6.75659 0.001 0 0 0 0.001 0 0 -6.75903 0.001 0 0 0 0.001 0 0 -6.76147 0.001 0 0 0 0.001 0 0 -6.76392 0.001 0 0 0 0.001 0 0 -6.76636 0.001 0 0 0 0.001 0 0 -6.7688 0.001 0 0 0 0.001 0 0 -6.77124 0.001 0 0 0 0.001 0 0 -6.77368 0.001 0 0 0 0.001 0 0 -6.77612 0.001 0 0 0 0.001 0 0 -6.77856 0.001 0 0 0 0.001 0 0 -6.78101 0.001 0 0 0 0.001 0 0 -6.78345 0.001 0 0 0 0.001 0 0 -6.78589 0.001 0 0 0 0.001 0 0 -6.78833 0.001 0 0 0 0.001 0 0 -6.79077 0.001 0 0 0 0.001 0 0 -6.79321 0.001 0 0 0 0.001 0 0 -6.79565 0.001 0 0 0 0.001 0 0 -6.7981 0.001 0 0 0 0.001 0 0 -6.80054 0.001 0 0 0 0.001 0 0 -6.80298 0.001 0 0 0 0.001 0 0 -6.80542 0.001 0 0 0 0.001 0 0 -6.80786 0.001 0 0 0 0.001 0 0 -6.8103 0.001 0 0 0 0.001 0 0 -6.81274 0.001 0 0 0 0.001 0 0 -6.81519 0.001 0 0 0 0.001 0 0 -6.81763 0.001 0 0 0 0.001 0 0 -6.82007 0.001 0 0 0 0.001 0 0 -6.82251 0.001 0 0 0 0.001 0 0 -6.82495 0.001 0 0 0 0.001 0 0 -6.82739 0.001 0 0 0 0.001 0 0 -6.82983 0.001 0 0 0 0.001 0 0 -6.83228 0.001 0 0 0 0.001 0 0 -6.83472 0.001 0 0 0 0.001 0 0 -6.83716 0.001 0 0 0 0.001 0 0 -6.8396 0.001 0 0 0 0.001 0 0 -6.84204 0.001 0 0 0 0.001 0 0 -6.84448 0.001 0 0 0 0.001 0 0 -6.84692 0.001 0 0 0 0.001 0 0 -6.84937 0.001 0 0 0 0.001 0 0 -6.85181 0.001 0 0 0 0.001 0 0 -6.85425 0.001 0 0 0 0.001 0 0 -6.85669 0.001 0 0 0 0.001 0 0 -6.85913 0.001 0 0 0 0.001 0 0 -6.86157 0.001 0 0 0 0.001 0 0 -6.86401 0.001 0 0 0 0.001 0 0 -6.86646 0.001 0 0 0 0.001 0 0 -6.8689 0.001 0 0 0 0.001 0 0 -6.87134 0.001 0 0 0 0.001 0 0 -6.87378 0.001 0 0 0 0.001 0 0 -6.87622 0.001 0 0 0 0.001 0 0 -6.87866 0.001 0 0 0 0.001 0 0 -6.8811 0.001 0 0 0 0.001 0 0 -6.88354 0.001 0 0 0 0.001 0 0 -6.88599 0.001 0 0 0 0.001 0 0 -6.88843 0.001 0 0 0 0.001 0 0 -6.89087 0.001 0 0 0 0.001 0 0 -6.89331 0.001 0 0 0 0.001 0 0 -6.89575 0.001 0 0 0 0.001 0 0 -6.89819 0.001 0 0 0 0.001 0 0 -6.90063 0.001 0 0 0 0.001 0 0 -6.90308 0.001 0 0 0 0.001 0 0 -6.90552 0.001 0 0 0 0.001 0 0 -6.90796 0.001 0 0 0 0.001 0 0 -6.9104 0.001 0 0 0 0.001 0 0 -6.91284 0.001 0 0 0 0.001 0 0 -6.91528 0.001 0 0 0 0.001 0 0 -6.91772 0.001 0 0 0 0.001 0 0 -6.92017 0.001 0 0 0 0.001 0 0 -6.92261 0.001 0 0 0 0.001 0 0 -6.92505 0.001 0 0 0 0.001 0 0 -6.92749 0.001 0 0 0 0.001 0 0 -6.92993 0.001 0 0 0 0.001 0 0 -6.93237 0.001 0 0 0 0.001 0 0 -6.93481 0.001 0 0 0 0.001 0 0 -6.93726 0.001 0 0 0 0.001 0 0 -6.9397 0.001 0 0 0 0.001 0 0 -6.94214 0.001 0 0 0 0.001 0 0 -6.94458 0.001 0 0 0 0.001 0 0 -6.94702 0.001 0 0 0 0.001 0 0 -6.94946 0.001 0 0 0 0.001 0 0 -6.9519 0.001 0 0 0 0.001 0 0 -6.95435 0.001 0 0 0 0.001 0 0 -6.95679 0.001 0 0 0 0.001 0 0 -6.95923 0.001 0 0 0 0.001 0 0 -6.96167 0.001 0 0 0 0.001 0 0 -6.96411 0.001 0 0 0 0.001 0 0 -6.96655 0.001 0 0 0 0.001 0 0 -6.96899 0.001 0 0 0 0.001 0 0 -6.97144 0.001 0 0 0 0.001 0 0 -6.97388 0.001 0 0 0 0.001 0 0 -6.97632 0.001 0 0 0 0.001 0 0 -6.97876 0.001 0 0 0 0.001 0 0 -6.9812 0.001 0 0 0 0.001 0 0 -6.98364 0.001 0 0 0 0.001 0 0 -6.98608 0.001 0 0 0 0.001 0 0 -6.98853 0.001 0 0 0 0.001 0 0 -6.99097 0.001 0 0 0 0.001 0 0 -6.99341 0.001 0 0 0 0.001 0 0 -6.99585 0.001 0 0 0 0.001 0 0 -6.99829 0.001 0 0 0 0.001 0 0 -7.00073 0.001 0 0 0 0.001 0 0 -7.00317 0.001 0 0 0 0.001 0 0 -7.00562 0.001 0 0 0 0.001 0 0 -7.00806 0.001 0 0 0 0.001 0 0 -7.0105 0.001 0 0 0 0.001 0 0 -7.01294 0.001 0 0 0 0.001 0 0 -7.01538 0.001 0 0 0 0.001 0 0 -7.01782 0.001 0 0 0 0.001 0 0 -7.02026 0.001 0 0 0 0.001 0 0 -7.02271 0.001 0 0 0 0.001 0 0 -7.02515 0.001 0 0 0 0.001 0 0 -7.02759 0.001 0 0 0 0.001 0 0 -7.03003 0.001 0 0 0 0.001 0 0 -7.03247 0.001 0 0 0 0.001 0 0 -7.03491 0.001 0 0 0 0.001 0 0 -7.03735 0.001 0 0 0 0.001 0 0 -7.03979 0.001 0 0 0 0.001 0 0 -7.04224 0.001 0 0 0 0.001 0 0 -7.04468 0.001 0 0 0 0.001 0 0 -7.04712 0.001 0 0 0 0.001 0 0 -7.04956 0.001 0 0 0 0.001 0 0 -7.052 0.001 0 0 0 0.001 0 0 -7.05444 0.001 0 0 0 0.001 0 0 -7.05688 0.001 0 0 0 0.001 0 0 -7.05933 0.001 0 0 0 0.001 0 0 -7.06177 0.001 0 0 0 0.001 0 0 -7.06421 0.001 0 0 0 0.001 0 0 -7.06665 0.001 0 0 0 0.001 0 0 -7.06909 0.001 0 0 0 0.001 0 0 -7.07153 0.001 0 0 0 0.001 0 0 -7.07397 0.001 0 0 0 0.001 0 0 -7.07642 0.001 0 0 0 0.001 0 0 -7.07886 0.001 0 0 0 0.001 0 0 -7.0813 0.001 0 0 0 0.001 0 0 -7.08374 0.001 0 0 0 0.001 0 0 -7.08618 0.001 0 0 0 0.001 0 0 -7.08862 0.001 0 0 0 0.001 0 0 -7.09106 0.001 0 0 0 0.001 0 0 -7.09351 0.001 0 0 0 0.001 0 0 -7.09595 0.001 0 0 0 0.001 0 0 -7.09839 0.001 0 0 0 0.001 0 0 -7.10083 0.001 0 0 0 0.001 0 0 -7.10327 0.001 0 0 0 0.001 0 0 -7.10571 0.001 0 0 0 0.001 0 0 -7.10815 0.001 0 0 0 0.001 0 0 -7.1106 0.001 0 0 0 0.001 0 0 -7.11304 0.001 0 0 0 0.001 0 0 -7.11548 0.001 0 0 0 0.001 0 0 -7.11792 0.001 0 0 0 0.001 0 0 -7.12036 0.001 0 0 0 0.001 0 0 -7.1228 0.001 0 0 0 0.001 0 0 -7.12524 0.001 0 0 0 0.001 0 0 -7.12769 0.001 0 0 0 0.001 0 0 -7.13013 0.001 0 0 0 0.001 0 0 -7.13257 0.001 0 0 0 0.001 0 0 -7.13501 0.001 0 0 0 0.001 0 0 -7.13745 0.001 0 0 0 0.001 0 0 -7.13989 0.001 0 0 0 0.001 0 0 -7.14233 0.001 0 0 0 0.001 0 0 -7.14478 0.001 0 0 0 0.001 0 0 -7.14722 0.001 0 0 0 0.001 0 0 -7.14966 0.001 0 0 0 0.001 0 0 -7.1521 0.001 0 0 0 0.001 0 0 -7.15454 0.001 0 0 0 0.001 0 0 -7.15698 0.001 0 0 0 0.001 0 0 -7.15942 0.001 0 0 0 0.001 0 0 -7.16187 0.001 0 0 0 0.001 0 0 -7.16431 0.001 0 0 0 0.001 0 0 -7.16675 0.001 0 0 0 0.001 0 0 -7.16919 0.001 0 0 0 0.001 0 0 -7.17163 0.001 0 0 0 0.001 0 0 -7.17407 0.001 0 0 0 0.001 0 0 -7.17651 0.001 0 0 0 0.001 0 0 -7.17896 0.001 0 0 0 0.001 0 0 -7.1814 0.001 0 0 0 0.001 0 0 -7.18384 0.001 0 0 0 0.001 0 0 -7.18628 0.001 0 0 0 0.001 0 0 -7.18872 0.001 0 0 0 0.001 0 0 -7.19116 0.001 0 0 0 0.001 0 0 -7.1936 0.001 0 0 0 0.001 0 0 -7.19604 0.001 0 0 0 0.001 0 0 -7.19849 0.001 0 0 0 0.001 0 0 -7.20093 0.001 0 0 0 0.001 0 0 -7.20337 0.001 0 0 0 0.001 0 0 -7.20581 0.001 0 0 0 0.001 0 0 -7.20825 0.001 0 0 0 0.001 0 0 -7.21069 0.001 0 0 0 0.001 0 0 -7.21313 0.001 0 0 0 0.001 0 0 -7.21558 0.001 0 0 0 0.001 0 0 -7.21802 0.001 0 0 0 0.001 0 0 -7.22046 0.001 0 0 0 0.001 0 0 -7.2229 0.001 0 0 0 0.001 0 0 -7.22534 0.001 0 0 0 0.001 0 0 -7.22778 0.001 0 0 0 0.001 0 0 -7.23022 0.001 0 0 0 0.001 0 0 -7.23267 0.001 0 0 0 0.001 0 0 -7.23511 0.001 0 0 0 0.001 0 0 -7.23755 0.001 0 0 0 0.001 0 0 -7.23999 0.001 0 0 0 0.001 0 0 -7.24243 0.001 0 0 0 0.001 0 0 -7.24487 0.001 0 0 0 0.001 0 0 -7.24731 0.001 0 0 0 0.001 0 0 -7.24976 0.001 0 0 0 0.001 0 0 -7.2522 0.001 0 0 0 0.001 0 0 -7.25464 0.001 0 0 0 0.001 0 0 -7.25708 0.001 0 0 0 0.001 0 0 -7.25952 0.001 0 0 0 0.001 0 0 -7.26196 0.001 0 0 0 0.001 0 0 -7.2644 0.001 0 0 0 0.001 0 0 -7.26685 0.001 0 0 0 0.001 0 0 -7.26929 0.001 0 0 0 0.001 0 0 -7.27173 0.001 0 0 0 0.001 0 0 -7.27417 0.001 0 0 0 0.001 0 0 -7.27661 0.001 0 0 0 0.001 0 0 -7.27905 0.001 0 0 0 0.001 0 0 -7.28149 0.001 0 0 0 0.001 0 0 -7.28394 0.001 0 0 0 0.001 0 0 -7.28638 0.001 0 0 0 0.001 0 0 -7.28882 0.001 0 0 0 0.001 0 0 -7.29126 0.001 0 0 0 0.001 0 0 -7.2937 0.001 0 0 0 0.001 0 0 -7.29614 0.001 0 0 0 0.001 0 0 -7.29858 0.001 0 0 0 0.001 0 0 -7.30103 0.001 0 0 0 0.001 0 0 -7.30347 0.001 0 0 0 0.001 0 0 -7.30591 0.001 0 0 0 0.001 0 0 -7.30835 0.001 0 0 0 0.001 0 0 -7.31079 0.001 0 0 0 0.001 0 0 -7.31323 0.001 0 0 0 0.001 0 0 -7.31567 0.001 0 0 0 0.001 0 0 -7.31812 0.001 0 0 0 0.001 0 0 -7.32056 0.001 0 0 0 0.001 0 0 -7.323 0.001 0 0 0 0.001 0 0 -7.32544 0.001 0 0 0 0.001 0 0 -7.32788 0.001 0 0 0 0.001 0 0 -7.33032 0.001 0 0 0 0.001 0 0 -7.33276 0.001 0 0 0 0.001 0 0 -7.33521 0.001 0 0 0 0.001 0 0 -7.33765 0.001 0 0 0 0.001 0 0 -7.34009 0.001 0 0 0 0.001 0 0 -7.34253 0.001 0 0 0 0.001 0 0 -7.34497 0.001 0 0 0 0.001 0 0 -7.34741 0.001 0 0 0 0.001 0 0 -7.34985 0.001 0 0 0 0.001 0 0 -7.35229 0.001 0 0 0 0.001 0 0 -7.35474 0.001 0 0 0 0.001 0 0 -7.35718 0.001 0 0 0 0.001 0 0 -7.35962 0.001 0 0 0 0.001 0 0 -7.36206 0.001 0 0 0 0.001 0 0 -7.3645 0.001 0 0 0 0.001 0 0 -7.36694 0.001 0 0 0 0.001 0 0 -7.36938 0.001 0 0 0 0.001 0 0 -7.37183 0.001 0 0 0 0.001 0 0 -7.37427 0.001 0 0 0 0.001 0 0 -7.37671 0.001 0 0 0 0.001 0 0 -7.37915 0.001 0 0 0 0.001 0 0 -7.38159 0.001 0 0 0 0.001 0 0 -7.38403 0.001 0 0 0 0.001 0 0 -7.38647 0.001 0 0 0 0.001 0 0 -7.38892 0.001 0 0 0 0.001 0 0 -7.39136 0.001 0 0 0 0.001 0 0 -7.3938 0.001 0 0 0 0.001 0 0 -7.39624 0.001 0 0 0 0.001 0 0 -7.39868 0.001 0 0 0 0.001 0 0 -7.40112 0.001 0 0 0 0.001 0 0 -7.40356 0.001 0 0 0 0.001 0 0 -7.40601 0.001 0 0 0 0.001 0 0 -7.40845 0.001 0 0 0 0.001 0 0 -7.41089 0.001 0 0 0 0.001 0 0 -7.41333 0.001 0 0 0 0.001 0 0 -7.41577 0.001 0 0 0 0.001 0 0 -7.41821 0.001 0 0 0 0.001 0 0 -7.42065 0.001 0 0 0 0.001 0 0 -7.4231 0.001 0 0 0 0.001 0 0 -7.42554 0.001 0 0 0 0.001 0 0 -7.42798 0.001 0 0 0 0.001 0 0 -7.43042 0.001 0 0 0 0.001 0 0 -7.43286 0.001 0 0 0 0.001 0 0 -7.4353 0.001 0 0 0 0.001 0 0 -7.43774 0.001 0 0 0 0.001 0 0 -7.44019 0.001 0 0 0 0.001 0 0 -7.44263 0.001 0 0 0 0.001 0 0 -7.44507 0.001 0 0 0 0.001 0 0 -7.44751 0.001 0 0 0 0.001 0 0 -7.44995 0.001 0 0 0 0.001 0 0 -7.45239 0.001 0 0 0 0.001 0 0 -7.45483 0.001 0 0 0 0.001 0 0 -7.45728 0.001 0 0 0 0.001 0 0 -7.45972 0.001 0 0 0 0.001 0 0 -7.46216 0.001 0 0 0 0.001 0 0 -7.4646 0.001 0 0 0 0.001 0 0 -7.46704 0.001 0 0 0 0.001 0 0 -7.46948 0.001 0 0 0 0.001 0 0 -7.47192 0.001 0 0 0 0.001 0 0 -7.47437 0.001 0 0 0 0.001 0 0 -7.47681 0.001 0 0 0 0.001 0 0 -7.47925 0.001 0 0 0 0.001 0 0 -7.48169 0.001 0 0 0 0.001 0 0 -7.48413 0.001 0 0 0 0.001 0 0 -7.48657 0.001 0 0 0 0.001 0 0 -7.48901 0.001 0 0 0 0.001 0 0 -7.49146 0.001 0 0 0 0.001 0 0 -7.4939 0.001 0 0 0 0.001 0 0 -7.49634 0.001 0 0 0 0.001 0 0 -7.49878 0.001 0 0 0 0.001 0 0 -7.50122 0.001 0 0 0 0.001 0 0 -7.50366 0.001 0 0 0 0.001 0 0 -7.5061 0.001 0 0 0 0.001 0 0 -7.50854 0.001 0 0 0 0.001 0 0 -7.51099 0.001 0 0 0 0.001 0 0 -7.51343 0.001 0 0 0 0.001 0 0 -7.51587 0.001 0 0 0 0.001 0 0 -7.51831 0.001 0 0 0 0.001 0 0 -7.52075 0.001 0 0 0 0.001 0 0 -7.52319 0.001 0 0 0 0.001 0 0 -7.52563 0.001 0 0 0 0.001 0 0 -7.52808 0.001 0 0 0 0.001 0 0 -7.53052 0.001 0 0 0 0.001 0 0 -7.53296 0.001 0 0 0 0.001 0 0 -7.5354 0.001 0 0 0 0.001 0 0 -7.53784 0.001 0 0 0 0.001 0 0 -7.54028 0.001 0 0 0 0.001 0 0 -7.54272 0.001 0 0 0 0.001 0 0 -7.54517 0.001 0 0 0 0.001 0 0 -7.54761 0.001 0 0 0 0.001 0 0 -7.55005 0.001 0 0 0 0.001 0 0 -7.55249 0.001 0 0 0 0.001 0 0 -7.55493 0.001 0 0 0 0.001 0 0 -7.55737 0.001 0 0 0 0.001 0 0 -7.55981 0.001 0 0 0 0.001 0 0 -7.56226 0.001 0 0 0 0.001 0 0 -7.5647 0.001 0 0 0 0.001 0 0 -7.56714 0.001 0 0 0 0.001 0 0 -7.56958 0.001 0 0 0 0.001 0 0 -7.57202 0.001 0 0 0 0.001 0 0 -7.57446 0.001 0 0 0 0.001 0 0 -7.5769 0.001 0 0 0 0.001 0 0 -7.57935 0.001 0 0 0 0.001 0 0 -7.58179 0.001 0 0 0 0.001 0 0 -7.58423 0.001 0 0 0 0.001 0 0 -7.58667 0.001 0 0 0 0.001 0 0 -7.58911 0.001 0 0 0 0.001 0 0 -7.59155 0.001 0 0 0 0.001 0 0 -7.59399 0.001 0 0 0 0.001 0 0 -7.59644 0.001 0 0 0 0.001 0 0 -7.59888 0.001 0 0 0 0.001 0 0 -7.60132 0.001 0 0 0 0.001 0 0 -7.60376 0.001 0 0 0 0.001 0 0 -7.6062 0.001 0 0 0 0.001 0 0 -7.60864 0.001 0 0 0 0.001 0 0 -7.61108 0.001 0 0 0 0.001 0 0 -7.61353 0.001 0 0 0 0.001 0 0 -7.61597 0.001 0 0 0 0.001 0 0 -7.61841 0.001 0 0 0 0.001 0 0 -7.62085 0.001 0 0 0 0.001 0 0 -7.62329 0.001 0 0 0 0.001 0 0 -7.62573 0.001 0 0 0 0.001 0 0 -7.62817 0.001 0 0 0 0.001 0 0 -7.63062 0.001 0 0 0 0.001 0 0 -7.63306 0.001 0 0 0 0.001 0 0 -7.6355 0.001 0 0 0 0.001 0 0 -7.63794 0.001 0 0 0 0.001 0 0 -7.64038 0.001 0 0 0 0.001 0 0 -7.64282 0.001 0 0 0 0.001 0 0 -7.64526 0.001 0 0 0 0.001 0 0 -7.64771 0.001 0 0 0 0.001 0 0 -7.65015 0.001 0 0 0 0.001 0 0 -7.65259 0.001 0 0 0 0.001 0 0 -7.65503 0.001 0 0 0 0.001 0 0 -7.65747 0.001 0 0 0 0.001 0 0 -7.65991 0.001 0 0 0 0.001 0 0 -7.66235 0.001 0 0 0 0.001 0 0 -7.66479 0.001 0 0 0 0.001 0 0 -7.66724 0.001 0 0 0 0.001 0 0 -7.66968 0.001 0 0 0 0.001 0 0 -7.67212 0.001 0 0 0 0.001 0 0 -7.67456 0.001 0 0 0 0.001 0 0 -7.677 0.001 0 0 0 0.001 0 0 -7.67944 0.001 0 0 0 0.001 0 0 -7.68188 0.001 0 0 0 0.001 0 0 -7.68433 0.001 0 0 0 0.001 0 0 -7.68677 0.001 0 0 0 0.001 0 0 -7.68921 0.001 0 0 0 0.001 0 0 -7.69165 0.001 0 0 0 0.001 0 0 -7.69409 0.001 0 0 0 0.001 0 0 -7.69653 0.001 0 0 0 0.001 0 0 -7.69897 0.001 0 0 0 0.001 0 0 -7.70142 0.001 0 0 0 0.001 0 0 -7.70386 0.001 0 0 0 0.001 0 0 -7.7063 0.001 0 0 0 0.001 0 0 -7.70874 0.001 0 0 0 0.001 0 0 -7.71118 0.001 0 0 0 0.001 0 0 -7.71362 0.001 0 0 0 0.001 0 0 -7.71606 0.001 0 0 0 0.001 0 0 -7.71851 0.001 0 0 0 0.001 0 0 -7.72095 0.001 0 0 0 0.001 0 0 -7.72339 0.001 0 0 0 0.001 0 0 -7.72583 0.001 0 0 0 0.001 0 0 -7.72827 0.001 0 0 0 0.001 0 0 -7.73071 0.001 0 0 0 0.001 0 0 -7.73315 0.001 0 0 0 0.001 0 0 -7.7356 0.001 0 0 0 0.001 0 0 -7.73804 0.001 0 0 0 0.001 0 0 -7.74048 0.001 0 0 0 0.001 0 0 -7.74292 0.001 0 0 0 0.001 0 0 -7.74536 0.001 0 0 0 0.001 0 0 -7.7478 0.001 0 0 0 0.001 0 0 -7.75024 0.001 0 0 0 0.001 0 0 -7.75269 0.001 0 0 0 0.001 0 0 -7.75513 0.001 0 0 0 0.001 0 0 -7.75757 0.001 0 0 0 0.001 0 0 -7.76001 0.001 0 0 0 0.001 0 0 -7.76245 0.001 0 0 0 0.001 0 0 -7.76489 0.001 0 0 0 0.001 0 0 -7.76733 0.001 0 0 0 0.001 0 0 -7.76978 0.001 0 0 0 0.001 0 0 -7.77222 0.001 0 0 0 0.001 0 0 -7.77466 0.001 0 0 0 0.001 0 0 -7.7771 0.001 0 0 0 0.001 0 0 -7.77954 0.001 0 0 0 0.001 0 0 -7.78198 0.001 0 0 0 0.001 0 0 -7.78442 0.001 0 0 0 0.001 0 0 -7.78687 0.001 0 0 0 0.001 0 0 -7.78931 0.001 0 0 0 0.001 0 0 -7.79175 0.001 0 0 0 0.001 0 0 -7.79419 0.001 0 0 0 0.001 0 0 -7.79663 0.001 0 0 0 0.001 0 0 -7.79907 0.001 0 0 0 0.001 0 0 -7.80151 0.001 0 0 0 0.001 0 0 -7.80396 0.001 0 0 0 0.001 0 0 -7.8064 0.001 0 0 0 0.001 0 0 -7.80884 0.001 0 0 0 0.001 0 0 -7.81128 0.001 0 0 0 0.001 0 0 -7.81372 0.001 0 0 0 0.001 0 0 -7.81616 0.001 0 0 0 0.001 0 0 -7.8186 0.001 0 0 0 0.001 0 0 -7.82104 0.001 0 0 0 0.001 0 0 -7.82349 0.001 0 0 0 0.001 0 0 -7.82593 0.001 0 0 0 0.001 0 0 -7.82837 0.001 0 0 0 0.001 0 0 -7.83081 0.001 0 0 0 0.001 0 0 -7.83325 0.001 0 0 0 0.001 0 0 -7.83569 0.001 0 0 0 0.001 0 0 -7.83813 0.001 0 0 0 0.001 0 0 -7.84058 0.001 0 0 0 0.001 0 0 -7.84302 0.001 0 0 0 0.001 0 0 -7.84546 0.001 0 0 0 0.001 0 0 -7.8479 0.001 0 0 0 0.001 0 0 -7.85034 0.001 0 0 0 0.001 0 0 -7.85278 0.001 0 0 0 0.001 0 0 -7.85522 0.001 0 0 0 0.001 0 0 -7.85767 0.001 0 0 0 0.001 0 0 -7.86011 0.001 0 0 0 0.001 0 0 -7.86255 0.001 0 0 0 0.001 0 0 -7.86499 0.001 0 0 0 0.001 0 0 -7.86743 0.001 0 0 0 0.001 0 0 -7.86987 0.001 0 0 0 0.001 0 0 -7.87231 0.001 0 0 0 0.001 0 0 -7.87476 0.001 0 0 0 0.001 0 0 -7.8772 0.001 0 0 0 0.001 0 0 -7.87964 0.001 0 0 0 0.001 0 0 -7.88208 0.001 0 0 0 0.001 0 0 -7.88452 0.001 0 0 0 0.001 0 0 -7.88696 0.001 0 0 0 0.001 0 0 -7.8894 0.001 0 0 0 0.001 0 0 -7.89185 0.001 0 0 0 0.001 0 0 -7.89429 0.001 0 0 0 0.001 0 0 -7.89673 0.001 0 0 0 0.001 0 0 -7.89917 0.001 0 0 0 0.001 0 0 -7.90161 0.001 0 0 0 0.001 0 0 -7.90405 0.001 0 0 0 0.001 0 0 -7.90649 0.001 0 0 0 0.001 0 0 -7.90894 0.001 0 0 0 0.001 0 0 -7.91138 0.001 0 0 0 0.001 0 0 -7.91382 0.001 0 0 0 0.001 0 0 -7.91626 0.001 0 0 0 0.001 0 0 -7.9187 0.001 0 0 0 0.001 0 0 -7.92114 0.001 0 0 0 0.001 0 0 -7.92358 0.001 0 0 0 0.001 0 0 -7.92603 0.001 0 0 0 0.001 0 0 -7.92847 0.001 0 0 0 0.001 0 0 -7.93091 0.001 0 0 0 0.001 0 0 -7.93335 0.001 0 0 0 0.001 0 0 -7.93579 0.001 0 0 0 0.001 0 0 -7.93823 0.001 0 0 0 0.001 0 0 -7.94067 0.001 0 0 0 0.001 0 0 -7.94312 0.001 0 0 0 0.001 0 0 -7.94556 0.001 0 0 0 0.001 0 0 -7.948 0.001 0 0 0 0.001 0 0 -7.95044 0.001 0 0 0 0.001 0 0 -7.95288 0.001 0 0 0 0.001 0 0 -7.95532 0.001 0 0 0 0.001 0 0 -7.95776 0.001 0 0 0 0.001 0 0 -7.96021 0.001 0 0 0 0.001 0 0 -7.96265 0.001 0 0 0 0.001 0 0 -7.96509 0.001 0 0 0 0.001 0 0 -7.96753 0.001 0 0 0 0.001 0 0 -7.96997 0.001 0 0 0 0.001 0 0 -7.97241 0.001 0 0 0 0.001 0 0 -7.97485 0.001 0 0 0 0.001 0 0 -7.97729 0.001 0 0 0 0.001 0 0 -7.97974 0.001 0 0 0 0.001 0 0 -7.98218 0.001 0 0 0 0.001 0 0 -7.98462 0.001 0 0 0 0.001 0 0 -7.98706 0.001 0 0 0 0.001 0 0 -7.9895 0.001 0 0 0 0.001 0 0 -7.99194 0.001 0 0 0 0.001 0 0 -7.99438 0.001 0 0 0 0.001 0 0 -7.99683 0.001 0 0 0 0.001 0 0 -7.99927 0.001 0 0 0 0.001 0 0 -8.00171 0.001 0 0 0 0.001 0 0 -8.00415 0.001 0 0 0 0.001 0 0 -8.00659 0.001 0 0 0 0.001 0 0 -8.00903 0.001 0 0 0 0.001 0 0 -8.01147 0.001 0 0 0 0.001 0 0 -8.01392 0.001 0 0 0 0.001 0 0 -8.01636 0.001 0 0 0 0.001 0 0 -8.0188 0.001 0 0 0 0.001 0 0 -8.02124 0.001 0 0 0 0.001 0 0 -8.02368 0.001 0 0 0 0.001 0 0 -8.02612 0.001 0 0 0 0.001 0 0 -8.02856 0.001 0 0 0 0.001 0 0 -8.03101 0.001 0 0 0 0.001 0 0 -8.03345 0.001 0 0 0 0.001 0 0 -8.03589 0.001 0 0 0 0.001 0 0 -8.03833 0.001 0 0 0 0.001 0 0 -8.04077 0.001 0 0 0 0.001 0 0 -8.04321 0.001 0 0 0 0.001 0 0 -8.04565 0.001 0 0 0 0.001 0 0 -8.0481 0.001 0 0 0 0.001 0 0 -8.05054 0.001 0 0 0 0.001 0 0 -8.05298 0.001 0 0 0 0.001 0 0 -8.05542 0.001 0 0 0 0.001 0 0 -8.05786 0.001 0 0 0 0.001 0 0 -8.0603 0.001 0 0 0 0.001 0 0 -8.06274 0.001 0 0 0 0.001 0 0 -8.06519 0.001 0 0 0 0.001 0 0 -8.06763 0.001 0 0 0 0.001 0 0 -8.07007 0.001 0 0 0 0.001 0 0 -8.07251 0.001 0 0 0 0.001 0 0 -8.07495 0.001 0 0 0 0.001 0 0 -8.07739 0.001 0 0 0 0.001 0 0 -8.07983 0.001 0 0 0 0.001 0 0 -8.08228 0.001 0 0 0 0.001 0 0 -8.08472 0.001 0 0 0 0.001 0 0 -8.08716 0.001 0 0 0 0.001 0 0 -8.0896 0.001 0 0 0 0.001 0 0 -8.09204 0.001 0 0 0 0.001 0 0 -8.09448 0.001 0 0 0 0.001 0 0 -8.09692 0.001 0 0 0 0.001 0 0 -8.09937 0.001 0 0 0 0.001 0 0 -8.10181 0.001 0 0 0 0.001 0 0 -8.10425 0.001 0 0 0 0.001 0 0 -8.10669 0.001 0 0 0 0.001 0 0 -8.10913 0.001 0 0 0 0.001 0 0 -8.11157 0.001 0 0 0 0.001 0 0 -8.11401 0.001 0 0 0 0.001 0 0 -8.11646 0.001 0 0 0 0.001 0 0 -8.1189 0.001 0 0 0 0.001 0 0 -8.12134 0.001 0 0 0 0.001 0 0 -8.12378 0.001 0 0 0 0.001 0 0 -8.12622 0.001 0 0 0 0.001 0 0 -8.12866 0.001 0 0 0 0.001 0 0 -8.1311 0.001 0 0 0 0.001 0 0 -8.13354 0.001 0 0 0 0.001 0 0 -8.13599 0.001 0 0 0 0.001 0 0 -8.13843 0.001 0 0 0 0.001 0 0 -8.14087 0.001 0 0 0 0.001 0 0 -8.14331 0.001 0 0 0 0.001 0 0 -8.14575 0.001 0 0 0 0.001 0 0 -8.14819 0.001 0 0 0 0.001 0 0 -8.15063 0.001 0 0 0 0.001 0 0 -8.15308 0.001 0 0 0 0.001 0 0 -8.15552 0.001 0 0 0 0.001 0 0 -8.15796 0.001 0 0 0 0.001 0 0 -8.1604 0.001 0 0 0 0.001 0 0 -8.16284 0.001 0 0 0 0.001 0 0 -8.16528 0.001 0 0 0 0.001 0 0 -8.16772 0.001 0 0 0 0.001 0 0 -8.17017 0.001 0 0 0 0.001 0 0 -8.17261 0.001 0 0 0 0.001 0 0 -8.17505 0.001 0 0 0 0.001 0 0 -8.17749 0.001 0 0 0 0.001 0 0 -8.17993 0.001 0 0 0 0.001 0 0 -8.18237 0.001 0 0 0 0.001 0 0 -8.18481 0.001 0 0 0 0.001 0 0 -8.18726 0.001 0 0 0 0.001 0 0 -8.1897 0.001 0 0 0 0.001 0 0 -8.19214 0.001 0 0 0 0.001 0 0 -8.19458 0.001 0 0 0 0.001 0 0 -8.19702 0.001 0 0 0 0.001 0 0 -8.19946 0.001 0 0 0 0.001 0 0 -8.2019 0.001 0 0 0 0.001 0 0 -8.20435 0.001 0 0 0 0.001 0 0 -8.20679 0.001 0 0 0 0.001 0 0 -8.20923 0.001 0 0 0 0.001 0 0 -8.21167 0.001 0 0 0 0.001 0 0 -8.21411 0.001 0 0 0 0.001 0 0 -8.21655 0.001 0 0 0 0.001 0 0 -8.21899 0.001 0 0 0 0.001 0 0 -8.22144 0.001 0 0 0 0.001 0 0 -8.22388 0.001 0 0 0 0.001 0 0 -8.22632 0.001 0 0 0 0.001 0 0 -8.22876 0.001 0 0 0 0.001 0 0 -8.2312 0.001 0 0 0 0.001 0 0 -8.23364 0.001 0 0 0 0.001 0 0 -8.23608 0.001 0 0 0 0.001 0 0 -8.23853 0.001 0 0 0 0.001 0 0 -8.24097 0.001 0 0 0 0.001 0 0 -8.24341 0.001 0 0 0 0.001 0 0 -8.24585 0.001 0 0 0 0.001 0 0 -8.24829 0.001 0 0 0 0.001 0 0 -8.25073 0.001 0 0 0 0.001 0 0 -8.25317 0.001 0 0 0 0.001 0 0 -8.25562 0.001 0 0 0 0.001 0 0 -8.25806 0.001 0 0 0 0.001 0 0 -8.2605 0.001 0 0 0 0.001 0 0 -8.26294 0.001 0 0 0 0.001 0 0 -8.26538 0.001 0 0 0 0.001 0 0 -8.26782 0.001 0 0 0 0.001 0 0 -8.27026 0.001 0 0 0 0.001 0 0 -8.27271 0.001 0 0 0 0.001 0 0 -8.27515 0.001 0 0 0 0.001 0 0 -8.27759 0.001 0 0 0 0.001 0 0 -8.28003 0.001 0 0 0 0.001 0 0 -8.28247 0.001 0 0 0 0.001 0 0 -8.28491 0.001 0 0 0 0.001 0 0 -8.28735 0.001 0 0 0 0.001 0 0 -8.28979 0.001 0 0 0 0.001 0 0 -8.29224 0.001 0 0 0 0.001 0 0 -8.29468 0.001 0 0 0 0.001 0 0 -8.29712 0.001 0 0 0 0.001 0 0 -8.29956 0.001 0 0 0 0.001 0 0 -8.302 0.001 0 0 0 0.001 0 0 -8.30444 0.001 0 0 0 0.001 0 0 -8.30688 0.001 0 0 0 0.001 0 0 -8.30933 0.001 0 0 0 0.001 0 0 -8.31177 0.001 0 0 0 0.001 0 0 -8.31421 0.001 0 0 0 0.001 0 0 -8.31665 0.001 0 0 0 0.001 0 0 -8.31909 0.001 0 0 0 0.001 0 0 -8.32153 0.001 0 0 0 0.001 0 0 -8.32397 0.001 0 0 0 0.001 0 0 -8.32642 0.001 0 0 0 0.001 0 0 -8.32886 0.001 0 0 0 0.001 0 0 -8.3313 0.001 0 0 0 0.001 0 0 -8.33374 0.001 0 0 0 0.001 0 0 -8.33618 0.001 0 0 0 0.001 0 0 -8.33862 0.001 0 0 0 0.001 0 0 -8.34106 0.001 0 0 0 0.001 0 0 -8.34351 0.001 0 0 0 0.001 0 0 -8.34595 0.001 0 0 0 0.001 0 0 -8.34839 0.001 0 0 0 0.001 0 0 -8.35083 0.001 0 0 0 0.001 0 0 -8.35327 0.001 0 0 0 0.001 0 0 -8.35571 0.001 0 0 0 0.001 0 0 -8.35815 0.001 0 0 0 0.001 0 0 -8.3606 0.001 0 0 0 0.001 0 0 -8.36304 0.001 0 0 0 0.001 0 0 -8.36548 0.001 0 0 0 0.001 0 0 -8.36792 0.001 0 0 0 0.001 0 0 -8.37036 0.001 0 0 0 0.001 0 0 -8.3728 0.001 0 0 0 0.001 0 0 -8.37524 0.001 0 0 0 0.001 0 0 -8.37769 0.001 0 0 0 0.001 0 0 -8.38013 0.001 0 0 0 0.001 0 0 -8.38257 0.001 0 0 0 0.001 0 0 -8.38501 0.001 0 0 0 0.001 0 0 -8.38745 0.001 0 0 0 0.001 0 0 -8.38989 0.001 0 0 0 0.001 0 0 -8.39233 0.001 0 0 0 0.001 0 0 -8.39478 0.001 0 0 0 0.001 0 0 -8.39722 0.001 0 0 0 0.001 0 0 -8.39966 0.001 0 0 0 0.001 0 0 -8.4021 0.001 0 0 0 0.001 0 0 -8.40454 0.001 0 0 0 0.001 0 0 -8.40698 0.001 0 0 0 0.001 0 0 -8.40942 0.001 0 0 0 0.001 0 0 -8.41187 0.001 0 0 0 0.001 0 0 -8.41431 0.001 0 0 0 0.001 0 0 -8.41675 0.001 0 0 0 0.001 0 0 -8.41919 0.001 0 0 0 0.001 0 0 -8.42163 0.001 0 0 0 0.001 0 0 -8.42407 0.001 0 0 0 0.001 0 0 -8.42651 0.001 0 0 0 0.001 0 0 -8.42896 0.001 0 0 0 0.001 0 0 -8.4314 0.001 0 0 0 0.001 0 0 -8.43384 0.001 0 0 0 0.001 0 0 -8.43628 0.001 0 0 0 0.001 0 0 -8.43872 0.001 0 0 0 0.001 0 0 -8.44116 0.001 0 0 0 0.001 0 0 -8.4436 0.001 0 0 0 0.001 0 0 -8.44604 0.001 0 0 0 0.001 0 0 -8.44849 0.001 0 0 0 0.001 0 0 -8.45093 0.001 0 0 0 0.001 0 0 -8.45337 0.001 0 0 0 0.001 0 0 -8.45581 0.001 0 0 0 0.001 0 0 -8.45825 0.001 0 0 0 0.001 0 0 -8.46069 0.001 0 0 0 0.001 0 0 -8.46313 0.001 0 0 0 0.001 0 0 -8.46558 0.001 0 0 0 0.001 0 0 -8.46802 0.001 0 0 0 0.001 0 0 -8.47046 0.001 0 0 0 0.001 0 0 -8.4729 0.001 0 0 0 0.001 0 0 -8.47534 0.001 0 0 0 0.001 0 0 -8.47778 0.001 0 0 0 0.001 0 0 -8.48022 0.001 0 0 0 0.001 0 0 -8.48267 0.001 0 0 0 0.001 0 0 -8.48511 0.001 0 0 0 0.001 0 0 -8.48755 0.001 0 0 0 0.001 0 0 -8.48999 0.001 0 0 0 0.001 0 0 -8.49243 0.001 0 0 0 0.001 0 0 -8.49487 0.001 0 0 0 0.001 0 0 -8.49731 0.001 0 0 0 0.001 0 0 -8.49976 0.001 0 0 0 0.001 0 0 -8.5022 0.001 0 0 0 0.001 0 0 -8.50464 0.001 0 0 0 0.001 0 0 -8.50708 0.001 0 0 0 0.001 0 0 -8.50952 0.001 0 0 0 0.001 0 0 -8.51196 0.001 0 0 0 0.001 0 0 -8.5144 0.001 0 0 0 0.001 0 0 -8.51685 0.001 0 0 0 0.001 0 0 -8.51929 0.001 0 0 0 0.001 0 0 -8.52173 0.001 0 0 0 0.001 0 0 -8.52417 0.001 0 0 0 0.001 0 0 -8.52661 0.001 0 0 0 0.001 0 0 -8.52905 0.001 0 0 0 0.001 0 0 -8.53149 0.001 0 0 0 0.001 0 0 -8.53394 0.001 0 0 0 0.001 0 0 -8.53638 0.001 0 0 0 0.001 0 0 -8.53882 0.001 0 0 0 0.001 0 0 -8.54126 0.001 0 0 0 0.001 0 0 -8.5437 0.001 0 0 0 0.001 0 0 -8.54614 0.001 0 0 0 0.001 0 0 -8.54858 0.001 0 0 0 0.001 0 0 -8.55103 0.001 0 0 0 0.001 0 0 -8.55347 0.001 0 0 0 0.001 0 0 -8.55591 0.001 0 0 0 0.001 0 0 -8.55835 0.001 0 0 0 0.001 0 0 -8.56079 0.001 0 0 0 0.001 0 0 -8.56323 0.001 0 0 0 0.001 0 0 -8.56567 0.001 0 0 0 0.001 0 0 -8.56812 0.001 0 0 0 0.001 0 0 -8.57056 0.001 0 0 0 0.001 0 0 -8.573 0.001 0 0 0 0.001 0 0 -8.57544 0.001 0 0 0 0.001 0 0 -8.57788 0.001 0 0 0 0.001 0 0 -8.58032 0.001 0 0 0 0.001 0 0 -8.58276 0.001 0 0 0 0.001 0 0 -8.58521 0.001 0 0 0 0.001 0 0 -8.58765 0.001 0 0 0 0.001 0 0 -8.59009 0.001 0 0 0 0.001 0 0 -8.59253 0.001 0 0 0 0.001 0 0 -8.59497 0.001 0 0 0 0.001 0 0 -8.59741 0.001 0 0 0 0.001 0 0 -8.59985 0.001 0 0 0 0.001 0 0 -8.60229 0.001 0 0 0 0.001 0 0 -8.60474 0.001 0 0 0 0.001 0 0 -8.60718 0.001 0 0 0 0.001 0 0 -8.60962 0.001 0 0 0 0.001 0 0 -8.61206 0.001 0 0 0 0.001 0 0 -8.6145 0.001 0 0 0 0.001 0 0 -8.61694 0.001 0 0 0 0.001 0 0 -8.61938 0.001 0 0 0 0.001 0 0 -8.62183 0.001 0 0 0 0.001 0 0 -8.62427 0.001 0 0 0 0.001 0 0 -8.62671 0.001 0 0 0 0.001 0 0 -8.62915 0.001 0 0 0 0.001 0 0 -8.63159 0.001 0 0 0 0.001 0 0 -8.63403 0.001 0 0 0 0.001 0 0 -8.63647 0.001 0 0 0 0.001 0 0 -8.63892 0.001 0 0 0 0.001 0 0 -8.64136 0.001 0 0 0 0.001 0 0 -8.6438 0.001 0 0 0 0.001 0 0 -8.64624 0.001 0 0 0 0.001 0 0 -8.64868 0.001 0 0 0 0.001 0 0 -8.65112 0.001 0 0 0 0.001 0 0 -8.65356 0.001 0 0 0 0.001 0 0 -8.65601 0.001 0 0 0 0.001 0 0 -8.65845 0.001 0 0 0 0.001 0 0 -8.66089 0.001 0 0 0 0.001 0 0 -8.66333 0.001 0 0 0 0.001 0 0 -8.66577 0.001 0 0 0 0.001 0 0 -8.66821 0.001 0 0 0 0.001 0 0 -8.67065 0.001 0 0 0 0.001 0 0 -8.6731 0.001 0 0 0 0.001 0 0 -8.67554 0.001 0 0 0 0.001 0 0 -8.67798 0.001 0 0 0 0.001 0 0 -8.68042 0.001 0 0 0 0.001 0 0 -8.68286 0.001 0 0 0 0.001 0 0 -8.6853 0.001 0 0 0 0.001 0 0 -8.68774 0.001 0 0 0 0.001 0 0 -8.69019 0.001 0 0 0 0.001 0 0 -8.69263 0.001 0 0 0 0.001 0 0 -8.69507 0.001 0 0 0 0.001 0 0 -8.69751 0.001 0 0 0 0.001 0 0 -8.69995 0.001 0 0 0 0.001 0 0 -8.70239 0.001 0 0 0 0.001 0 0 -8.70483 0.001 0 0 0 0.001 0 0 -8.70728 0.001 0 0 0 0.001 0 0 -8.70972 0.001 0 0 0 0.001 0 0 -8.71216 0.001 0 0 0 0.001 0 0 -8.7146 0.001 0 0 0 0.001 0 0 -8.71704 0.001 0 0 0 0.001 0 0 -8.71948 0.001 0 0 0 0.001 0 0 -8.72192 0.001 0 0 0 0.001 0 0 -8.72437 0.001 0 0 0 0.001 0 0 -8.72681 0.001 0 0 0 0.001 0 0 -8.72925 0.001 0 0 0 0.001 0 0 -8.73169 0.001 0 0 0 0.001 0 0 -8.73413 0.001 0 0 0 0.001 0 0 -8.73657 0.001 0 0 0 0.001 0 0 -8.73901 0.001 0 0 0 0.001 0 0 -8.74146 0.001 0 0 0 0.001 0 0 -8.7439 0.001 0 0 0 0.001 0 0 -8.74634 0.001 0 0 0 0.001 0 0 -8.74878 0.001 0 0 0 0.001 0 0 -8.75122 0.001 0 0 0 0.001 0 0 -8.75366 0.001 0 0 0 0.001 0 0 -8.7561 0.001 0 0 0 0.001 0 0 -8.75854 0.001 0 0 0 0.001 0 0 -8.76099 0.001 0 0 0 0.001 0 0 -8.76343 0.001 0 0 0 0.001 0 0 -8.76587 0.001 0 0 0 0.001 0 0 -8.76831 0.001 0 0 0 0.001 0 0 -8.77075 0.001 0 0 0 0.001 0 0 -8.77319 0.001 0 0 0 0.001 0 0 -8.77563 0.001 0 0 0 0.001 0 0 -8.77808 0.001 0 0 0 0.001 0 0 -8.78052 0.001 0 0 0 0.001 0 0 -8.78296 0.001 0 0 0 0.001 0 0 -8.7854 0.001 0 0 0 0.001 0 0 -8.78784 0.001 0 0 0 0.001 0 0 -8.79028 0.001 0 0 0 0.001 0 0 -8.79272 0.001 0 0 0 0.001 0 0 -8.79517 0.001 0 0 0 0.001 0 0 -8.79761 0.001 0 0 0 0.001 0 0 -8.80005 0.001 0 0 0 0.001 0 0 -8.80249 0.001 0 0 0 0.001 0 0 -8.80493 0.001 0 0 0 0.001 0 0 -8.80737 0.001 0 0 0 0.001 0 0 -8.80981 0.001 0 0 0 0.001 0 0 -8.81226 0.001 0 0 0 0.001 0 0 -8.8147 0.001 0 0 0 0.001 0 0 -8.81714 0.001 0 0 0 0.001 0 0 -8.81958 0.001 0 0 0 0.001 0 0 -8.82202 0.001 0 0 0 0.001 0 0 -8.82446 0.001 0 0 0 0.001 0 0 -8.8269 0.001 0 0 0 0.001 0 0 -8.82935 0.001 0 0 0 0.001 0 0 -8.83179 0.001 0 0 0 0.001 0 0 -8.83423 0.001 0 0 0 0.001 0 0 -8.83667 0.001 0 0 0 0.001 0 0 -8.83911 0.001 0 0 0 0.001 0 0 -8.84155 0.001 0 0 0 0.001 0 0 -8.84399 0.001 0 0 0 0.001 0 0 -8.84644 0.001 0 0 0 0.001 0 0 -8.84888 0.001 0 0 0 0.001 0 0 -8.85132 0.001 0 0 0 0.001 0 0 -8.85376 0.001 0 0 0 0.001 0 0 -8.8562 0.001 0 0 0 0.001 0 0 -8.85864 0.001 0 0 0 0.001 0 0 -8.86108 0.001 0 0 0 0.001 0 0 -8.86353 0.001 0 0 0 0.001 0 0 -8.86597 0.001 0 0 0 0.001 0 0 -8.86841 0.001 0 0 0 0.001 0 0 -8.87085 0.001 0 0 0 0.001 0 0 -8.87329 0.001 0 0 0 0.001 0 0 -8.87573 0.001 0 0 0 0.001 0 0 -8.87817 0.001 0 0 0 0.001 0 0 -8.88062 0.001 0 0 0 0.001 0 0 -8.88306 0.001 0 0 0 0.001 0 0 -8.8855 0.001 0 0 0 0.001 0 0 -8.88794 0.001 0 0 0 0.001 0 0 -8.89038 0.001 0 0 0 0.001 0 0 -8.89282 0.001 0 0 0 0.001 0 0 -8.89526 0.001 0 0 0 0.001 0 0 -8.89771 0.001 0 0 0 0.001 0 0 -8.90015 0.001 0 0 0 0.001 0 0 -8.90259 0.001 0 0 0 0.001 0 0 -8.90503 0.001 0 0 0 0.001 0 0 -8.90747 0.001 0 0 0 0.001 0 0 -8.90991 0.001 0 0 0 0.001 0 0 -8.91235 0.001 0 0 0 0.001 0 0 -8.91479 0.001 0 0 0 0.001 0 0 -8.91724 0.001 0 0 0 0.001 0 0 -8.91968 0.001 0 0 0 0.001 0 0 -8.92212 0.001 0 0 0 0.001 0 0 -8.92456 0.001 0 0 0 0.001 0 0 -8.927 0.001 0 0 0 0.001 0 0 -8.92944 0.001 0 0 0 0.001 0 0 -8.93188 0.001 0 0 0 0.001 0 0 -8.93433 0.001 0 0 0 0.001 0 0 -8.93677 0.001 0 0 0 0.001 0 0 -8.93921 0.001 0 0 0 0.001 0 0 -8.94165 0.001 0 0 0 0.001 0 0 -8.94409 0.001 0 0 0 0.001 0 0 -8.94653 0.001 0 0 0 0.001 0 0 -8.94897 0.001 0 0 0 0.001 0 0 -8.95142 0.001 0 0 0 0.001 0 0 -8.95386 0.001 0 0 0 0.001 0 0 -8.9563 0.001 0 0 0 0.001 0 0 -8.95874 0.001 0 0 0 0.001 0 0 -8.96118 0.001 0 0 0 0.001 0 0 -8.96362 0.001 0 0 0 0.001 0 0 -8.96606 0.001 0 0 0 0.001 0 0 -8.96851 0.001 0 0 0 0.001 0 0 -8.97095 0.001 0 0 0 0.001 0 0 -8.97339 0.001 0 0 0 0.001 0 0 -8.97583 0.001 0 0 0 0.001 0 0 -8.97827 0.001 0 0 0 0.001 0 0 -8.98071 0.001 0 0 0 0.001 0 0 -8.98315 0.001 0 0 0 0.001 0 0 -8.9856 0.001 0 0 0 0.001 0 0 -8.98804 0.001 0 0 0 0.001 0 0 -8.99048 0.001 0 0 0 0.001 0 0 -8.99292 0.001 0 0 0 0.001 0 0 -8.99536 0.001 0 0 0 0.001 0 0 -8.9978 0.001 0 0 0 0.001 0 0 -9.00024 0.001 0 0 0 0.001 0 0 -9.00269 0.001 0 0 0 0.001 0 0 -9.00513 0.001 0 0 0 0.001 0 0 -9.00757 0.001 0 0 0 0.001 0 0 -9.01001 0.001 0 0 0 0.001 0 0 -9.01245 0.001 0 0 0 0.001 0 0 -9.01489 0.001 0 0 0 0.001 0 0 -9.01733 0.001 0 0 0 0.001 0 0 -9.01978 0.001 0 0 0 0.001 0 0 -9.02222 0.001 0 0 0 0.001 0 0 -9.02466 0.001 0 0 0 0.001 0 0 -9.0271 0.001 0 0 0 0.001 0 0 -9.02954 0.001 0 0 0 0.001 0 0 -9.03198 0.001 0 0 0 0.001 0 0 -9.03442 0.001 0 0 0 0.001 0 0 -9.03687 0.001 0 0 0 0.001 0 0 -9.03931 0.001 0 0 0 0.001 0 0 -9.04175 0.001 0 0 0 0.001 0 0 -9.04419 0.001 0 0 0 0.001 0 0 -9.04663 0.001 0 0 0 0.001 0 0 -9.04907 0.001 0 0 0 0.001 0 0 -9.05151 0.001 0 0 0 0.001 0 0 -9.05396 0.001 0 0 0 0.001 0 0 -9.0564 0.001 0 0 0 0.001 0 0 -9.05884 0.001 0 0 0 0.001 0 0 -9.06128 0.001 0 0 0 0.001 0 0 -9.06372 0.001 0 0 0 0.001 0 0 -9.06616 0.001 0 0 0 0.001 0 0 -9.0686 0.001 0 0 0 0.001 0 0 -9.07104 0.001 0 0 0 0.001 0 0 -9.07349 0.001 0 0 0 0.001 0 0 -9.07593 0.001 0 0 0 0.001 0 0 -9.07837 0.001 0 0 0 0.001 0 0 -9.08081 0.001 0 0 0 0.001 0 0 -9.08325 0.001 0 0 0 0.001 0 0 -9.08569 0.001 0 0 0 0.001 0 0 -9.08813 0.001 0 0 0 0.001 0 0 -9.09058 0.001 0 0 0 0.001 0 0 -9.09302 0.001 0 0 0 0.001 0 0 -9.09546 0.001 0 0 0 0.001 0 0 -9.0979 0.001 0 0 0 0.001 0 0 -9.10034 0.001 0 0 0 0.001 0 0 -9.10278 0.001 0 0 0 0.001 0 0 -9.10522 0.001 0 0 0 0.001 0 0 -9.10767 0.001 0 0 0 0.001 0 0 -9.11011 0.001 0 0 0 0.001 0 0 -9.11255 0.001 0 0 0 0.001 0 0 -9.11499 0.001 0 0 0 0.001 0 0 -9.11743 0.001 0 0 0 0.001 0 0 -9.11987 0.001 0 0 0 0.001 0 0 -9.12231 0.001 0 0 0 0.001 0 0 -9.12476 0.001 0 0 0 0.001 0 0 -9.1272 0.001 0 0 0 0.001 0 0 -9.12964 0.001 0 0 0 0.001 0 0 -9.13208 0.001 0 0 0 0.001 0 0 -9.13452 0.001 0 0 0 0.001 0 0 -9.13696 0.001 0 0 0 0.001 0 0 -9.1394 0.001 0 0 0 0.001 0 0 -9.14185 0.001 0 0 0 0.001 0 0 -9.14429 0.001 0 0 0 0.001 0 0 -9.14673 0.001 0 0 0 0.001 0 0 -9.14917 0.001 0 0 0 0.001 0 0 -9.15161 0.001 0 0 0 0.001 0 0 -9.15405 0.001 0 0 0 0.001 0 0 -9.15649 0.001 0 0 0 0.001 0 0 -9.15894 0.001 0 0 0 0.001 0 0 -9.16138 0.001 0 0 0 0.001 0 0 -9.16382 0.001 0 0 0 0.001 0 0 -9.16626 0.001 0 0 0 0.001 0 0 -9.1687 0.001 0 0 0 0.001 0 0 -9.17114 0.001 0 0 0 0.001 0 0 -9.17358 0.001 0 0 0 0.001 0 0 -9.17603 0.001 0 0 0 0.001 0 0 -9.17847 0.001 0 0 0 0.001 0 0 -9.18091 0.001 0 0 0 0.001 0 0 -9.18335 0.001 0 0 0 0.001 0 0 -9.18579 0.001 0 0 0 0.001 0 0 -9.18823 0.001 0 0 0 0.001 0 0 -9.19067 0.001 0 0 0 0.001 0 0 -9.19312 0.001 0 0 0 0.001 0 0 -9.19556 0.001 0 0 0 0.001 0 0 -9.198 0.001 0 0 0 0.001 0 0 -9.20044 0.001 0 0 0 0.001 0 0 -9.20288 0.001 0 0 0 0.001 0 0 -9.20532 0.001 0 0 0 0.001 0 0 -9.20776 0.001 0 0 0 0.001 0 0 -9.21021 0.001 0 0 0 0.001 0 0 -9.21265 0.001 0 0 0 0.001 0 0 -9.21509 0.001 0 0 0 0.001 0 0 -9.21753 0.001 0 0 0 0.001 0 0 -9.21997 0.001 0 0 0 0.001 0 0 -9.22241 0.001 0 0 0 0.001 0 0 -9.22485 0.001 0 0 0 0.001 0 0 -9.22729 0.001 0 0 0 0.001 0 0 -9.22974 0.001 0 0 0 0.001 0 0 -9.23218 0.001 0 0 0 0.001 0 0 -9.23462 0.001 0 0 0 0.001 0 0 -9.23706 0.001 0 0 0 0.001 0 0 -9.2395 0.001 0 0 0 0.001 0 0 -9.24194 0.001 0 0 0 0.001 0 0 -9.24438 0.001 0 0 0 0.001 0 0 -9.24683 0.001 0 0 0 0.001 0 0 -9.24927 0.001 0 0 0 0.001 0 0 -9.25171 0.001 0 0 0 0.001 0 0 -9.25415 0.001 0 0 0 0.001 0 0 -9.25659 0.001 0 0 0 0.001 0 0 -9.25903 0.001 0 0 0 0.001 0 0 -9.26147 0.001 0 0 0 0.001 0 0 -9.26392 0.001 0 0 0 0.001 0 0 -9.26636 0.001 0 0 0 0.001 0 0 -9.2688 0.001 0 0 0 0.001 0 0 -9.27124 0.001 0 0 0 0.001 0 0 -9.27368 0.001 0 0 0 0.001 0 0 -9.27612 0.001 0 0 0 0.001 0 0 -9.27856 0.001 0 0 0 0.001 0 0 -9.28101 0.001 0 0 0 0.001 0 0 -9.28345 0.001 0 0 0 0.001 0 0 -9.28589 0.001 0 0 0 0.001 0 0 -9.28833 0.001 0 0 0 0.001 0 0 -9.29077 0.001 0 0 0 0.001 0 0 -9.29321 0.001 0 0 0 0.001 0 0 -9.29565 0.001 0 0 0 0.001 0 0 -9.2981 0.001 0 0 0 0.001 0 0 -9.30054 0.001 0 0 0 0.001 0 0 -9.30298 0.001 0 0 0 0.001 0 0 -9.30542 0.001 0 0 0 0.001 0 0 -9.30786 0.001 0 0 0 0.001 0 0 -9.3103 0.001 0 0 0 0.001 0 0 -9.31274 0.001 0 0 0 0.001 0 0 -9.31519 0.001 0 0 0 0.001 0 0 -9.31763 0.001 0 0 0 0.001 0 0 -9.32007 0.001 0 0 0 0.001 0 0 -9.32251 0.001 0 0 0 0.001 0 0 -9.32495 0.001 0 0 0 0.001 0 0 -9.32739 0.001 0 0 0 0.001 0 0 -9.32983 0.001 0 0 0 0.001 0 0 -9.33228 0.001 0 0 0 0.001 0 0 -9.33472 0.001 0 0 0 0.001 0 0 -9.33716 0.001 0 0 0 0.001 0 0 -9.3396 0.001 0 0 0 0.001 0 0 -9.34204 0.001 0 0 0 0.001 0 0 -9.34448 0.001 0 0 0 0.001 0 0 -9.34692 0.001 0 0 0 0.001 0 0 -9.34937 0.001 0 0 0 0.001 0 0 -9.35181 0.001 0 0 0 0.001 0 0 -9.35425 0.001 0 0 0 0.001 0 0 -9.35669 0.001 0 0 0 0.001 0 0 -9.35913 0.001 0 0 0 0.001 0 0 -9.36157 0.001 0 0 0 0.001 0 0 -9.36401 0.001 0 0 0 0.001 0 0 -9.36646 0.001 0 0 0 0.001 0 0 -9.3689 0.001 0 0 0 0.001 0 0 -9.37134 0.001 0 0 0 0.001 0 0 -9.37378 0.001 0 0 0 0.001 0 0 -9.37622 0.001 0 0 0 0.001 0 0 -9.37866 0.001 0 0 0 0.001 0 0 -9.3811 0.001 0 0 0 0.001 0 0 -9.38354 0.001 0 0 0 0.001 0 0 -9.38599 0.001 0 0 0 0.001 0 0 -9.38843 0.001 0 0 0 0.001 0 0 -9.39087 0.001 0 0 0 0.001 0 0 -9.39331 0.001 0 0 0 0.001 0 0 -9.39575 0.001 0 0 0 0.001 0 0 -9.39819 0.001 0 0 0 0.001 0 0 -9.40063 0.001 0 0 0 0.001 0 0 -9.40308 0.001 0 0 0 0.001 0 0 -9.40552 0.001 0 0 0 0.001 0 0 -9.40796 0.001 0 0 0 0.001 0 0 -9.4104 0.001 0 0 0 0.001 0 0 -9.41284 0.001 0 0 0 0.001 0 0 -9.41528 0.001 0 0 0 0.001 0 0 -9.41772 0.001 0 0 0 0.001 0 0 -9.42017 0.001 0 0 0 0.001 0 0 -9.42261 0.001 0 0 0 0.001 0 0 -9.42505 0.001 0 0 0 0.001 0 0 -9.42749 0.001 0 0 0 0.001 0 0 -9.42993 0.001 0 0 0 0.001 0 0 -9.43237 0.001 0 0 0 0.001 0 0 -9.43481 0.001 0 0 0 0.001 0 0 -9.43726 0.001 0 0 0 0.001 0 0 -9.4397 0.001 0 0 0 0.001 0 0 -9.44214 0.001 0 0 0 0.001 0 0 -9.44458 0.001 0 0 0 0.001 0 0 -9.44702 0.001 0 0 0 0.001 0 0 -9.44946 0.001 0 0 0 0.001 0 0 -9.4519 0.001 0 0 0 0.001 0 0 -9.45435 0.001 0 0 0 0.001 0 0 -9.45679 0.001 0 0 0 0.001 0 0 -9.45923 0.001 0 0 0 0.001 0 0 -9.46167 0.001 0 0 0 0.001 0 0 -9.46411 0.001 0 0 0 0.001 0 0 -9.46655 0.001 0 0 0 0.001 0 0 -9.46899 0.001 0 0 0 0.001 0 0 -9.47144 0.001 0 0 0 0.001 0 0 -9.47388 0.001 0 0 0 0.001 0 0 -9.47632 0.001 0 0 0 0.001 0 0 -9.47876 0.001 0 0 0 0.001 0 0 -9.4812 0.001 0 0 0 0.001 0 0 -9.48364 0.001 0 0 0 0.001 0 0 -9.48608 0.001 0 0 0 0.001 0 0 -9.48853 0.001 0 0 0 0.001 0 0 -9.49097 0.001 0 0 0 0.001 0 0 -9.49341 0.001 0 0 0 0.001 0 0 -9.49585 0.001 0 0 0 0.001 0 0 -9.49829 0.001 0 0 0 0.001 0 0 -9.50073 0.001 0 0 0 0.001 0 0 -9.50317 0.001 0 0 0 0.001 0 0 -9.50562 0.001 0 0 0 0.001 0 0 -9.50806 0.001 0 0 0 0.001 0 0 -9.5105 0.001 0 0 0 0.001 0 0 -9.51294 0.001 0 0 0 0.001 0 0 -9.51538 0.001 0 0 0 0.001 0 0 -9.51782 0.001 0 0 0 0.001 0 0 -9.52026 0.001 0 0 0 0.001 0 0 -9.52271 0.001 0 0 0 0.001 0 0 -9.52515 0.001 0 0 0 0.001 0 0 -9.52759 0.001 0 0 0 0.001 0 0 -9.53003 0.001 0 0 0 0.001 0 0 -9.53247 0.001 0 0 0 0.001 0 0 -9.53491 0.001 0 0 0 0.001 0 0 -9.53735 0.001 0 0 0 0.001 0 0 -9.53979 0.001 0 0 0 0.001 0 0 -9.54224 0.001 0 0 0 0.001 0 0 -9.54468 0.001 0 0 0 0.001 0 0 -9.54712 0.001 0 0 0 0.001 0 0 -9.54956 0.001 0 0 0 0.001 0 0 -9.552 0.001 0 0 0 0.001 0 0 -9.55444 0.001 0 0 0 0.001 0 0 -9.55688 0.001 0 0 0 0.001 0 0 -9.55933 0.001 0 0 0 0.001 0 0 -9.56177 0.001 0 0 0 0.001 0 0 -9.56421 0.001 0 0 0 0.001 0 0 -9.56665 0.001 0 0 0 0.001 0 0 -9.56909 0.001 0 0 0 0.001 0 0 -9.57153 0.001 0 0 0 0.001 0 0 -9.57397 0.001 0 0 0 0.001 0 0 -9.57642 0.001 0 0 0 0.001 0 0 -9.57886 0.001 0 0 0 0.001 0 0 -9.5813 0.001 0 0 0 0.001 0 0 -9.58374 0.001 0 0 0 0.001 0 0 -9.58618 0.001 0 0 0 0.001 0 0 -9.58862 0.001 0 0 0 0.001 0 0 -9.59106 0.001 0 0 0 0.001 0 0 -9.59351 0.001 0 0 0 0.001 0 0 -9.59595 0.001 0 0 0 0.001 0 0 -9.59839 0.001 0 0 0 0.001 0 0 -9.60083 0.001 0 0 0 0.001 0 0 -9.60327 0.001 0 0 0 0.001 0 0 -9.60571 0.001 0 0 0 0.001 0 0 -9.60815 0.001 0 0 0 0.001 0 0 -9.6106 0.001 0 0 0 0.001 0 0 -9.61304 0.001 0 0 0 0.001 0 0 -9.61548 0.001 0 0 0 0.001 0 0 -9.61792 0.001 0 0 0 0.001 0 0 -9.62036 0.001 0 0 0 0.001 0 0 -9.6228 0.001 0 0 0 0.001 0 0 -9.62524 0.001 0 0 0 0.001 0 0 -9.62769 0.001 0 0 0 0.001 0 0 -9.63013 0.001 0 0 0 0.001 0 0 -9.63257 0.001 0 0 0 0.001 0 0 -9.63501 0.001 0 0 0 0.001 0 0 -9.63745 0.001 0 0 0 0.001 0 0 -9.63989 0.001 0 0 0 0.001 0 0 -9.64233 0.001 0 0 0 0.001 0 0 -9.64478 0.001 0 0 0 0.001 0 0 -9.64722 0.001 0 0 0 0.001 0 0 -9.64966 0.001 0 0 0 0.001 0 0 -9.6521 0.001 0 0 0 0.001 0 0 -9.65454 0.001 0 0 0 0.001 0 0 -9.65698 0.001 0 0 0 0.001 0 0 -9.65942 0.001 0 0 0 0.001 0 0 -9.66187 0.001 0 0 0 0.001 0 0 -9.66431 0.001 0 0 0 0.001 0 0 -9.66675 0.001 0 0 0 0.001 0 0 -9.66919 0.001 0 0 0 0.001 0 0 -9.67163 0.001 0 0 0 0.001 0 0 -9.67407 0.001 0 0 0 0.001 0 0 -9.67651 0.001 0 0 0 0.001 0 0 -9.67896 0.001 0 0 0 0.001 0 0 -9.6814 0.001 0 0 0 0.001 0 0 -9.68384 0.001 0 0 0 0.001 0 0 -9.68628 0.001 0 0 0 0.001 0 0 -9.68872 0.001 0 0 0 0.001 0 0 -9.69116 0.001 0 0 0 0.001 0 0 -9.6936 0.001 0 0 0 0.001 0 0 -9.69604 0.001 0 0 0 0.001 0 0 -9.69849 0.001 0 0 0 0.001 0 0 -9.70093 0.001 0 0 0 0.001 0 0 -9.70337 0.001 0 0 0 0.001 0 0 -9.70581 0.001 0 0 0 0.001 0 0 -9.70825 0.001 0 0 0 0.001 0 0 -9.71069 0.001 0 0 0 0.001 0 0 -9.71313 0.001 0 0 0 0.001 0 0 -9.71558 0.001 0 0 0 0.001 0 0 -9.71802 0.001 0 0 0 0.001 0 0 -9.72046 0.001 0 0 0 0.001 0 0 -9.7229 0.001 0 0 0 0.001 0 0 -9.72534 0.001 0 0 0 0.001 0 0 -9.72778 0.001 0 0 0 0.001 0 0 -9.73022 0.001 0 0 0 0.001 0 0 -9.73267 0.001 0 0 0 0.001 0 0 -9.73511 0.001 0 0 0 0.001 0 0 -9.73755 0.001 0 0 0 0.001 0 0 -9.73999 0.001 0 0 0 0.001 0 0 -9.74243 0.001 0 0 0 0.001 0 0 -9.74487 0.001 0 0 0 0.001 0 0 -9.74731 0.001 0 0 0 0.001 0 0 -9.74976 0.001 0 0 0 0.001 0 0 -9.7522 0.001 0 0 0 0.001 0 0 -9.75464 0.001 0 0 0 0.001 0 0 -9.75708 0.001 0 0 0 0.001 0 0 -9.75952 0.001 0 0 0 0.001 0 0 -9.76196 0.001 0 0 0 0.001 0 0 -9.7644 0.001 0 0 0 0.001 0 0 -9.76685 0.001 0 0 0 0.001 0 0 -9.76929 0.001 0 0 0 0.001 0 0 -9.77173 0.001 0 0 0 0.001 0 0 -9.77417 0.001 0 0 0 0.001 0 0 -9.77661 0.001 0 0 0 0.001 0 0 -9.77905 0.001 0 0 0 0.001 0 0 -9.78149 0.001 0 0 0 0.001 0 0 -9.78394 0.001 0 0 0 0.001 0 0 -9.78638 0.001 0 0 0 0.001 0 0 -9.78882 0.001 0 0 0 0.001 0 0 -9.79126 0.001 0 0 0 0.001 0 0 -9.7937 0.001 0 0 0 0.001 0 0 -9.79614 0.001 0 0 0 0.001 0 0 -9.79858 0.001 0 0 0 0.001 0 0 -9.80103 0.001 0 0 0 0.001 0 0 -9.80347 0.001 0 0 0 0.001 0 0 -9.80591 0.001 0 0 0 0.001 0 0 -9.80835 0.001 0 0 0 0.001 0 0 -9.81079 0.001 0 0 0 0.001 0 0 -9.81323 0.001 0 0 0 0.001 0 0 -9.81567 0.001 0 0 0 0.001 0 0 -9.81812 0.001 0 0 0 0.001 0 0 -9.82056 0.001 0 0 0 0.001 0 0 -9.823 0.001 0 0 0 0.001 0 0 -9.82544 0.001 0 0 0 0.001 0 0 -9.82788 0.001 0 0 0 0.001 0 0 -9.83032 0.001 0 0 0 0.001 0 0 -9.83276 0.001 0 0 0 0.001 0 0 -9.83521 0.001 0 0 0 0.001 0 0 -9.83765 0.001 0 0 0 0.001 0 0 -9.84009 0.001 0 0 0 0.001 0 0 -9.84253 0.001 0 0 0 0.001 0 0 -9.84497 0.001 0 0 0 0.001 0 0 -9.84741 0.001 0 0 0 0.001 0 0 -9.84985 0.001 0 0 0 0.001 0 0 -9.85229 0.001 0 0 0 0.001 0 0 -9.85474 0.001 0 0 0 0.001 0 0 -9.85718 0.001 0 0 0 0.001 0 0 -9.85962 0.001 0 0 0 0.001 0 0 -9.86206 0.001 0 0 0 0.001 0 0 -9.8645 0.001 0 0 0 0.001 0 0 -9.86694 0.001 0 0 0 0.001 0 0 -9.86938 0.001 0 0 0 0.001 0 0 -9.87183 0.001 0 0 0 0.001 0 0 -9.87427 0.001 0 0 0 0.001 0 0 -9.87671 0.001 0 0 0 0.001 0 0 -9.87915 0.001 0 0 0 0.001 0 0 -9.88159 0.001 0 0 0 0.001 0 0 -9.88403 0.001 0 0 0 0.001 0 0 -9.88647 0.001 0 0 0 0.001 0 0 -9.88892 0.001 0 0 0 0.001 0 0 -9.89136 0.001 0 0 0 0.001 0 0 -9.8938 0.001 0 0 0 0.001 0 0 -9.89624 0.001 0 0 0 0.001 0 0 -9.89868 0.001 0 0 0 0.001 0 0 -9.90112 0.001 0 0 0 0.001 0 0 -9.90356 0.001 0 0 0 0.001 0 0 -9.90601 0.001 0 0 0 0.001 0 0 -9.90845 0.001 0 0 0 0.001 0 0 -9.91089 0.001 0 0 0 0.001 0 0 -9.91333 0.001 0 0 0 0.001 0 0 -9.91577 0.001 0 0 0 0.001 0 0 -9.91821 0.001 0 0 0 0.001 0 0 -9.92065 0.001 0 0 0 0.001 0 0 -9.9231 0.001 0 0 0 0.001 0 0 -9.92554 0.001 0 0 0 0.001 0 0 -9.92798 0.001 0 0 0 0.001 0 0 -9.93042 0.001 0 0 0 0.001 0 0 -9.93286 0.001 0 0 0 0.001 0 0 -9.9353 0.001 0 0 0 0.001 0 0 -9.93774 0.001 0 0 0 0.001 0 0 -9.94019 0.001 0 0 0 0.001 0 0 -9.94263 0.001 0 0 0 0.001 0 0 -9.94507 0.001 0 0 0 0.001 0 0 -9.94751 0.001 0 0 0 0.001 0 0 -9.94995 0.001 0 0 0 0.001 0 0 -9.95239 0.001 0 0 0 0.001 0 0 -9.95483 0.001 0 0 0 0.001 0 0 -9.95728 0.001 0 0 0 0.001 0 0 -9.95972 0.001 0 0 0 0.001 0 0 -9.96216 0.001 0 0 0 0.001 0 0 -9.9646 0.001 0 0 0 0.001 0 0 -9.96704 0.001 0 0 0 0.001 0 0 -9.96948 0.001 0 0 0 0.001 0 0 -9.97192 0.001 0 0 0 0.001 0 0 -9.97437 0.001 0 0 0 0.001 0 0 -9.97681 0.001 0 0 0 0.001 0 0 -9.97925 0.001 0 0 0 0.001 0 0 -9.98169 0.001 0 0 0 0.001 0 0 -9.98413 0.001 0 0 0 0.001 0 0 -9.98657 0.001 0 0 0 0.001 0 0 -9.98901 0.001 0 0 0 0.001 0 0 -9.99146 0.001 0 0 0 0.001 0 0 -9.9939 0.001 0 0 0 0.001 0 0 -9.99634 0.001 0 0 0 0.001 0 0 -9.99878 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=512.csv b/reference/swashes_1_nx=512.csv deleted file mode 100644 index 0a2916f..0000000 --- a/reference/swashes_1_nx=512.csv +++ /dev/null @@ -1,530 +0,0 @@ -############################################################################## -# Generated by SWASHES version 1.03.00, 2016-01-29 -############################################################################## -# Dimension: 1 -# Type: 3 (=Dam break) -# Domain: 1 -# Choice: 1 (=on a wet domain without friction (Stoker's solution)) -############################################################################## -# PARAMETERS OF THE SOLUTION -# -# Length of the domain: 10 meters -# Space step: 0.0195312 meters -# Number of cells: 512 -# Position of the dam: x=5 meters -# Time value: 6 seconds -############################################################################## -# -#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] -0.00976562 0.005 0 0 0 0.005 0 0 -0.0292969 0.005 0 0 0 0.005 0 0 -0.0488281 0.005 0 0 0 0.005 0 0 -0.0683594 0.005 0 0 0 0.005 0 0 -0.0878906 0.005 0 0 0 0.005 0 0 -0.107422 0.005 0 0 0 0.005 0 0 -0.126953 0.005 0 0 0 0.005 0 0 -0.146484 0.005 0 0 0 0.005 0 0 -0.166016 0.005 0 0 0 0.005 0 0 -0.185547 0.005 0 0 0 0.005 0 0 -0.205078 0.005 0 0 0 0.005 0 0 -0.224609 0.005 0 0 0 0.005 0 0 -0.244141 0.005 0 0 0 0.005 0 0 -0.263672 0.005 0 0 0 0.005 0 0 -0.283203 0.005 0 0 0 0.005 0 0 -0.302734 0.005 0 0 0 0.005 0 0 -0.322266 0.005 0 0 0 0.005 0 0 -0.341797 0.005 0 0 0 0.005 0 0 -0.361328 0.005 0 0 0 0.005 0 0 -0.380859 0.005 0 0 0 0.005 0 0 -0.400391 0.005 0 0 0 0.005 0 0 -0.419922 0.005 0 0 0 0.005 0 0 -0.439453 0.005 0 0 0 0.005 0 0 -0.458984 0.005 0 0 0 0.005 0 0 -0.478516 0.005 0 0 0 0.005 0 0 -0.498047 0.005 0 0 0 0.005 0 0 -0.517578 0.005 0 0 0 0.005 0 0 -0.537109 0.005 0 0 0 0.005 0 0 -0.556641 0.005 0 0 0 0.005 0 0 -0.576172 0.005 0 0 0 0.005 0 0 -0.595703 0.005 0 0 0 0.005 0 0 -0.615234 0.005 0 0 0 0.005 0 0 -0.634766 0.005 0 0 0 0.005 0 0 -0.654297 0.005 0 0 0 0.005 0 0 -0.673828 0.005 0 0 0 0.005 0 0 -0.693359 0.005 0 0 0 0.005 0 0 -0.712891 0.005 0 0 0 0.005 0 0 -0.732422 0.005 0 0 0 0.005 0 0 -0.751953 0.005 0 0 0 0.005 0 0 -0.771484 0.005 0 0 0 0.005 0 0 -0.791016 0.005 0 0 0 0.005 0 0 -0.810547 0.005 0 0 0 0.005 0 0 -0.830078 0.005 0 0 0 0.005 0 0 -0.849609 0.005 0 0 0 0.005 0 0 -0.869141 0.005 0 0 0 0.005 0 0 -0.888672 0.005 0 0 0 0.005 0 0 -0.908203 0.005 0 0 0 0.005 0 0 -0.927734 0.005 0 0 0 0.005 0 0 -0.947266 0.005 0 0 0 0.005 0 0 -0.966797 0.005 0 0 0 0.005 0 0 -0.986328 0.005 0 0 0 0.005 0 0 -1.00586 0.005 0 0 0 0.005 0 0 -1.02539 0.005 0 0 0 0.005 0 0 -1.04492 0.005 0 0 0 0.005 0 0 -1.06445 0.005 0 0 0 0.005 0 0 -1.08398 0.005 0 0 0 0.005 0 0 -1.10352 0.005 0 0 0 0.005 0 0 -1.12305 0.005 0 0 0 0.005 0 0 -1.14258 0.005 0 0 0 0.005 0 0 -1.16211 0.005 0 0 0 0.005 0 0 -1.18164 0.005 0 0 0 0.005 0 0 -1.20117 0.005 0 0 0 0.005 0 0 -1.2207 0.005 0 0 0 0.005 0 0 -1.24023 0.005 0 0 0 0.005 0 0 -1.25977 0.005 0 0 0 0.005 0 0 -1.2793 0.005 0 0 0 0.005 0 0 -1.29883 0.005 0 0 0 0.005 0 0 -1.31836 0.005 0 0 0 0.005 0 0 -1.33789 0.005 0 0 0 0.005 0 0 -1.35742 0.005 0 0 0 0.005 0 0 -1.37695 0.005 0 0 0 0.005 0 0 -1.39648 0.005 0 0 0 0.005 0 0 -1.41602 0.005 0 0 0 0.005 0 0 -1.43555 0.005 0 0 0 0.005 0 0 -1.45508 0.005 0 0 0 0.005 0 0 -1.47461 0.005 0 0 0 0.005 0 0 -1.49414 0.005 0 0 0 0.005 0 0 -1.51367 0.005 0 0 0 0.005 0 0 -1.5332 0.005 0 0 0 0.005 0 0 -1.55273 0.005 0 0 0 0.005 0 0 -1.57227 0.005 0 0 0 0.005 0 0 -1.5918 0.005 0 0 0 0.005 0 0 -1.61133 0.005 0 0 0 0.005 0 0 -1.63086 0.005 0 0 0 0.005 0 0 -1.65039 0.005 0 0 0 0.005 0 0 -1.66992 0.005 0 0 0 0.005 0 0 -1.68945 0.005 0 0 0 0.005 0 0 -1.70898 0.005 0 0 0 0.005 0 0 -1.72852 0.005 0 0 0 0.005 0 0 -1.74805 0.005 0 0 0 0.005 0 0 -1.76758 0.005 0 0 0 0.005 0 0 -1.78711 0.005 0 0 0 0.005 0 0 -1.80664 0.005 0 0 0 0.005 0 0 -1.82617 0.005 0 0 0 0.005 0 0 -1.8457 0.005 0 0 0 0.005 0 0 -1.86523 0.005 0 0 0 0.005 0 0 -1.88477 0.005 0 0 0 0.005 0 0 -1.9043 0.005 0 0 0 0.005 0 0 -1.92383 0.005 0 0 0 0.005 0 0 -1.94336 0.005 0 0 0 0.005 0 0 -1.96289 0.005 0 0 0 0.005 0 0 -1.98242 0.005 0 0 0 0.005 0 0 -2.00195 0.005 0 0 0 0.005 0 0 -2.02148 0.005 0 0 0 0.005 0 0 -2.04102 0.005 0 0 0 0.005 0 0 -2.06055 0.005 0 0 0 0.005 0 0 -2.08008 0.005 0 0 0 0.005 0 0 -2.09961 0.005 0 0 0 0.005 0 0 -2.11914 0.005 0 0 0 0.005 0 0 -2.13867 0.005 0 0 0 0.005 0 0 -2.1582 0.005 0 0 0 0.005 0 0 -2.17773 0.005 0 0 0 0.005 0 0 -2.19727 0.005 0 0 0 0.005 0 0 -2.2168 0.005 0 0 0 0.005 0 0 -2.23633 0.005 0 0 0 0.005 0 0 -2.25586 0.005 0 0 0 0.005 0 0 -2.27539 0.005 0 0 0 0.005 0 0 -2.29492 0.005 0 0 0 0.005 0 0 -2.31445 0.005 0 0 0 0.005 0 0 -2.33398 0.005 0 0 0 0.005 0 0 -2.35352 0.005 0 0 0 0.005 0 0 -2.37305 0.005 0 0 0 0.005 0 0 -2.39258 0.005 0 0 0 0.005 0 0 -2.41211 0.005 0 0 0 0.005 0 0 -2.43164 0.005 0 0 0 0.005 0 0 -2.45117 0.005 0 0 0 0.005 0 0 -2.4707 0.005 0 0 0 0.005 0 0 -2.49023 0.005 0 0 0 0.005 0 0 -2.50977 0.005 0 0 0 0.005 0 0 -2.5293 0.005 0 0 0 0.005 0 0 -2.54883 0.005 0 0 0 0.005 0 0 -2.56836 0.005 0 0 0 0.005 0 0 -2.58789 0.005 0 0 0 0.005 0 0 -2.60742 0.005 0 0 0 0.005 0 0 -2.62695 0.005 0 0 0 0.005 0 0 -2.64648 0.005 0 0 0 0.005 0 0 -2.66602 0.005 0 0 0 0.005 0 0 -2.68555 0.005 0 0 0 0.005 0 0 -2.70508 0.005 0 0 0 0.005 0 0 -2.72461 0.005 0 0 0 0.005 0 0 -2.74414 0.005 0 0 0 0.005 0 0 -2.76367 0.005 0 0 0 0.005 0 0 -2.7832 0.005 0 0 0 0.005 0 0 -2.80273 0.005 0 0 0 0.005 0 0 -2.82227 0.005 0 0 0 0.005 0 0 -2.8418 0.005 0 0 0 0.005 0 0 -2.86133 0.005 0 0 0 0.005 0 0 -2.88086 0.005 0 0 0 0.005 0 0 -2.90039 0.005 0 0 0 0.005 0 0 -2.91992 0.005 0 0 0 0.005 0 0 -2.93945 0.005 0 0 0 0.005 0 0 -2.95898 0.005 0 0 0 0.005 0 0 -2.97852 0.005 0 0 0 0.005 0 0 -2.99805 0.005 0 0 0 0.005 0 0 -3.01758 0.005 0 0 0 0.005 0 0 -3.03711 0.005 0 0 0 0.005 0 0 -3.05664 0.005 0 0 0 0.005 0 0 -3.07617 0.005 0 0 0 0.005 0 0 -3.0957 0.005 0 0 0 0.005 0 0 -3.11523 0.005 0 0 0 0.005 0 0 -3.13477 0.005 0 0 0 0.005 0 0 -3.1543 0.005 0 0 0 0.005 0 0 -3.17383 0.005 0 0 0 0.005 0 0 -3.19336 0.005 0 0 0 0.005 0 0 -3.21289 0.005 0 0 0 0.005 0 0 -3.23242 0.005 0 0 0 0.005 0 0 -3.25195 0.005 0 0 0 0.005 0 0 -3.27148 0.005 0 0 0 0.005 0 0 -3.29102 0.005 0 0 0 0.005 0 0 -3.31055 0.005 0 0 0 0.005 0 0 -3.33008 0.005 0 0 0 0.005 0 0 -3.34961 0.005 0 0 0 0.005 0 0 -3.36914 0.005 0 0 0 0.005 0 0 -3.38867 0.005 0 0 0 0.005 0 0 -3.4082 0.005 0 0 0 0.005 0 0 -3.42773 0.005 0 0 0 0.005 0 0 -3.44727 0.005 0 0 0 0.005 0 0 -3.4668 0.005 0 0 0 0.005 0 0 -3.48633 0.005 0 0 0 0.005 0 0 -3.50586 0.005 0 0 0 0.005 0 0 -3.52539 0.005 0 0 0 0.005 0 0 -3.54492 0.005 0 0 0 0.005 0 0 -3.56445 0.005 0 0 0 0.005 0 0 -3.58398 0.005 0 0 0 0.005 0 0 -3.60352 0.005 0 0 0 0.005 0 0 -3.62305 0.005 0 0 0 0.005 0 0 -3.64258 0.005 0 0 0 0.005 0 0 -3.66211 0.005 0 0 0 0.005 0 0 -3.68164 0.00497376 0.00116386 0 5.78874e-006 0.00497376 0.00526893 0.000150603 -3.70117 0.00492501 0.00333399 0 1.642e-005 0.00492501 0.0151679 0.000301781 -3.7207 0.00487651 0.00550413 0 2.6841e-005 0.00487651 0.0251652 0.00041877 -3.74023 0.00482825 0.00767427 0 3.70533e-005 0.00482825 0.0352621 0.000519192 -3.75977 0.00478022 0.00984441 0 4.70585e-005 0.00478022 0.0454602 0.000608885 -3.7793 0.00473244 0.0120146 0 5.68581e-005 0.00473244 0.055761 0.000690725 -3.79883 0.00468489 0.0141847 0 6.64537e-005 0.00468489 0.0661661 0.000766402 -3.81836 0.00463759 0.0163548 0 7.58469e-005 0.00463759 0.0766771 0.00083702 -3.83789 0.00459052 0.018525 0 8.50393e-005 0.00459052 0.0872955 0.000903351 -3.85742 0.0045437 0.0206951 0 9.40323e-005 0.0045437 0.0980231 0.000965966 -3.87695 0.00449711 0.0228652 0 0.000102828 0.00449711 0.108862 0.0010253 -3.89648 0.00445077 0.0250354 0 0.000111427 0.00445077 0.119813 0.00108169 -3.91602 0.00440467 0.0272055 0 0.000119831 0.00440467 0.130878 0.00113542 -3.93555 0.0043588 0.0293757 0 0.000128043 0.0043588 0.142059 0.00118672 -3.95508 0.00431318 0.0315458 0 0.000136063 0.00431318 0.153359 0.00123577 -3.97461 0.00426779 0.0337159 0 0.000143893 0.00426779 0.164778 0.00128273 -3.99414 0.00422265 0.0358861 0 0.000151534 0.00422265 0.176319 0.00132775 -4.01367 0.00417774 0.0380562 0 0.000158989 0.00417774 0.187984 0.00137095 -4.0332 0.00413308 0.0402264 0 0.000166259 0.00413308 0.199774 0.00141243 -4.05273 0.00408866 0.0423965 0 0.000173345 0.00408866 0.211692 0.00145228 -4.07227 0.00404447 0.0445666 0 0.000180248 0.00404447 0.22374 0.00149059 -4.0918 0.00400053 0.0467368 0 0.000186972 0.00400053 0.23592 0.00152743 -4.11133 0.00395682 0.0489069 0 0.000193516 0.00395682 0.248235 0.00156287 -4.13086 0.00391336 0.0510771 0 0.000199883 0.00391336 0.260685 0.00159696 -4.15039 0.00387014 0.0532472 0 0.000206074 0.00387014 0.273274 0.00162977 -4.16992 0.00382715 0.0554173 0 0.000212091 0.00382715 0.286005 0.00166134 -4.18945 0.00378441 0.0575875 0 0.000217935 0.00378441 0.298878 0.00169172 -4.20898 0.0037419 0.0597576 0 0.000223607 0.0037419 0.311898 0.00172095 -4.22852 0.00369964 0.0619277 0 0.00022911 0.00369964 0.325066 0.00174907 -4.24805 0.00365762 0.0640979 0 0.000234446 0.00365762 0.338384 0.00177612 -4.26758 0.00361583 0.066268 0 0.000239614 0.00361583 0.351856 0.00180213 -4.28711 0.00357429 0.0684382 0 0.000244618 0.00357429 0.365484 0.00182713 -4.30664 0.00353299 0.0706083 0 0.000249458 0.00353299 0.379272 0.00185115 -4.32617 0.00349192 0.0727784 0 0.000254137 0.00349192 0.39322 0.00187423 -4.3457 0.0034511 0.0749486 0 0.000258655 0.0034511 0.407334 0.00189638 -4.36523 0.00341052 0.0771187 0 0.000263015 0.00341052 0.421614 0.00191762 -4.38477 0.00337017 0.0792889 0 0.000267217 0.00337017 0.436065 0.001938 -4.4043 0.00333007 0.081459 0 0.000271264 0.00333007 0.45069 0.00195752 -4.42383 0.00329021 0.0836291 0 0.000275157 0.00329021 0.465491 0.0019762 -4.44336 0.00325058 0.0857993 0 0.000278898 0.00325058 0.480472 0.00199407 -4.46289 0.0032112 0.0879694 0 0.000282487 0.0032112 0.495637 0.00201114 -4.48242 0.00317206 0.0901396 0 0.000285928 0.00317206 0.510988 0.00202744 -4.50195 0.00313315 0.0923097 0 0.00028922 0.00313315 0.526529 0.00204297 -4.52148 0.00309449 0.0944798 0 0.000292367 0.00309449 0.542263 0.00205776 -4.54102 0.00305607 0.09665 0 0.000295369 0.00305607 0.558195 0.00207183 -4.56055 0.00301788 0.0988201 0 0.000298228 0.00301788 0.574327 0.00208517 -4.58008 0.00297994 0.10099 0 0.000300945 0.00297994 0.590665 0.00209782 -4.59961 0.00294224 0.10316 0 0.000303522 0.00294224 0.607211 0.00210978 -4.61914 0.00290477 0.105331 0 0.000305961 0.00290477 0.62397 0.00212107 -4.63867 0.00286755 0.107501 0 0.000308264 0.00286755 0.640945 0.0021317 -4.6582 0.00283057 0.109671 0 0.000310431 0.00283057 0.658142 0.00214167 -4.67773 0.00279383 0.111841 0 0.000312464 0.00279383 0.675564 0.00215102 -4.69727 0.00275732 0.114011 0 0.000314365 0.00275732 0.693216 0.00215973 -4.7168 0.00272106 0.116181 0 0.000316136 0.00272106 0.711103 0.00216784 -4.73633 0.00268504 0.118351 0 0.000317778 0.00268504 0.729228 0.00217533 -4.75586 0.00264925 0.120521 0 0.000319292 0.00264925 0.747598 0.00218224 -4.77539 0.00261371 0.122692 0 0.00032068 0.00261371 0.766217 0.00218856 -4.79492 0.00257841 0.124862 0 0.000321945 0.00257841 0.785089 0.00219431 -4.81445 0.00254335 0.127032 0 0.000323086 0.00254335 0.804221 0.00219949 -4.83398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.85352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.87305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.89258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.91211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.93164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.95117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.9707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -4.99023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.00977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.0293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.04883 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.06836 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.08789 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.10742 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.12695 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.14648 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.16602 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.18555 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.20508 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.22461 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.24414 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.26367 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.2832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.30273 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.32227 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.3418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.36133 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.38086 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.40039 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.41992 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.43945 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.45898 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.47852 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.49805 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.51758 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.53711 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.55664 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.57617 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.5957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.61523 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.63477 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.6543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.67383 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.69336 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.71289 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.73242 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.75195 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.77148 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.79102 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.81055 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.83008 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.84961 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.86914 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.88867 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.92773 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.94727 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.9668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -5.98633 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.00586 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.02539 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.04492 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.06445 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.08398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.10352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.12305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.14258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.16211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.18164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.20117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.24023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.25977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 -6.2793 0.001 0 0 0 0.001 0 0 -6.29883 0.001 0 0 0 0.001 0 0 -6.31836 0.001 0 0 0 0.001 0 0 -6.33789 0.001 0 0 0 0.001 0 0 -6.35742 0.001 0 0 0 0.001 0 0 -6.37695 0.001 0 0 0 0.001 0 0 -6.39648 0.001 0 0 0 0.001 0 0 -6.41602 0.001 0 0 0 0.001 0 0 -6.43555 0.001 0 0 0 0.001 0 0 -6.45508 0.001 0 0 0 0.001 0 0 -6.47461 0.001 0 0 0 0.001 0 0 -6.49414 0.001 0 0 0 0.001 0 0 -6.51367 0.001 0 0 0 0.001 0 0 -6.5332 0.001 0 0 0 0.001 0 0 -6.55273 0.001 0 0 0 0.001 0 0 -6.57227 0.001 0 0 0 0.001 0 0 -6.5918 0.001 0 0 0 0.001 0 0 -6.61133 0.001 0 0 0 0.001 0 0 -6.63086 0.001 0 0 0 0.001 0 0 -6.65039 0.001 0 0 0 0.001 0 0 -6.66992 0.001 0 0 0 0.001 0 0 -6.68945 0.001 0 0 0 0.001 0 0 -6.70898 0.001 0 0 0 0.001 0 0 -6.72852 0.001 0 0 0 0.001 0 0 -6.74805 0.001 0 0 0 0.001 0 0 -6.76758 0.001 0 0 0 0.001 0 0 -6.78711 0.001 0 0 0 0.001 0 0 -6.80664 0.001 0 0 0 0.001 0 0 -6.82617 0.001 0 0 0 0.001 0 0 -6.8457 0.001 0 0 0 0.001 0 0 -6.86523 0.001 0 0 0 0.001 0 0 -6.88477 0.001 0 0 0 0.001 0 0 -6.9043 0.001 0 0 0 0.001 0 0 -6.92383 0.001 0 0 0 0.001 0 0 -6.94336 0.001 0 0 0 0.001 0 0 -6.96289 0.001 0 0 0 0.001 0 0 -6.98242 0.001 0 0 0 0.001 0 0 -7.00195 0.001 0 0 0 0.001 0 0 -7.02148 0.001 0 0 0 0.001 0 0 -7.04102 0.001 0 0 0 0.001 0 0 -7.06055 0.001 0 0 0 0.001 0 0 -7.08008 0.001 0 0 0 0.001 0 0 -7.09961 0.001 0 0 0 0.001 0 0 -7.11914 0.001 0 0 0 0.001 0 0 -7.13867 0.001 0 0 0 0.001 0 0 -7.1582 0.001 0 0 0 0.001 0 0 -7.17773 0.001 0 0 0 0.001 0 0 -7.19727 0.001 0 0 0 0.001 0 0 -7.2168 0.001 0 0 0 0.001 0 0 -7.23633 0.001 0 0 0 0.001 0 0 -7.25586 0.001 0 0 0 0.001 0 0 -7.27539 0.001 0 0 0 0.001 0 0 -7.29492 0.001 0 0 0 0.001 0 0 -7.31445 0.001 0 0 0 0.001 0 0 -7.33398 0.001 0 0 0 0.001 0 0 -7.35352 0.001 0 0 0 0.001 0 0 -7.37305 0.001 0 0 0 0.001 0 0 -7.39258 0.001 0 0 0 0.001 0 0 -7.41211 0.001 0 0 0 0.001 0 0 -7.43164 0.001 0 0 0 0.001 0 0 -7.45117 0.001 0 0 0 0.001 0 0 -7.4707 0.001 0 0 0 0.001 0 0 -7.49023 0.001 0 0 0 0.001 0 0 -7.50977 0.001 0 0 0 0.001 0 0 -7.5293 0.001 0 0 0 0.001 0 0 -7.54883 0.001 0 0 0 0.001 0 0 -7.56836 0.001 0 0 0 0.001 0 0 -7.58789 0.001 0 0 0 0.001 0 0 -7.60742 0.001 0 0 0 0.001 0 0 -7.62695 0.001 0 0 0 0.001 0 0 -7.64648 0.001 0 0 0 0.001 0 0 -7.66602 0.001 0 0 0 0.001 0 0 -7.68555 0.001 0 0 0 0.001 0 0 -7.70508 0.001 0 0 0 0.001 0 0 -7.72461 0.001 0 0 0 0.001 0 0 -7.74414 0.001 0 0 0 0.001 0 0 -7.76367 0.001 0 0 0 0.001 0 0 -7.7832 0.001 0 0 0 0.001 0 0 -7.80273 0.001 0 0 0 0.001 0 0 -7.82227 0.001 0 0 0 0.001 0 0 -7.8418 0.001 0 0 0 0.001 0 0 -7.86133 0.001 0 0 0 0.001 0 0 -7.88086 0.001 0 0 0 0.001 0 0 -7.90039 0.001 0 0 0 0.001 0 0 -7.91992 0.001 0 0 0 0.001 0 0 -7.93945 0.001 0 0 0 0.001 0 0 -7.95898 0.001 0 0 0 0.001 0 0 -7.97852 0.001 0 0 0 0.001 0 0 -7.99805 0.001 0 0 0 0.001 0 0 -8.01758 0.001 0 0 0 0.001 0 0 -8.03711 0.001 0 0 0 0.001 0 0 -8.05664 0.001 0 0 0 0.001 0 0 -8.07617 0.001 0 0 0 0.001 0 0 -8.0957 0.001 0 0 0 0.001 0 0 -8.11523 0.001 0 0 0 0.001 0 0 -8.13477 0.001 0 0 0 0.001 0 0 -8.1543 0.001 0 0 0 0.001 0 0 -8.17383 0.001 0 0 0 0.001 0 0 -8.19336 0.001 0 0 0 0.001 0 0 -8.21289 0.001 0 0 0 0.001 0 0 -8.23242 0.001 0 0 0 0.001 0 0 -8.25195 0.001 0 0 0 0.001 0 0 -8.27148 0.001 0 0 0 0.001 0 0 -8.29102 0.001 0 0 0 0.001 0 0 -8.31055 0.001 0 0 0 0.001 0 0 -8.33008 0.001 0 0 0 0.001 0 0 -8.34961 0.001 0 0 0 0.001 0 0 -8.36914 0.001 0 0 0 0.001 0 0 -8.38867 0.001 0 0 0 0.001 0 0 -8.4082 0.001 0 0 0 0.001 0 0 -8.42773 0.001 0 0 0 0.001 0 0 -8.44727 0.001 0 0 0 0.001 0 0 -8.4668 0.001 0 0 0 0.001 0 0 -8.48633 0.001 0 0 0 0.001 0 0 -8.50586 0.001 0 0 0 0.001 0 0 -8.52539 0.001 0 0 0 0.001 0 0 -8.54492 0.001 0 0 0 0.001 0 0 -8.56445 0.001 0 0 0 0.001 0 0 -8.58398 0.001 0 0 0 0.001 0 0 -8.60352 0.001 0 0 0 0.001 0 0 -8.62305 0.001 0 0 0 0.001 0 0 -8.64258 0.001 0 0 0 0.001 0 0 -8.66211 0.001 0 0 0 0.001 0 0 -8.68164 0.001 0 0 0 0.001 0 0 -8.70117 0.001 0 0 0 0.001 0 0 -8.7207 0.001 0 0 0 0.001 0 0 -8.74023 0.001 0 0 0 0.001 0 0 -8.75977 0.001 0 0 0 0.001 0 0 -8.7793 0.001 0 0 0 0.001 0 0 -8.79883 0.001 0 0 0 0.001 0 0 -8.81836 0.001 0 0 0 0.001 0 0 -8.83789 0.001 0 0 0 0.001 0 0 -8.85742 0.001 0 0 0 0.001 0 0 -8.87695 0.001 0 0 0 0.001 0 0 -8.89648 0.001 0 0 0 0.001 0 0 -8.91602 0.001 0 0 0 0.001 0 0 -8.93555 0.001 0 0 0 0.001 0 0 -8.95508 0.001 0 0 0 0.001 0 0 -8.97461 0.001 0 0 0 0.001 0 0 -8.99414 0.001 0 0 0 0.001 0 0 -9.01367 0.001 0 0 0 0.001 0 0 -9.0332 0.001 0 0 0 0.001 0 0 -9.05273 0.001 0 0 0 0.001 0 0 -9.07227 0.001 0 0 0 0.001 0 0 -9.0918 0.001 0 0 0 0.001 0 0 -9.11133 0.001 0 0 0 0.001 0 0 -9.13086 0.001 0 0 0 0.001 0 0 -9.15039 0.001 0 0 0 0.001 0 0 -9.16992 0.001 0 0 0 0.001 0 0 -9.18945 0.001 0 0 0 0.001 0 0 -9.20898 0.001 0 0 0 0.001 0 0 -9.22852 0.001 0 0 0 0.001 0 0 -9.24805 0.001 0 0 0 0.001 0 0 -9.26758 0.001 0 0 0 0.001 0 0 -9.28711 0.001 0 0 0 0.001 0 0 -9.30664 0.001 0 0 0 0.001 0 0 -9.32617 0.001 0 0 0 0.001 0 0 -9.3457 0.001 0 0 0 0.001 0 0 -9.36523 0.001 0 0 0 0.001 0 0 -9.38477 0.001 0 0 0 0.001 0 0 -9.4043 0.001 0 0 0 0.001 0 0 -9.42383 0.001 0 0 0 0.001 0 0 -9.44336 0.001 0 0 0 0.001 0 0 -9.46289 0.001 0 0 0 0.001 0 0 -9.48242 0.001 0 0 0 0.001 0 0 -9.50195 0.001 0 0 0 0.001 0 0 -9.52148 0.001 0 0 0 0.001 0 0 -9.54102 0.001 0 0 0 0.001 0 0 -9.56055 0.001 0 0 0 0.001 0 0 -9.58008 0.001 0 0 0 0.001 0 0 -9.59961 0.001 0 0 0 0.001 0 0 -9.61914 0.001 0 0 0 0.001 0 0 -9.63867 0.001 0 0 0 0.001 0 0 -9.6582 0.001 0 0 0 0.001 0 0 -9.67773 0.001 0 0 0 0.001 0 0 -9.69727 0.001 0 0 0 0.001 0 0 -9.7168 0.001 0 0 0 0.001 0 0 -9.73633 0.001 0 0 0 0.001 0 0 -9.75586 0.001 0 0 0 0.001 0 0 -9.77539 0.001 0 0 0 0.001 0 0 -9.79492 0.001 0 0 0 0.001 0 0 -9.81445 0.001 0 0 0 0.001 0 0 -9.83398 0.001 0 0 0 0.001 0 0 -9.85352 0.001 0 0 0 0.001 0 0 -9.87305 0.001 0 0 0 0.001 0 0 -9.89258 0.001 0 0 0 0.001 0 0 -9.91211 0.001 0 0 0 0.001 0 0 -9.93164 0.001 0 0 0 0.001 0 0 -9.95117 0.001 0 0 0 0.001 0 0 -9.9707 0.001 0 0 0 0.001 0 0 -9.99023 0.001 0 0 0 0.001 0 0 From 1d7c50241f9ffe8fb63f6433f0f58ca568653e14 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 27 Feb 2024 15:49:56 +0100 Subject: [PATCH 44/55] Recover deleted files from commit e2b1281 --- GPUSimulators/Autotuner.py | 304 ++++++ GPUSimulators/Common.py | 879 ++++++++++++++++++ GPUSimulators/CudaContext.py | 328 +++++++ GPUSimulators/CudaContext_cu.py | 272 ++++++ GPUSimulators/EE2D_KP07_dimsplit.py | 575 ++++++++++++ GPUSimulators/FORCE.py | 242 +++++ GPUSimulators/HLL.py | 235 +++++ GPUSimulators/HLL2.py | 247 +++++ GPUSimulators/IPythonMagic.py | 193 ++++ GPUSimulators/KP07.py | 252 +++++ GPUSimulators/KP07_dimsplit.py | 251 +++++ GPUSimulators/LxF.py | 238 +++++ GPUSimulators/MPISimulator.py | 535 +++++++++++ GPUSimulators/SHMEMSimulator.py | 266 ++++++ GPUSimulators/SHMEMSimulatorGroup.py | 413 ++++++++ GPUSimulators/Simulator.py | 286 ++++++ GPUSimulators/WAF.py | 241 +++++ GPUSimulators/__init__.py | 5 + .../__pycache__/MPISimulator.cpython-39.pyc | Bin 0 -> 11058 bytes .../__pycache__/Simulator.cpython-39.pyc | Bin 0 -> 8433 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 182 bytes GPUSimulators/cuda/EE2D_KP07_dimsplit.cu | 250 +++++ GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip | 251 +++++ GPUSimulators/cuda/EulerCommon.h | 187 ++++ GPUSimulators/cuda/SWE2D_FORCE.cu | 143 +++ GPUSimulators/cuda/SWE2D_FORCE.cu.hip | 144 +++ GPUSimulators/cuda/SWE2D_HLL.cu | 161 ++++ GPUSimulators/cuda/SWE2D_HLL.cu.hip | 162 ++++ GPUSimulators/cuda/SWE2D_HLL2.cu | 216 +++++ GPUSimulators/cuda/SWE2D_HLL2.cu.hip | 217 +++++ GPUSimulators/cuda/SWE2D_KP07.cu | 233 +++++ GPUSimulators/cuda/SWE2D_KP07.cu.hip | 234 +++++ GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu | 216 +++++ GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip | 217 +++++ GPUSimulators/cuda/SWE2D_LxF.cu | 168 ++++ GPUSimulators/cuda/SWE2D_LxF.cu.hip | 169 ++++ GPUSimulators/cuda/SWE2D_WAF.cu | 178 ++++ GPUSimulators/cuda/SWE2D_WAF.cu.hip | 179 ++++ GPUSimulators/cuda/SWECommon.h | 533 +++++++++++ GPUSimulators/cuda/common.h | 557 +++++++++++ GPUSimulators/cuda/limiters.h | 118 +++ GPUSimulators/helpers/InitialConditions.py | 355 +++++++ GPUSimulators/helpers/Visualization.py | 61 ++ 43 files changed, 10711 insertions(+) create mode 100644 GPUSimulators/Autotuner.py create mode 100644 GPUSimulators/Common.py create mode 100644 GPUSimulators/CudaContext.py create mode 100644 GPUSimulators/CudaContext_cu.py create mode 100644 GPUSimulators/EE2D_KP07_dimsplit.py create mode 100644 GPUSimulators/FORCE.py create mode 100644 GPUSimulators/HLL.py create mode 100644 GPUSimulators/HLL2.py create mode 100644 GPUSimulators/IPythonMagic.py create mode 100644 GPUSimulators/KP07.py create mode 100644 GPUSimulators/KP07_dimsplit.py create mode 100644 GPUSimulators/LxF.py create mode 100644 GPUSimulators/MPISimulator.py create mode 100644 GPUSimulators/SHMEMSimulator.py create mode 100644 GPUSimulators/SHMEMSimulatorGroup.py create mode 100644 GPUSimulators/Simulator.py create mode 100644 GPUSimulators/WAF.py create mode 100644 GPUSimulators/__init__.py create mode 100644 GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc create mode 100644 GPUSimulators/__pycache__/Simulator.cpython-39.pyc create mode 100644 GPUSimulators/__pycache__/__init__.cpython-39.pyc create mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu create mode 100644 GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip create mode 100644 GPUSimulators/cuda/EulerCommon.h create mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu create mode 100644 GPUSimulators/cuda/SWE2D_FORCE.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu create mode 100644 GPUSimulators/cuda/SWE2D_HLL.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu create mode 100644 GPUSimulators/cuda/SWE2D_HLL2.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu create mode 100644 GPUSimulators/cuda/SWE2D_KP07.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu create mode 100644 GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu create mode 100644 GPUSimulators/cuda/SWE2D_LxF.cu.hip create mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu create mode 100644 GPUSimulators/cuda/SWE2D_WAF.cu.hip create mode 100644 GPUSimulators/cuda/SWECommon.h create mode 100644 GPUSimulators/cuda/common.h create mode 100644 GPUSimulators/cuda/limiters.h create mode 100644 GPUSimulators/helpers/InitialConditions.py create mode 100644 GPUSimulators/helpers/Visualization.py diff --git a/GPUSimulators/Autotuner.py b/GPUSimulators/Autotuner.py new file mode 100644 index 0000000..84aedc2 --- /dev/null +++ b/GPUSimulators/Autotuner.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the different helper functions and +classes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import os +import gc +import numpy as np +import logging +from socket import gethostname + +#import pycuda.driver as cuda +from hip import hip,hiprtc + +from GPUSimulators import Common, Simulator, CudaContext + +class Autotuner: + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + nx=2048, ny=2048, + block_widths=range(8, 32, 1), + block_heights=range(8, 32, 1)): + logger = logging.getLogger(__name__) + self.filename = "autotuning_data_" + gethostname() + ".npz" + self.nx = nx + self.ny = ny + self.block_widths = block_widths + self.block_heights = block_heights + self.performance = {} + + + def benchmark(self, simulator, force=False): + logger = logging.getLogger(__name__) + + #Run through simulators and benchmark + key = str(simulator.__name__) + logger.info("Benchmarking %s to %s", key, self.filename) + + #If this simulator has been benchmarked already, skip it + if (force==False and os.path.isfile(self.filename)): + with np.load(self.filename) as data: + if key in data["simulators"]: + logger.info("%s already benchmarked - skipping", key) + return + + # Set arguments to send to the simulators during construction + context = CudaContext.CudaContext(autotuning=False) + g = 9.81 + h0, hu0, hv0, dx, dy, dt = Autotuner.gen_test_data(nx=self.nx, ny=self.ny, g=g) + arguments = { + 'context': context, + 'h0': h0, 'hu0': hu0, 'hv0': hv0, + 'nx': self.nx, 'ny': self.ny, + 'dx': dx, 'dy': dy, 'dt': 0.9*dt, + 'g': g + } + + # Load existing data into memory + benchmark_data = { + "simulators": [], + } + if (os.path.isfile(self.filename)): + with np.load(self.filename) as data: + for k, v in data.items(): + benchmark_data[k] = v + + # Run benchmark + benchmark_data[key + "_megacells"] = Autotuner.benchmark_single_simulator(simulator, arguments, self.block_widths, self.block_heights) + benchmark_data[key + "_block_widths"] = self.block_widths + benchmark_data[key + "_block_heights"] = self.block_heights + benchmark_data[key + "_arguments"] = str(arguments) + + existing_sims = benchmark_data["simulators"] + if (isinstance(existing_sims, np.ndarray)): + existing_sims = existing_sims.tolist() + if (key not in existing_sims): + benchmark_data["simulators"] = existing_sims + [key] + + # Save to file + np.savez_compressed(self.filename, **benchmark_data) + + + + """ + Function which reads a numpy file with autotuning data + and reports the maximum performance and block size + """ + def get_peak_performance(self, simulator): + logger = logging.getLogger(__name__) + + assert issubclass(simulator, Simulator.BaseSimulator) + key = simulator.__name__ + + if (key in self.performance): + return self.performance[key] + else: + #Run simulation if required + if (not os.path.isfile(self.filename)): + logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) + self.benchmark(simulator) + + with np.load(self.filename) as data: + if key not in data['simulators']: + logger.debug("Could not get autotuned peak performance for %s: benchmarking", key) + data.close() + self.benchmark(simulator) + data = np.load(self.filename) + + def find_max_index(megacells): + max_index = np.nanargmax(megacells) + return np.unravel_index(max_index, megacells.shape) + + megacells = data[key + '_megacells'] + block_widths = data[key + '_block_widths'] + block_heights = data[key + '_block_heights'] + j, i = find_max_index(megacells) + + self.performance[key] = { "block_width": block_widths[i], + "block_height": block_heights[j], + "megacells": megacells[j, i] } + logger.debug("Returning %s as peak performance parameters", self.performance[key]) + return self.performance[key] + + #This should never happen + raise "Something wrong: Could not get autotuning data!" + return None + + + + """ + Runs a set of benchmarks for a single simulator + """ + def benchmark_single_simulator(simulator, arguments, block_widths, block_heights): + logger = logging.getLogger(__name__) + + megacells = np.empty((len(block_heights), len(block_widths))) + megacells.fill(np.nan) + + logger.debug("Running %d benchmarks with %s", len(block_heights)*len(block_widths), simulator.__name__) + + sim_arguments = arguments.copy() + + with Common.Timer(simulator.__name__) as t: + for j, block_height in enumerate(block_heights): + sim_arguments.update({'block_height': block_height}) + for i, block_width in enumerate(block_widths): + sim_arguments.update({'block_width': block_width}) + megacells[j, i] = Autotuner.run_benchmark(simulator, sim_arguments) + + + logger.debug("Completed %s in %f seconds", simulator.__name__, t.secs) + + return megacells + + + """ + Runs a benchmark, and returns the number of megacells achieved + """ + def run_benchmark(simulator, arguments, timesteps=10, warmup_timesteps=2): + logger = logging.getLogger(__name__) + + #Initialize simulator + try: + sim = simulator(**arguments) + except: + #An exception raised - not possible to continue + logger.debug("Failed creating %s with arguments %s", simulator.__name__, str(arguments)) + return np.nan + + #Create timer events + #start = cuda.Event() + #end = cuda.Event() + stream = hip_check(hip.hipStreamCreate()) + + start = hip_check(hip.hipEventCreate()) + end = hip_check(hip.hipEventCreate()) + + #Warmup + for i in range(warmup_timesteps): + sim.stepEuler(sim.dt) + + #Run simulation with timer + #start.record(sim.stream) + #start recording + hip_check(hip.hipEventRecord(start, stream)) + for i in range(timesteps): + sim.stepEuler(sim.dt) + #end.record(sim.stream) + #stop recording and synchronize + hip_check(hip.hipEventRecord(end, stream)) + + #Synchronize end event + #end.synchronize() + hip_check(hip.hipEventSynchronize(end)) + + #Compute megacells + #gpu_elapsed = end.time_since(start)*1.0e-3 + gpu_elapsed = hip_check(hip.hipEventElapsedTime(start, end)) + + megacells = (sim.nx*sim.ny*timesteps / (1000*1000)) / gpu_elapsed + + #Sanity check solution + h, hu, hv = sim.download() + sane = True + sane = sane and Autotuner.sanity_check(h, 0.3, 0.7) + sane = sane and Autotuner.sanity_check(hu, -0.2, 0.2) + sane = sane and Autotuner.sanity_check(hv, -0.2, 0.2) + + if (sane): + logger.debug("%s [%d x %d] succeeded: %f megacells, gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], megacells, gpu_elapsed) + return megacells + else: + logger.debug("%s [%d x %d] failed: gpu elapsed %f", simulator.__name__, arguments["block_width"], arguments["block_height"], gpu_elapsed) + return np.nan + + + + """ + Generates test dataset + """ + def gen_test_data(nx, ny, g): + width = 100.0 + height = 100.0 + dx = width / float(nx) + dy = height / float(ny) + + x_center = dx*nx/2.0 + y_center = dy*ny/2.0 + + #Create a gaussian "dam break" that will not form shocks + size = width / 5.0 + dt = 10**10 + + h = np.zeros((ny, nx), dtype=np.float32); + hu = np.zeros((ny, nx), dtype=np.float32); + hv = np.zeros((ny, nx), dtype=np.float32); + + extent = 1.0/np.sqrt(2.0) + x = (dx*(np.arange(0, nx, dtype=np.float32)+0.5) - x_center) / size + y = (dy*(np.arange(0, ny, dtype=np.float32)+0.5) - y_center) / size + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + r = np.minimum(1.0, np.sqrt(xv**2 + yv**2)) + xv = None + yv = None + gc.collect() + + #Generate highres + cos = np.cos(np.pi*r) + h = 0.5 + 0.1*0.5*(1.0 + cos) + hu = 0.1*0.5*(1.0 + cos) + hv = hu.copy() + + scale = 0.7 + max_h_estimate = 0.6 + max_u_estimate = 0.1*np.sqrt(2.0) + dx = width/nx + dy = height/ny + dt = scale * min(dx, dy) / (max_u_estimate + np.sqrt(g*max_h_estimate)) + + return h, hu, hv, dx, dy, dt + + """ + Checks that a variable is "sane" + """ + def sanity_check(variable, bound_min, bound_max): + maxval = np.amax(variable) + minval = np.amin(variable) + if (np.isnan(maxval) + or np.isnan(minval) + or maxval > bound_max + or minval < bound_min): + return False + else: + return True diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py new file mode 100644 index 0000000..6681450 --- /dev/null +++ b/GPUSimulators/Common.py @@ -0,0 +1,879 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the different helper functions and +classes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import os + +import numpy as np +import time +import signal +import subprocess +import tempfile +import re +import io +import hashlib +import logging +import gc +import netCDF4 +import json + +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda +#from pycuda.tools import PageLockedMemoryPool + +from hip import hip, hiprtc +from hip import hipblas + +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + +def safeCall(cmd): + logger = logging.getLogger(__name__) + try: + #git rev-parse HEAD + current_dir = os.path.dirname(os.path.realpath(__file__)) + params = dict() + params['stderr'] = subprocess.STDOUT + params['cwd'] = current_dir + params['universal_newlines'] = True #text=True in more recent python + params['shell'] = False + if os.name == 'nt': + params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + stdout = subprocess.check_output(cmd, **params) + except subprocess.CalledProcessError as e: + output = e.output + logger.error("Git failed, \nReturn code: " + str(e.returncode) + "\nOutput: " + output) + raise e + + return stdout + +def getGitHash(): + return safeCall(["git", "rev-parse", "HEAD"]) + +def getGitStatus(): + return safeCall(["git", "status", "--porcelain", "-uno"]) + +def toJson(in_dict, compressed=True): + """ + Creates JSON string from a dictionary + """ + logger = logging.getLogger(__name__) + out_dict = in_dict.copy() + for key in out_dict: + if isinstance(out_dict[key], np.ndarray): + out_dict[key] = out_dict[key].tolist() + else: + try: + json.dumps(out_dict[key]) + except: + value = str(out_dict[key]) + logger.warning("JSON: Converting {:s} to string ({:s})".format(key, value)) + out_dict[key] = value + return json.dumps(out_dict) + +def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names=[], dt=None): + """ + Runs a simulation, and stores output in netcdf file. Stores the times given in + save_times, and saves all of the variables in list save_var_names. Elements in + save_var_names can be set to None if you do not want to save them + """ + profiling_data_sim_runner = { 'start': {}, 'end': {} } + profiling_data_sim_runner["start"]["t_sim_init"] = 0 + profiling_data_sim_runner["end"]["t_sim_init"] = 0 + profiling_data_sim_runner["start"]["t_nc_write"] = 0 + profiling_data_sim_runner["end"]["t_nc_write"] = 0 + profiling_data_sim_runner["start"]["t_full_step"] = 0 + profiling_data_sim_runner["end"]["t_full_step"] = 0 + + profiling_data_sim_runner["start"]["t_sim_init"] = time.time() + + logger = logging.getLogger(__name__) + + assert len(save_times) > 0, "Need to specify which times to save" + + with Timer("construct") as t: + sim = simulator(**simulator_args) + logger.info("Constructed in " + str(t.secs) + " seconds") + + #Create netcdf file and simulate + with DataDumper(outfile, mode='w', clobber=False) as outdata: + + #Create attributes (metadata) + outdata.ncfile.created = time.ctime(time.time()) + outdata.ncfile.git_hash = getGitHash() + outdata.ncfile.git_status = getGitStatus() + outdata.ncfile.simulator = str(simulator) + + # do not write fields to attributes (they are to large) + simulator_args_for_ncfile = simulator_args.copy() + del simulator_args_for_ncfile["rho"] + del simulator_args_for_ncfile["rho_u"] + del simulator_args_for_ncfile["rho_v"] + del simulator_args_for_ncfile["E"] + outdata.ncfile.sim_args = toJson(simulator_args_for_ncfile) + + #Create dimensions + outdata.ncfile.createDimension('time', len(save_times)) + outdata.ncfile.createDimension('x', simulator_args['nx']) + outdata.ncfile.createDimension('y', simulator_args['ny']) + + #Create variables for dimensions + ncvars = {} + ncvars['time'] = outdata.ncfile.createVariable('time', np.dtype('float32').char, 'time') + ncvars['x'] = outdata.ncfile.createVariable( 'x', np.dtype('float32').char, 'x') + ncvars['y'] = outdata.ncfile.createVariable( 'y', np.dtype('float32').char, 'y') + + #Fill variables with proper values + ncvars['time'][:] = save_times + extent = sim.getExtent() + ncvars['x'][:] = np.linspace(extent[0], extent[1], simulator_args['nx']) + ncvars['y'][:] = np.linspace(extent[2], extent[3], simulator_args['ny']) + + #Choose which variables to download (prune None from list, but keep the index) + download_vars = [] + for i, var_name in enumerate(save_var_names): + if var_name is not None: + download_vars += [i] + save_var_names = list(save_var_names[i] for i in download_vars) + + #Create variables + for var_name in save_var_names: + ncvars[var_name] = outdata.ncfile.createVariable(var_name, np.dtype('float32').char, ('time', 'y', 'x'), zlib=True, least_significant_digit=3) + + #Create step sizes between each save + t_steps = np.empty_like(save_times) + t_steps[0] = save_times[0] + t_steps[1:] = save_times[1:] - save_times[0:-1] + + profiling_data_sim_runner["end"]["t_sim_init"] = time.time() + + #Start simulation loop + progress_printer = ProgressPrinter(save_times[-1], print_every=10) + for k in range(len(save_times)): + #Get target time and step size there + t_step = t_steps[k] + t_end = save_times[k] + + #Sanity check simulator + try: + sim.check() + except AssertionError as e: + logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e))) + return outdata.filename + + profiling_data_sim_runner["start"]["t_full_step"] += time.time() + + #Simulate + if (t_step > 0.0): + sim.simulate(t_step, dt) + + profiling_data_sim_runner["end"]["t_full_step"] += time.time() + + profiling_data_sim_runner["start"]["t_nc_write"] += time.time() + + #Download + save_vars = sim.download(download_vars) + + #Save to file + for i, var_name in enumerate(save_var_names): + ncvars[var_name][k, :] = save_vars[i] + + profiling_data_sim_runner["end"]["t_nc_write"] += time.time() + + #Write progress to screen + print_string = progress_printer.getPrintString(t_end) + if (print_string): + logger.debug(print_string) + + logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps())) + + return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi + + + + + + +class Timer(object): + """ + Class which keeps track of time spent for a section of code + """ + def __init__(self, tag, log_level=logging.DEBUG): + self.tag = tag + self.log_level = log_level + self.logger = logging.getLogger(__name__) + + def __enter__(self): + self.start = time.time() + return self + + def __exit__(self, *args): + self.end = time.time() + self.secs = self.end - self.start + self.msecs = self.secs * 1000 # millisecs + self.logger.log(self.log_level, "%s: %f ms", self.tag, self.msecs) + + def elapsed(self): + return time.time() - self.start + + + + + +class PopenFileBuffer(object): + """ + Simple class for holding a set of tempfiles + for communicating with a subprocess + """ + def __init__(self): + self.stdout = tempfile.TemporaryFile(mode='w+t') + self.stderr = tempfile.TemporaryFile(mode='w+t') + + def __del__(self): + self.stdout.close() + self.stderr.close() + + def read(self): + self.stdout.seek(0) + cout = self.stdout.read() + self.stdout.seek(0, 2) + + self.stderr.seek(0) + cerr = self.stderr.read() + self.stderr.seek(0, 2) + + return cout, cerr + +class IPEngine(object): + """ + Class for starting IPEngines for MPI processing in IPython + """ + def __init__(self, n_engines): + self.logger = logging.getLogger(__name__) + + #Start ipcontroller + self.logger.info("Starting IPController") + self.c_buff = PopenFileBuffer() + c_cmd = ["ipcontroller", "--ip='*'"] + c_params = dict() + c_params['stderr'] = self.c_buff.stderr + c_params['stdout'] = self.c_buff.stdout + c_params['shell'] = False + if os.name == 'nt': + c_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + self.c = subprocess.Popen(c_cmd, **c_params) + + #Wait until controller is running + time.sleep(3) + + #Start engines + self.logger.info("Starting IPEngines") + self.e_buff = PopenFileBuffer() + e_cmd = ["mpiexec", "-n", str(n_engines), "ipengine", "--mpi"] + e_params = dict() + e_params['stderr'] = self.e_buff.stderr + e_params['stdout'] = self.e_buff.stdout + e_params['shell'] = False + if os.name == 'nt': + e_params['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + self.e = subprocess.Popen(e_cmd, **e_params) + + # attach to a running cluster + import ipyparallel + self.cluster = ipyparallel.Client()#profile='mpi') + time.sleep(3) + while(len(self.cluster.ids) != n_engines): + time.sleep(0.5) + self.logger.info("Waiting for cluster...") + self.cluster = ipyparallel.Client()#profile='mpi') + + self.logger.info("Done") + + def __del__(self): + self.shutdown() + + def shutdown(self): + if (self.e is not None): + if (os.name == 'nt'): + self.logger.warn("Sending CTRL+C to IPEngine") + self.e.send_signal(signal.CTRL_C_EVENT) + + try: + self.e.communicate(timeout=3) + self.e.kill() + except subprocess.TimeoutExpired: + self.logger.warn("Killing IPEngine") + self.e.kill() + self.e.communicate() + self.e = None + + cout, cerr = self.e_buff.read() + self.logger.info("IPEngine cout: {:s}".format(cout)) + self.logger.info("IPEngine cerr: {:s}".format(cerr)) + self.e_buff = None + + gc.collect() + + if (self.c is not None): + if (os.name == 'nt'): + self.logger.warn("Sending CTRL+C to IPController") + self.c.send_signal(signal.CTRL_C_EVENT) + + try: + self.c.communicate(timeout=3) + self.c.kill() + except subprocess.TimeoutExpired: + self.logger.warn("Killing IPController") + self.c.kill() + self.c.communicate() + self.c = None + + cout, cerr = self.c_buff.read() + self.logger.info("IPController cout: {:s}".format(cout)) + self.logger.info("IPController cerr: {:s}".format(cerr)) + self.c_buff = None + + gc.collect() + + + + + + +class DataDumper(object): + """ + Simple class for holding a netCDF4 object + (handles opening and closing in a nice way) + Use as + with DataDumper("filename") as data: + ... + """ + def __init__(self, filename, *args, **kwargs): + self.logger = logging.getLogger(__name__) + + #Create directory if needed + filename = os.path.abspath(filename) + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + self.logger.info("Creating directory " + dirname) + os.makedirs(dirname) + + #Get mode of file if we have that + mode = None + if (args): + mode = args[0] + elif (kwargs and 'mode' in kwargs.keys()): + mode = kwargs['mode'] + + #Create new unique file if writing + if (mode): + if (("w" in mode) or ("+" in mode) or ("a" in mode)): + i = 0 + stem, ext = os.path.splitext(filename) + while (os.path.isfile(filename)): + filename = "{:s}_{:04d}{:s}".format(stem, i, ext) + i = i+1 + self.filename = os.path.abspath(filename) + + #Save arguments + self.args = args + self.kwargs = kwargs + + #Log output + self.logger.info("Initialized " + self.filename) + + + def __enter__(self): + self.logger.info("Opening " + self.filename) + if (self.args): + self.logger.info("Arguments: " + str(self.args)) + if (self.kwargs): + self.logger.info("Keyword arguments: " + str(self.kwargs)) + self.ncfile = netCDF4.Dataset(self.filename, *self.args, **self.kwargs) + return self + + def __exit__(self, *args): + self.logger.info("Closing " + self.filename) + self.ncfile.close() + + + def toJson(in_dict): + out_dict = in_dict.copy() + + for key in out_dict: + if isinstance(out_dict[key], np.ndarray): + out_dict[key] = out_dict[key].tolist() + else: + try: + json.dumps(out_dict[key]) + except: + out_dict[key] = str(out_dict[key]) + + return json.dumps(out_dict) + + + + + +class ProgressPrinter(object): + """ + Small helper class for + """ + def __init__(self, total_steps, print_every=5): + self.logger = logging.getLogger(__name__) + self.start = time.time() + self.total_steps = total_steps + self.print_every = print_every + self.next_print_time = self.print_every + self.last_step = 0 + self.secs_per_iter = None + + def getPrintString(self, step): + elapsed = time.time() - self.start + if (elapsed > self.next_print_time): + dt = elapsed - (self.next_print_time - self.print_every) + dsteps = step - self.last_step + steps_remaining = self.total_steps - step + + if (dsteps == 0): + return + + self.last_step = step + self.next_print_time = elapsed + self.print_every + + if not self.secs_per_iter: + self.secs_per_iter = dt / dsteps + self.secs_per_iter = 0.2*self.secs_per_iter + 0.8*(dt / dsteps) + + remaining_time = steps_remaining * self.secs_per_iter + + return "{:s}. Total: {:s}, elapsed: {:s}, remaining: {:s}".format( + ProgressPrinter.progressBar(step, self.total_steps), + ProgressPrinter.timeString(elapsed + remaining_time), + ProgressPrinter.timeString(elapsed), + ProgressPrinter.timeString(remaining_time)) + + def timeString(seconds): + seconds = int(max(seconds, 1)) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + periods = [('h', hours), ('m', minutes), ('s', seconds)] + time_string = ' '.join('{}{}'.format(value, name) + for name, value in periods + if value) + return time_string + + def progressBar(step, total_steps, width=30): + progress = np.round(width * step / total_steps).astype(np.int32) + progressbar = "0% [" + "#"*(progress) + "="*(width-progress) + "] 100%" + return progressbar + + + + + + + +""" +Class that holds 2D data +""" +class CudaArray2D: + """ + Uploads initial data to the CUDA device + """ + def __init__(self, stream, nx, ny, x_halo, y_halo, cpu_data=None, dtype=np.float32): + self.logger = logging.getLogger(__name__) + self.nx = nx + self.ny = ny + self.x_halo = x_halo + self.y_halo = y_halo + + nx_halo = nx + 2*x_halo + ny_halo = ny + 2*y_halo + + #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny) + #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here + #Initialize an array on GPU with zeros + #self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) + self.data_h = np.zeros((ny_halo, nx_halo), dtype="float32") + num_bytes = self.data_h.size * self.data_h.itemsize + + # init device array and upload host data + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(ny_halo, nx_halo)) + + # copy data from host to device + hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + + #For returning to download (No counterpart in hip-python) + #self.memorypool = PageLockedMemoryPool() + + #If we don't have any data, just allocate and return + if cpu_data is None: + return + + #Make sure data is in proper format + assert cpu_data.shape == (ny_halo, nx_halo) or cpu_data.shape == (self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.ny, self.nx)), str((ny_halo, nx_halo))) + assert cpu_data.itemsize == 4, "Wrong size of data type" + assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" + + #Create copy object from host to device + x = (nx_halo - cpu_data.shape[1]) // 2 + y = (ny_halo - cpu_data.shape[0]) // 2 + self.upload(stream, cpu_data, extent=[x, y, cpu_data.shape[1], cpu_data.shape[0]]) + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) + + + def __del__(self, *args): + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) + self.data.gpudata.free() + self.data = None + + """ + Enables downloading data from GPU to Python + """ + def download(self, stream, cpu_data=None, asynch=False, extent=None): + if (extent is None): + x = self.x_halo + y = self.y_halo + nx = self.nx + ny = self.ny + else: + x, y, nx, ny = extent + + if (cpu_data is None): + #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) + #Allocate host memory + #The following fails, don't know why (crashes python) + #cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) + #see here type of memory: https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemoryType + cpu_data = np.empty((ny, nx), dtype=np.float32) + num_bytes = cpu_data.size * cpu_data.itemsize + #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can #be accessed directly by the GPU device + #hipHostMallocDefault:Memory is mapped and portable (default allocation) + #hipHostMallocPortable: memory is explicitely portable across different devices + cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) + #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32) + #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32) + + assert nx == cpu_data.shape[1] + assert ny == cpu_data.shape[0] + assert x+nx <= self.nx + 2*self.x_halo + assert y+ny <= self.ny + 2*self.y_halo + + #Create copy object from device to host + #copy = cuda.Memcpy2D() + #copy.set_src_device(self.data.gpudata) + #copy.set_dst_host(cpu_data) + + #Set offsets and pitch of source + #copy.src_x_in_bytes = int(x)*self.data.strides[1] + #copy.src_y = int(y) + #copy.src_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + #copy.width_in_bytes = int(nx)*cpu_data.itemsize + #copy.height = int(ny) + + #The equivalent of cuda.Memcpy2D in hip-python would be: but it fails with an error pointing to cpu_data + #and a message: "RuntimeError: hipError_t.hipErrorInvalidValue" + #shape = (nx,ny) + #num_bytes = cpu_data.size * cpu_data.itemsize + #dst_pitch_bytes = cpu_data.strides[0] + #src_pitch_bytes = num_bytes // shape[0] + #src_pitch_bytes = data.strides[0] + #width_bytes = int(nx)*cpu_data.itemsize + #height_Nrows = int(ny) + #hipMemcpy2D(dst, unsigned long dpitch, src, unsigned long spitch, unsigned long width, unsigned long height, kind) + #copy = hip_check(hip.hipMemcpy2D(cpu_data, #pointer to destination + # dst_pitch_bytes, #pitch of destination array + # data, #pointer to source + # src_pitch_bytes, #pitch of source array + # width_bytes, #number of bytes in each row + # height_Nrows, #number of rows to copy + # hip.hipMemcpyKind.hipMemcpyDeviceToHost)) #kind + + #this is an alternative: + #copy from device to host + cpu_data = np.empty((ny, nx), dtype=np.float32) + num_bytes = cpu_data.size * cpu_data.itemsize + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + + copy(stream) + if asynch==False: + stream.synchronize() + + return cpu_data + + + def upload(self, stream, cpu_data, extent=None): + if (extent is None): + x = self.x_halo + y = self.y_halo + nx = self.nx + ny = self.ny + else: + x, y, nx, ny = extent + + assert(nx == cpu_data.shape[1]) + assert(ny == cpu_data.shape[0]) + assert(x+nx <= self.nx + 2*self.x_halo) + assert(y+ny <= self.ny + 2*self.y_halo) + + #Create copy object from device to host + #Well this copy from src:host to dst:device AND NOT from device to host + #copy = cuda.Memcpy2D() + #copy.set_dst_device(self.data.gpudata) + #copy.set_src_host(cpu_data) + + #Set offsets and pitch of source + #copy.dst_x_in_bytes = int(x)*self.data.strides[1] + #copy.dst_y = int(y) + #copy.dst_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + #copy.width_in_bytes = int(nx)*cpu_data.itemsize + #copy.height = int(ny) + + #copy from host de device + num_bytes = cpu_data.size * cpu_data.itemsize + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=cpu_data.shape) + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + + copy(stream) + + + + +""" +Class that holds 2D data +""" +class CudaArray3D: + """ + Uploads initial data to the CL device + """ + def __init__(self, stream, nx, ny, nz, x_halo, y_halo, z_halo, cpu_data=None, dtype=np.float32): + self.logger = logging.getLogger(__name__) + self.nx = nx + self.ny = ny + self.nz = nz + self.x_halo = x_halo + self.y_halo = y_halo + self.z_halo = z_halo + + nx_halo = nx + 2*x_halo + ny_halo = ny + 2*y_halo + nz_halo = nz + 2*z_halo + + #self.logger.debug("Allocating [%dx%dx%d] buffer", self.nx, self.ny, self.nz) + #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here + #self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) + + self.data_h = np.zeros((nz_halo, ny_halo, nx_halo), dtype="float32") + num_bytes = self.data_h.size * self.data_h.itemsize + + # init device array and upload host data + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(nz_halo, ny_halo, nx_halo)) + + # copy data from host to device + hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + + #For returning to download + #self.memorypool = PageLockedMemoryPool() + + #If we don't have any data, just allocate and return + if cpu_data is None: + return + + #Make sure data is in proper format + assert cpu_data.shape == (nz_halo, ny_halo, nx_halo) or cpu_data.shape == (self.nz, self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.nz, self.ny, self.nx)), str((nz_halo, ny_halo, nx_halo))) + assert cpu_data.itemsize == 4, "Wrong size of data type" + assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" + + #Create copy object from host to device + #copy = cuda.Memcpy3D() + #copy.set_src_host(cpu_data) + #copy.set_dst_device(self.data.gpudata) + + #Set offsets of destination + #x_offset = (nx_halo - cpu_data.shape[2]) // 2 + #y_offset = (ny_halo - cpu_data.shape[1]) // 2 + #z_offset = (nz_halo - cpu_data.shape[0]) // 2 + #copy.dst_x_in_bytes = x_offset*self.data.strides[1] + #copy.dst_y = y_offset + #copy.dst_z = z_offset + + #Set pitch of destination + #copy.dst_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + #width = max(self.nx, cpu_data.shape[2]) + #height = max(self.ny, cpu_data.shape[1]) + #depth = max(self.nz, cpu-data.shape[0]) + #copy.width_in_bytes = width*cpu_data.itemsize + #copy.height = height + #copy.depth = depth + + #copy from host to device + num_bytes = cpu_data.size * cpu_data.itemsize + self.data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=cpu_data.shape) + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + + #Perform the copy + copy(stream) + + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) + + + def __del__(self, *args): + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) + self.data.gpudata.free() + self.data = None + + """ + Enables downloading data from GPU to Python + """ + def download(self, stream, asynch=False): + #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) + #Allocate host memory + #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32) + cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) + #cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) + + #Create copy object from device to host + #copy = cuda.Memcpy2D() + #copy.set_src_device(self.data.gpudata) + #copy.set_dst_host(cpu_data) + + #Set offsets and pitch of source + #copy.src_x_in_bytes = self.x_halo*self.data.strides[1] + #copy.src_y = self.y_halo + #copy.src_z = self.z_halo + #copy.src_pitch = self.data.strides[0] + + #Set width in bytes to copy for each row and + #number of rows to copy + #copy.width_in_bytes = self.nx*cpu_data.itemsize + #copy.height = self.ny + #copy.depth = self.nz + + #copy from device to host + num_bytes = cpu_data.size * cpu_data.itemsize + #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) + copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + + copy(stream) + if asynch==False: + stream.synchronize() + + return cpu_data + + +""" +A class representing an Arakawa A type (unstaggered, logically Cartesian) grid +""" +class ArakawaA2D: + def __init__(self, stream, nx, ny, halo_x, halo_y, cpu_variables): + """ + Uploads initial data to the GPU device + """ + self.logger = logging.getLogger(__name__) + self.gpu_variables = [] + for cpu_variable in cpu_variables: + self.gpu_variables += [CudaArray2D(stream, nx, ny, halo_x, halo_y, cpu_variable)] + + def __getitem__(self, key): + assert type(key) == int, "Indexing is int based" + if (key > len(self.gpu_variables) or key < 0): + raise IndexError("Out of bounds") + return self.gpu_variables[key] + + def download(self, stream, variables=None): + """ + Enables downloading data from the GPU device to Python + """ + if variables is None: + variables=range(len(self.gpu_variables)) + + cpu_variables = [] + for i in variables: + assert i < len(self.gpu_variables), "Variable {:d} is out of range".format(i) + cpu_variables += [self.gpu_variables[i].download(stream, asynch=True)] + + #stream.synchronize() + return cpu_variables + + #hipblas + def sum_hipblas(self, num_elements, data): + num_bytes_r = np.dtype(np.float32).itemsize + result_d = hip_check(hip.hipMalloc(num_bytes_r)) + result_h = np.zeros(1, dtype=np.float32) + print("--bytes:", num_bytes_r) + + # call hipblasSaxpy + initialization + handle = hip_check(hipblas.hipblasCreate()) + #hip_check(hipblas.hipblasSaxpy(handle, num_elements, ctypes.addressof(alpha), x_d, 1, y_d, 1)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasSasum(handle, num_elements, data, 1, result_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in result_d) back to host (store in result_h) + hip_check(hip.hipMemcpy(result_h,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + + # clean up + hip_check(hip.hipFree(data)) + return result_h + + def check(self): + """ + Checks that data is still sane + """ + for i, gpu_variable in enumerate(self.gpu_variables): + #compute sum with hipblas + #var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() + var_sum = self.sum_hipblas(gpu_variable.ny,gpu_variable.data) + + self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny)) + assert np.isnan(var_sum) == False, "Data contains NaN values!" + diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py new file mode 100644 index 0000000..e77ef06 --- /dev/null +++ b/GPUSimulators/CudaContext.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements Cuda context handling + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import os + +import numpy as np +import time +import re +import io +import hashlib +import logging +import gc + +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda + +from hip import hip,hiprtc +from hip import rccl + +from GPUSimulators import Autotuner, Common + + +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): + """ + Create a new CUDA context + Set device to an id or pci_bus_id to select a specific GPU + Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context + """ + self.use_cache = use_cache + self.logger = logging.getLogger(__name__) + self.modules = {} + + self.module_path = os.path.dirname(os.path.realpath(__file__)) + + #Initialize cuda (must be first call to PyCUDA) + ##cuda.init(flags=0) + + ##self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) + + #Print some info about CUDA + ##self.logger.info("CUDA version %s", str(cuda.get_version())) + #self.logger.info("Driver version %s", str(cuda.get_driver_version())) + self.logger.info("Driver version %s", str(hip_check(hip.hipDriverGetVersion()))) + + if device is None: + device = 0 + + self.cuda_device = hip.Device(device) + #self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) + self.logger.info("Using device %d/%d '%s' (%s) GPU", device, hip_check(hip.hipGetDeviceCount())) + #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) + self.logger.debug(" => compute capability: %s", str(self.hip.hipDeviceComputeCapability(device))) + + # Create the CUDA context + #In HIP there is no need to specify a scheduling policy (it is abstracted). Here the HIP runtime system manages the workload to fit a specifc target architecture + #if context_flags is None: + # context_flags=cuda.ctx_flags.SCHED_AUTO + + #self.cuda_context = self.cuda_device.make_context(flags=context_flags) + + #free, total = cuda.mem_get_info() + total = hip_check(hip.hipDeviceTotalMem(device)) + #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) + self.logger.debug(" => memory: %d / %d MB available", int(total/(1024*1024))) + + #self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + + #Create cache dir for cubin files + self.cache_path = os.path.join(self.module_path, "cuda_cache") + if (self.use_cache): + if not os.path.isdir(self.cache_path): + os.mkdir(self.cache_path) + self.logger.info("Using CUDA cache dir %s", self.cache_path) + + self.autotuner = None + if (autotuning): + self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") + self.autotuner = Autotuner.Autotuner() + + +# def __del__(self, *args): +# self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + + # Loop over all contexts in stack, and remove "this" +# other_contexts = [] +# while (cuda.Context.get_current() != None): +# context = cuda.Context.get_current() +# if (context.handle != self.cuda_context.handle): +# self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) +# other_contexts = [context] + other_contexts +# cuda.Context.pop() +# else: +# self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) +# cuda.Context.pop() + + # Add all the contexts we popped that were not our own +# for context in other_contexts: +# self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) +# cuda.Context.push(context) + +# self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) +# self.cuda_context.detach() + + +# def __str__(self): +# return "CudaContext id " + str(self.cuda_context.handle) + + + def hash_kernel(kernel_filename, include_dirs): + # Generate a kernel ID for our caches + num_includes = 0 + max_includes = 100 + kernel_hasher = hashlib.md5() + logger = logging.getLogger(__name__) + + # Loop over file and includes, and check if something has changed + files = [kernel_filename] + while len(files): + + if (num_includes > max_includes): + raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) + + filename = files.pop() + + #logger.debug("Hashing %s", filename) + + modified = os.path.getmtime(filename) + + # Open the file + with io.open(filename, "r") as file: + + # Search for #inclue and also hash the file + file_str = file.read() + kernel_hasher.update(file_str.encode('utf-8')) + kernel_hasher.update(str(modified).encode('utf-8')) + + #Find all includes + includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) + + # Loop over everything that looks like an include + for include_file in includes: + + #Search through include directories for the file + file_path = os.path.dirname(filename) + for include_path in [file_path] + include_dirs: + + # If we find it, add it to list of files to check + temp_path = os.path.join(include_path, include_file) + if (os.path.isfile(temp_path)): + files = files + [temp_path] + num_includes = num_includes + 1 #For circular includes... + break + + return kernel_hasher.hexdigest() + + + """ + Reads a text file and creates an OpenCL kernel from that + """ + def get_module(self, kernel_filename, + include_dirs=[], \ + defines={}, \ + compile_args={'no_extern_c', True}, jit_compile_args={}): + """ + Helper function to print compilation output + """ + def cuda_compile_message_handler(compile_success_bool, info_str, error_str): + self.logger.debug("Compilation returned %s", str(compile_success_bool)) + if info_str: + self.logger.debug("Info: %s", info_str) + if error_str: + self.logger.debug("Error: %s", error_str) + + kernel_filename = os.path.normpath(kernel_filename) + kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) + #self.logger.debug("Getting %s", kernel_filename) + + # Create a hash of the kernel options + options_hasher = hashlib.md5() + options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); + options_hash = options_hasher.hexdigest() + + # Create hash of kernel souce + source_hash = CudaContext.hash_kernel( \ + kernel_path, \ + include_dirs=[self.module_path] + include_dirs) + + # Create final hash + root, ext = os.path.splitext(kernel_filename) + kernel_hash = root \ + + "_" + source_hash \ + + "_" + options_hash \ + + ext + cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) + + # If we have the kernel in our hashmap, return it + if (kernel_hash in self.modules.keys()): + self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) + return self.modules[kernel_hash] + + # If we have it on disk, return it + elif (self.use_cache and os.path.isfile(cached_kernel_filename)): + self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) + + with io.open(cached_kernel_filename, "rb") as file: + file_str = file.read() + #No hip counterpart of module_from_buffer + module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) + + self.modules[kernel_hash] = module + return module + + # Otherwise, compile it from source + else: + self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) + + #Create kernel string + kernel_string = "" + for key, value in defines.items(): + kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) + kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) + if (self.use_cache): + cached_kernel_dir = os.path.dirname(cached_kernel_filename) + if not os.path.isdir(cached_kernel_dir): + os.mkdir(cached_kernel_dir) + with io.open(cached_kernel_filename + ".txt", "w") as file: + file.write(kernel_string) + + + with Common.Timer("compiler") as timer: + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) + + #cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + #module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + + #HIP version of compilation: but "name_of_fct" needs to be defined. e.g. + #source = b"""\ + #extern "C" __global__ void name_of_fct(float factor, int n, short unused1, int unused2, float unused3, float *x) { + #int tid = threadIdx.x + blockIdx.x * blockDim.x; + #if (tid < n) { + #x[tid] *= factor; + # } + #} + #""" + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_string, b"name_of_fct", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + #kernel = hip_check(hip.hipModuleGetFunction(module, b"name_of_fct")) + + if (self.use_cache): + with io.open(cached_kernel_filename, "wb") as file: + file.write(cubin) + + self.modules[kernel_hash] = module + return module + + """ + Clears the kernel cache (useful for debugging & development) + """ + def clear_kernel_cache(self): + self.logger.debug("Clearing cache") + self.modules = {} + gc.collect() + + """ + Synchronizes all streams etc + """ + def synchronize(self): + self.cuda_context.synchronize() diff --git a/GPUSimulators/CudaContext_cu.py b/GPUSimulators/CudaContext_cu.py new file mode 100644 index 0000000..6c90636 --- /dev/null +++ b/GPUSimulators/CudaContext_cu.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements Cuda context handling + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import os + +import numpy as np +import time +import re +import io +import hashlib +import logging +import gc + +import pycuda.compiler as cuda_compiler +import pycuda.gpuarray +import pycuda.driver as cuda + +from GPUSimulators import Autotuner, Common + + + +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): + """ + Create a new CUDA context + Set device to an id or pci_bus_id to select a specific GPU + Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context + """ + self.use_cache = use_cache + self.logger = logging.getLogger(__name__) + self.modules = {} + + self.module_path = os.path.dirname(os.path.realpath(__file__)) + + #Initialize cuda (must be first call to PyCUDA) + cuda.init(flags=0) + + self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) + + #Print some info about CUDA + self.logger.info("CUDA version %s", str(cuda.get_version())) + self.logger.info("Driver version %s", str(cuda.get_driver_version())) + + if device is None: + device = 0 + + self.cuda_device = cuda.Device(device) + self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) + self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) + + # Create the CUDA context + if context_flags is None: + context_flags=cuda.ctx_flags.SCHED_AUTO + + self.cuda_context = self.cuda_device.make_context(flags=context_flags) + + free, total = cuda.mem_get_info() + self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) + + self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + + #Create cache dir for cubin files + self.cache_path = os.path.join(self.module_path, "cuda_cache") + if (self.use_cache): + if not os.path.isdir(self.cache_path): + os.mkdir(self.cache_path) + self.logger.info("Using CUDA cache dir %s", self.cache_path) + + self.autotuner = None + if (autotuning): + self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") + self.autotuner = Autotuner.Autotuner() + + + def __del__(self, *args): + self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + + # Loop over all contexts in stack, and remove "this" + other_contexts = [] + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + if (context.handle != self.cuda_context.handle): + self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) + other_contexts = [context] + other_contexts + cuda.Context.pop() + else: + self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) + cuda.Context.pop() + + # Add all the contexts we popped that were not our own + for context in other_contexts: + self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) + cuda.Context.push(context) + + self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) + self.cuda_context.detach() + + + def __str__(self): + return "CudaContext id " + str(self.cuda_context.handle) + + + def hash_kernel(kernel_filename, include_dirs): + # Generate a kernel ID for our caches + num_includes = 0 + max_includes = 100 + kernel_hasher = hashlib.md5() + logger = logging.getLogger(__name__) + + # Loop over file and includes, and check if something has changed + files = [kernel_filename] + while len(files): + + if (num_includes > max_includes): + raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) + + filename = files.pop() + + #logger.debug("Hashing %s", filename) + + modified = os.path.getmtime(filename) + + # Open the file + with io.open(filename, "r") as file: + + # Search for #inclue and also hash the file + file_str = file.read() + kernel_hasher.update(file_str.encode('utf-8')) + kernel_hasher.update(str(modified).encode('utf-8')) + + #Find all includes + includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) + + # Loop over everything that looks like an include + for include_file in includes: + + #Search through include directories for the file + file_path = os.path.dirname(filename) + for include_path in [file_path] + include_dirs: + + # If we find it, add it to list of files to check + temp_path = os.path.join(include_path, include_file) + if (os.path.isfile(temp_path)): + files = files + [temp_path] + num_includes = num_includes + 1 #For circular includes... + break + + return kernel_hasher.hexdigest() + + + """ + Reads a text file and creates an OpenCL kernel from that + """ + def get_module(self, kernel_filename, + include_dirs=[], \ + defines={}, \ + compile_args={'no_extern_c', True}, jit_compile_args={}): + """ + Helper function to print compilation output + """ + def cuda_compile_message_handler(compile_success_bool, info_str, error_str): + self.logger.debug("Compilation returned %s", str(compile_success_bool)) + if info_str: + self.logger.debug("Info: %s", info_str) + if error_str: + self.logger.debug("Error: %s", error_str) + + kernel_filename = os.path.normpath(kernel_filename) + kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) + #self.logger.debug("Getting %s", kernel_filename) + + # Create a hash of the kernel options + options_hasher = hashlib.md5() + options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); + options_hash = options_hasher.hexdigest() + + # Create hash of kernel souce + source_hash = CudaContext.hash_kernel( \ + kernel_path, \ + include_dirs=[self.module_path] + include_dirs) + + # Create final hash + root, ext = os.path.splitext(kernel_filename) + kernel_hash = root \ + + "_" + source_hash \ + + "_" + options_hash \ + + ext + cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) + + # If we have the kernel in our hashmap, return it + if (kernel_hash in self.modules.keys()): + self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) + return self.modules[kernel_hash] + + # If we have it on disk, return it + elif (self.use_cache and os.path.isfile(cached_kernel_filename)): + self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) + + with io.open(cached_kernel_filename, "rb") as file: + file_str = file.read() + module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) + + self.modules[kernel_hash] = module + return module + + # Otherwise, compile it from source + else: + self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) + + #Create kernel string + kernel_string = "" + for key, value in defines.items(): + kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) + kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) + if (self.use_cache): + cached_kernel_dir = os.path.dirname(cached_kernel_filename) + if not os.path.isdir(cached_kernel_dir): + os.mkdir(cached_kernel_dir) + with io.open(cached_kernel_filename + ".txt", "w") as file: + file.write(kernel_string) + + + with Common.Timer("compiler") as timer: + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) + cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) + if (self.use_cache): + with io.open(cached_kernel_filename, "wb") as file: + file.write(cubin) + + self.modules[kernel_hash] = module + return module + + """ + Clears the kernel cache (useful for debugging & development) + """ + def clear_kernel_cache(self): + self.logger.debug("Clearing cache") + self.modules = {} + gc.collect() + + """ + Synchronizes all streams etc + """ + def synchronize(self): + self.cuda_context.synchronize() \ No newline at end of file diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py new file mode 100644 index 0000000..935eb90 --- /dev/null +++ b/GPUSimulators/EE2D_KP07_dimsplit.py @@ -0,0 +1,575 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the 2nd order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class EE2D_KP07_dimsplit (BaseSimulator): + + """ + Initialization routine + rho: Density + rho_u: Momentum along x-axis + rho_v: Momentum along y-axis + E: energy + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis + dy: Grid cell spacing along y-axis + dt: Size of each timestep + g: Gravitational constant + gamma: Gas constant + p: pressure + """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + rho, rho_u, rho_v, E, + nx, ny, + dx, dy, + g, + gamma, + theta=1.3, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=8): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height) + self.g = np.float32(g) + self.gamma = np.float32(gamma) + self.theta = np.float32(theta) + + #Get kernels + #module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", + # defines={ + # 'BLOCK_WIDTH': self.block_size[0], + # 'BLOCK_HEIGHT': self.block_size[1] + # }, + # compile_args={ + # 'no_extern_c': True, + # 'options': ["--use_fast_math"], + # }, + # jit_compile_args={}) + #self.kernel = module.get_function("KP07DimsplitKernel") + #self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") + # + kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [rho, rho_u, rho_v, E]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + # init device array cfl_data + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho))) + dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho))) + self.dt = min(dt_x, dt_y) + self.cfl_data.fill(self.dt, stream=self.stream) + + + def substep(self, dt, step_number, external=True, internal=True): + self.substepDimsplit(0.5*dt, step_number, external, internal) + + def substepDimsplit(self, dt, substep, external, internal): + if external and internal: + #print("COMPLETE DOMAIN (dt=" + str(dt) + ")") + +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# self.nx, self.ny) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--External & Internal: Launching Kernel is ok") + + return + + if external and not internal: + ################################### + # XXX: Corners are treated twice! # + ################################### + + ns_grid_size = (self.grid_size[0], 1) + + # NORTH + # (x0, y0) x (x1, y1) + # (0, ny-y_halo) x (nx, ny) +# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, self.ny - int(self.u0[0].y_halo), +# self.nx, self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *ns_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo), + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + + + # SOUTH + # (x0, y0) x (x1, y1) + # (0, 0) x (nx, y_halo) +# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# self.nx, int(self.u0[0].y_halo)) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *ns_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo) + ) + ) + ) + + + we_grid_size = (1, self.grid_size[1]) + + # WEST + # (x0, y0) x (x1, y1) + # (0, 0) x (x_halo, ny) +# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# 0, 0, +# int(self.u0[0].x_halo), self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *we_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + 0, 0, + ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) + ) + ) + ) + + + # EAST + # (x0, y0) x (x1, y1) + # (nx-x_halo, 0) x (nx, ny) +# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.gamma, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u0[3].data.gpudata, self.u0[3].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.u1[3].data.gpudata, self.u1[3].data.strides[0], +# self.cfl_data.gpudata, +# self.nx - int(self.u0[0].x_halo), 0, +# self.nx, self.ny) + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *we_grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), 0, + ctypes.c_int(self.nx), ctypes.c_int(self.ny) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--External and not Internal: Launching Kernel is ok") + + return + + if internal and not external: + + # INTERNAL DOMAIN + # (x0, y0) x (x1, y1) + # (x_halo, y_halo) x (nx - x_halo, ny - y_halo) + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.internal_stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + int(self.u0[0].x_halo), int(self.u0[0].y_halo), + self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo)) + + + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.internal_stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.gamma), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.cfl_data, + ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.u0[0].y_halo), + ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo) + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Internal and not External: Launching Kernel is ok") + return + + def swapBuffers(self): + self.u0, self.u1 = self.u1, self.u0 + return + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + return + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py new file mode 100644 index 0000000..092711a --- /dev/null +++ b/GPUSimulators/FORCE.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the FORCE flux +for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + + + + + + +""" +Class that solves the SW equations +""" +class FORCE (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height) + self.g = np.float32(g) + + #Get kernels +# module = context.get_module("cuda/SWE2D_FORCE.cu.hip", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("FORCEKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_FORCE.cu')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .FORCEKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"FORCEKernel")) + + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) +# self.u0, self.u1 = self.u1, self.u0 + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 + + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .FORCEKernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py new file mode 100644 index 0000000..792d3c6 --- /dev/null +++ b/GPUSimulators/HLL.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + + +""" +Class that solves the SW equations using the Harten-Lax -van Leer approximate Riemann solver +""" +class HLL (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height); + self.g = np.float32(g) + + #Get kernels +# module = context.get_module("cuda/SWE2D_HLL.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("HLLKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .HLLKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"HLLKernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .HLLKernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py new file mode 100644 index 0000000..b5c0dc0 --- /dev/null +++ b/GPUSimulators/HLL2.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the 2nd order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class HLL2 (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.8, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height); + self.g = np.float32(g) + self.theta = np.float32(theta) + + #Get kernels +# module = context.get_module("cuda/SWE2D_HLL2.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("HLL2Kernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL2.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .HLL2Kernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"HLL2Kernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .HLL2Kernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 + diff --git a/GPUSimulators/IPythonMagic.py b/GPUSimulators/IPythonMagic.py new file mode 100644 index 0000000..fa452df --- /dev/null +++ b/GPUSimulators/IPythonMagic.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements helpers for IPython / Jupyter and CUDA + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging +import gc + +from IPython.core import magic_arguments +from IPython.core.magic import line_magic, Magics, magics_class +import pycuda.driver as cuda + +from GPUSimulators import Common, CudaContext + + +@magics_class +class MagicCudaContext(Magics): + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--blocking', '-b', action="store_true", help='Enable blocking context') + @magic_arguments.argument( + '--no_cache', '-nc', action="store_true", help='Disable caching of kernels') + @magic_arguments.argument( + '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels') + def cuda_context_handler(self, line): + args = magic_arguments.parse_argstring(self.cuda_context_handler, line) + self.logger = logging.getLogger(__name__) + + self.logger.info("Registering %s in user workspace", args.name) + + context_flags = None + if (args.blocking): + context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC + + if args.name in self.shell.user_ns.keys(): + self.logger.debug("Context already registered! Ignoring") + return + else: + self.logger.debug("Creating context") + use_cache = False if args.no_cache else True + use_autotuning = False if args.no_autotuning else True + self.shell.user_ns[args.name] = CudaContext.CudaContext(context_flags=context_flags, use_cache=use_cache, autotuning=use_autotuning) + + # this function will be called on exceptions in any cell + def custom_exc(shell, etype, evalue, tb, tb_offset=None): + self.logger.exception("Exception caught: Resetting to CUDA context %s", args.name) + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + self.logger.info("Popping <%s>", str(context.handle)) + cuda.Context.pop() + + if args.name in self.shell.user_ns.keys(): + self.logger.info("Pushing <%s>", str(self.shell.user_ns[args.name].cuda_context.handle)) + self.shell.user_ns[args.name].cuda_context.push() + else: + self.logger.error("No CUDA context called %s found (something is wrong)", args.name) + self.logger.error("CUDA will not work now") + + self.logger.debug("==================================================================") + + # still show the error within the notebook, don't just swallow it + shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset) + + # this registers a custom exception handler for the whole current notebook + get_ipython().set_custom_exc((Exception,), custom_exc) + + + # Handle CUDA context when exiting python + import atexit + def exitfunc(): + self.logger.info("Exitfunc: Resetting CUDA context stack") + while (cuda.Context.get_current() != None): + context = cuda.Context.get_current() + self.logger.info("`-> Popping <%s>", str(context.handle)) + cuda.Context.pop() + self.logger.debug("==================================================================") + atexit.register(exitfunc) + + + + + + + + +@magics_class +class MagicLogger(Magics): + logger_initialized = False + + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--out', '-o', type=str, default='output.log', help='The filename to store the log to') + @magic_arguments.argument( + '--level', '-l', type=int, default=20, help='The level of logging to screen [0, 50]') + @magic_arguments.argument( + '--file_level', '-f', type=int, default=10, help='The level of logging to file [0, 50]') + def setup_logging(self, line): + if (self.logger_initialized): + logging.getLogger('GPUSimulators').info("Global logger already initialized!") + return; + else: + self.logger_initialized = True + + args = magic_arguments.parse_argstring(self.setup_logging, line) + import sys + + #Get root logger + logger = logging.getLogger('GPUSimulators') + logger.setLevel(min(args.level, args.file_level)) + + #Add log to screen + ch = logging.StreamHandler() + ch.setLevel(args.level) + logger.addHandler(ch) + logger.log(args.level, "Console logger using level %s", logging.getLevelName(args.level)) + + #Get the outfilename (try to evaluate if Python expression...) + try: + outfile = eval(args.out, self.shell.user_global_ns, self.shell.user_ns) + except: + outfile = args.out + + #Add log to file + logger.log(args.level, "File logger using level %s to %s", logging.getLevelName(args.file_level), outfile) + + fh = logging.FileHandler(outfile) + formatter = logging.Formatter('%(asctime)s:%(name)s:%(levelname)s: %(message)s') + fh.setFormatter(formatter) + fh.setLevel(args.file_level) + logger.addHandler(fh) + + logger.info("Python version %s", sys.version) + self.shell.user_ns[args.name] = logger + + + + + + +@magics_class +class MagicMPI(Magics): + + @line_magic + @magic_arguments.magic_arguments() + @magic_arguments.argument( + 'name', type=str, help='Name of context to create') + @magic_arguments.argument( + '--num_engines', '-n', type=int, default=4, help='Number of engines to start') + def setup_mpi(self, line): + args = magic_arguments.parse_argstring(self.setup_mpi, line) + logger = logging.getLogger('GPUSimulators') + if args.name in self.shell.user_ns.keys(): + logger.warning("MPI alreay set up, resetting") + self.shell.user_ns[args.name].shutdown() + self.shell.user_ns[args.name] = None + gc.collect() + self.shell.user_ns[args.name] = Common.IPEngine(args.num_engines) + + + + + + + + +# Register +ip = get_ipython() +ip.register_magics(MagicCudaContext) +ip.register_magics(MagicLogger) +ip.register_magics(MagicMPI) + diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py new file mode 100644 index 0000000..93ce5e9 --- /dev/null +++ b/GPUSimulators/KP07.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class KP07 (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.3, + cfl_scale=0.9, + order=2, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + order, + block_width, block_height); + self.g = np.float32(g) + self.theta = np.float32(theta) + self.order = np.int32(order) + + #Get kernels +# module = context.get_module("cuda/SWE2D_KP07.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("KP07Kernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .KP07Kernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07Kernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + + def substep(self, dt, step_number): + self.substepRK(dt, step_number) + + + def substepRK(self, dt, substep): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# Simulator.stepOrderToCodedInt(step=substep, order=self.order), +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + Simulator.stepOrderToCodedInt(step=substep, order=self.order), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .KP07Kernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + return max_dt*0.5**(self.order-1) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py new file mode 100644 index 0000000..0a5cfc7 --- /dev/null +++ b/GPUSimulators/KP07_dimsplit.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + +""" +Class that solves the SW equations using the dimentionally split KP07 scheme +""" +class KP07_dimsplit(Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + theta=1.3, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height) + self.gc_x = 2 + self.gc_y = 2 + self.g = np.float32(g) + self.theta = np.float32(theta) + + #Get kernels +# module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("KP07DimsplitKernel") +# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07_dimsplit.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .KP07DimsplitKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + self.gc_x, self.gc_y, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + self.gc_x, self.gc_y, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.theta, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_float(self.theta), + ctypes.c_int(substep) + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + + self.u0, self.u1 = self.u1, self.u0 + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .KP07DimsplitKernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py new file mode 100644 index 0000000..98e54c6 --- /dev/null +++ b/GPUSimulators/LxF.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the classical Lax-Friedrichs numerical +scheme for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + + + +""" +Class that solves the SW equations using the Lax Friedrichs scheme +""" +class LxF (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 1, + block_width, block_height); + self.g = np.float32(g) + + # Get kernels +# module = context.get_module("cuda/SWE2D_LxF.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("LxFKernel") +# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_LxF.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .LxFKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"LxFKernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 1, 1, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .LxFKernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py new file mode 100644 index 0000000..f13de52 --- /dev/null +++ b/GPUSimulators/MPISimulator.py @@ -0,0 +1,535 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements MPI simulator class + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator +import numpy as np +from mpi4py import MPI +import time + +#import pycuda.driver as cuda +#import nvtx +from hip import hip, hiprtc + + +class MPIGrid(object): + """ + Class which represents an MPI grid of nodes. Facilitates easy communication between + neighboring nodes + """ + def __init__(self, comm, ndims=2): + self.logger = logging.getLogger(__name__) + + assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" + assert comm.size >= 1, "Must have at least one node" + + self.grid = MPIGrid.getGrid(comm.size, ndims) + self.comm = comm + + self.logger.debug("Created MPI grid: {:}. Rank {:d} has coordinate {:}".format( + self.grid, self.comm.rank, self.getCoordinate())) + + def getCoordinate(self, rank=None): + if (rank is None): + rank = self.comm.rank + i = (rank % self.grid[0]) + j = (rank // self.grid[0]) + return i, j + + def getRank(self, i, j): + return j*self.grid[0] + i + + def getEast(self): + i, j = self.getCoordinate(self.comm.rank) + i = (i+1) % self.grid[0] + return self.getRank(i, j) + + def getWest(self): + i, j = self.getCoordinate(self.comm.rank) + i = (i+self.grid[0]-1) % self.grid[0] + return self.getRank(i, j) + + def getNorth(self): + i, j = self.getCoordinate(self.comm.rank) + j = (j+1) % self.grid[1] + return self.getRank(i, j) + + def getSouth(self): + i, j = self.getCoordinate(self.comm.rank) + j = (j+self.grid[1]-1) % self.grid[1] + return self.getRank(i, j) + + def getGrid(num_nodes, num_dims): + assert(isinstance(num_nodes, int)) + assert(isinstance(num_dims, int)) + + # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors + # Original code by https://stackoverflow.com/users/3928385/ishamael + # Factorizes a number into n roughly equal factors + + #Dictionary to remember already computed permutations + memo = {} + def dp(n, left): # returns tuple (cost, [factors]) + """ + Recursively searches through all factorizations + """ + + #Already tried: return existing result + if (n, left) in memo: + return memo[(n, left)] + + #Spent all factors: return number itself + if left == 1: + return (n, [n]) + + #Find new factor + i = 2 + best = n + bestTuple = [n] + while i * i < n: + #If factor found + if n % i == 0: + #Factorize remainder + rem = dp(n // i, left - 1) + + #If new permutation better, save it + if rem[0] + i < best: + best = rem[0] + i + bestTuple = [i] + rem[1] + i += 1 + + #Store calculation + memo[(n, left)] = (best, bestTuple) + return memo[(n, left)] + + + grid = dp(num_nodes, num_dims)[1] + + if (len(grid) < num_dims): + #Split problematic 4 + if (4 in grid): + grid.remove(4) + grid.append(2) + grid.append(2) + + #Pad with ones to guarantee num_dims + grid = grid + [1]*(num_dims - len(grid)) + + #Sort in descending order + grid = np.sort(grid) + grid = grid[::-1] + + # XXX: We only use vertical (north-south) partitioning for now + grid[0] = 1 + grid[1] = num_nodes + + return grid + + + def gather(self, data, root=0): + out_data = None + if (self.comm.rank == root): + out_data = np.empty([self.comm.size] + list(data.shape), dtype=data.dtype) + self.comm.Gather(data, out_data, root) + return out_data + + def getLocalRank(self): + """ + Returns the local rank on this node for this MPI process + """ + + # This function has been adapted from + # https://github.com/SheffieldML/PyDeepGP/blob/master/deepgp/util/parallel.py + # by Zhenwen Dai released under BSD 3-Clause "New" or "Revised" License: + # + # Copyright (c) 2016, Zhenwen Dai + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # * Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. + # + # * Redistributions in binary form must reproduce the above copyright notice, + # this list of conditions and the following disclaimer in the documentation + # and/or other materials provided with the distribution. + # + # * Neither the name of DGP nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + #Get this ranks unique (physical) node name + node_name = MPI.Get_processor_name() + + #Gather the list of all node names on all nodes + node_names = self.comm.allgather(node_name) + + #Loop over all node names up until our rank + #and count how many duplicates of our nodename we find + local_rank = len([0 for name in node_names[:self.comm.rank] if name==node_name]) + + return local_rank + + +class MPISimulator(Simulator.BaseSimulator): + """ + Class which handles communication between simulators on different MPI nodes + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, sim, grid): + self.profiling_data_mpi = { 'start': {}, 'end': {} } + self.profiling_data_mpi["start"]["t_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] = 0 + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] = 0 + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] = 0 + self.profiling_data_mpi["start"]["t_mpi_step"] = 0 + self.profiling_data_mpi["end"]["t_mpi_step"] = 0 + self.profiling_data_mpi["n_time_steps"] = 0 + self.logger = logging.getLogger(__name__) + + autotuner = sim.context.autotuner + sim.context.autotuner = None; + boundary_conditions = sim.getBoundaryConditions() + super().__init__(sim.context, + sim.nx, sim.ny, + sim.dx, sim.dy, + boundary_conditions, + sim.cfl_scale, + sim.num_substeps, + sim.block_size[0], sim.block_size[1]) + sim.context.autotuner = autotuner + + self.sim = sim + self.grid = grid + + #Get neighbor node ids + self.east = grid.getEast() + self.west = grid.getWest() + self.north = grid.getNorth() + self.south = grid.getSouth() + + #Get coordinate of this node + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate() + #print("gi: " + str(gi) + ", gj: " + str(gj)) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars = len(self.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(self.sim.getOutput()[0].x_halo) + gc_y = int(self.sim.getOutput()[0].y_halo) + nx = int(self.sim.nx) + ny = int(self.sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n = np.array([gc_x, ny, nx, gc_y]) + self.read_s = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e = self.read_e + np.array([gc_x, 0, 0, 0]) + self.write_w = self.read_w - np.array([gc_x, 0, 0, 0]) + self.write_n = self.read_n + np.array([0, gc_y, 0, 0]) + self.write_s = self.read_s - np.array([0, gc_y, 0, 0]) + + #Allocate data for receiving + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + ##self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) + + ##self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) + ##self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) + ##self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) + + self.in_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + num_bytes_e = self.in_e.size * self.in_e.itemsize + #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can be accessed directly by the GPU device + #hipHostMallocDefault:Memory is mapped and portable (default allocation) + #hipHostMallocPortable: memory is explicitely portable across different devices + self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + + self.in_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + num_bytes_w = self.in_w.size * self.in_w.itemsize + self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + + self.in_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + num_bytes_n = self.in_n.size * self.in_n.itemsize + self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + + self.in_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + num_bytes_s = self.in_s.size * self.in_s.itemsize + self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + + #Allocate data for sending + #self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) + #self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) + #self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) + #self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) + + self.out_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + num_bytes_e = self.out_e.size * self.out_e.itemsize + self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + + self.out_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + num_bytes_w = self.out_w.size * self.out_w.itemsize + self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + + self.out_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + num_bytes_n = self.out_n.size * self.out_n.itemsize + self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + + self.out_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + num_bytes_s = self.out_s.size * self.out_s.itemsize + self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + + + self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) + + self.full_exchange() + sim.context.synchronize() + + def substep(self, dt, step_number): + + #nvtx.mark("substep start", color="yellow") + + self.profiling_data_mpi["start"]["t_mpi_step"] += time.time() + + #nvtx.mark("substep external", color="blue") + self.sim.substep(dt, step_number, external=True, internal=False) # only "internal ghost cells" + + #nvtx.mark("substep internal", color="red") + self.sim.substep(dt, step_number, internal=True, external=False) # "internal ghost cells" excluded + + #nvtx.mark("substep full", color="blue") + #self.sim.substep(dt, step_number, external=True, internal=True) + + self.sim.swapBuffers() + + self.profiling_data_mpi["end"]["t_mpi_step"] += time.time() + + #nvtx.mark("exchange", color="blue") + self.full_exchange() + + #nvtx.mark("sync start", color="blue") + #self.sim.stream.synchronize() + #self.sim.internal_stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) + hip_check(hip.hipStreamSynchronize(self.sim.internal_stream)) + #nvtx.mark("sync end", color="blue") + + self.profiling_data_mpi["n_time_steps"] += 1 + + def getOutput(self): + return self.sim.getOutput() + + def synchronize(self): + self.sim.synchronize() + + def check(self): + return self.sim.check() + + def computeDt(self): + local_dt = np.array([np.float32(self.sim.computeDt())]); + global_dt = np.empty(1, dtype=np.float32) + self.grid.comm.Allreduce(local_dt, global_dt, op=MPI.MIN) + self.logger.debug("Local dt: {:f}, global dt: {:f}".format(local_dt[0], global_dt[0])) + return global_dt[0] + + + def getExtent(self): + """ + Function which returns the extent of node with rank + rank in the grid + """ + width = self.sim.nx*self.sim.dx + height = self.sim.ny*self.sim.dy + i, j = self.grid.getCoordinate() + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def full_exchange(self): + #### + # First transfer internal cells north-south + #### + + #Download from the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() + + if self.north is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_n[k,:,:], asynch=True, extent=self.read_n) + if self.south is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_s[k,:,:], asynch=True, extent=self.read_s) + #self.sim.stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() + + #Send/receive to north/south neighbours + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + comm_send = [] + comm_recv = [] + if self.north is not None: + comm_send += [self.grid.comm.Isend(self.out_n, dest=self.north, tag=4*self.nt + 0)] + comm_recv += [self.grid.comm.Irecv(self.in_n, source=self.north, tag=4*self.nt + 1)] + if self.south is not None: + comm_send += [self.grid.comm.Isend(self.out_s, dest=self.south, tag=4*self.nt + 1)] + comm_recv += [self.grid.comm.Irecv(self.in_s, source=self.south, tag=4*self.nt + 0)] + + #Wait for incoming transfers to complete + for comm in comm_recv: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #Upload to the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() + + if self.north is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_n[k,:,:], extent=self.write_n) + if self.south is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_s[k,:,:], extent=self.write_s) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() + + #Wait for sending to complete + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + for comm in comm_send: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + + #Download from the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_download"] += time.time() + + if self.east is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_e[k,:,:], asynch=True, extent=self.read_e) + if self.west is not None: + for k in range(self.nvars): + self.sim.u0[k].download(self.sim.stream, cpu_data=self.out_w[k,:,:], asynch=True, extent=self.read_w) + #self.sim.stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sim.stream)) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_download"] += time.time() + + #Send/receive to east/west neighbours + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + comm_send = [] + comm_recv = [] + if self.east is not None: + comm_send += [self.grid.comm.Isend(self.out_e, dest=self.east, tag=4*self.nt + 2)] + comm_recv += [self.grid.comm.Irecv(self.in_e, source=self.east, tag=4*self.nt + 3)] + if self.west is not None: + comm_send += [self.grid.comm.Isend(self.out_w, dest=self.west, tag=4*self.nt + 3)] + comm_recv += [self.grid.comm.Irecv(self.in_w, source=self.west, tag=4*self.nt + 2)] + + #Wait for incoming transfers to complete + for comm in comm_recv: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + #Upload to the GPU + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_upload"] += time.time() + + if self.east is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_e[k,:,:], extent=self.write_e) + if self.west is not None: + for k in range(self.nvars): + self.sim.u0[k].upload(self.sim.stream, self.in_w[k,:,:], extent=self.write_w) + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_upload"] += time.time() + + #Wait for sending to complete + self.profiling_data_mpi["start"]["t_mpi_halo_exchange_sendreceive"] += time.time() + + for comm in comm_send: + comm.wait() + + self.profiling_data_mpi["end"]["t_mpi_halo_exchange_sendreceive"] += time.time() diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py new file mode 100644 index 0000000..cfee8f3 --- /dev/null +++ b/GPUSimulators/SHMEMSimulator.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements SHMEM simulator group class + +Copyright (C) 2020 Norwegian Meteorological Institute + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator, CudaContext +import numpy as np + +#import pycuda.driver as cuda +from hip import hip, hiprtc + +import time + +class SHMEMSimulator(Simulator.BaseSimulator): + """ + Class which handles communication and synchronization between simulators in different + contexts (presumably on different GPUs) + """ + def __init__(self, sims, grid): + self.logger = logging.getLogger(__name__) + + assert(len(sims) > 1) + + self.sims = sims + + # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? + # See also getOutput() and check(). + # + # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of + # SHMEMSimulators that have BaseSimulator as a superclass. + # + # This would also eliminate the need for all the array bookkeeping in this class. + autotuner = sims[0].context.autotuner + sims[0].context.autotuner = None + boundary_conditions = sims[0].getBoundaryConditions() + super().__init__(sims[0].context, + sims[0].nx, sims[0].ny, + sims[0].dx, sims[0].dy, + boundary_conditions, + sims[0].cfl_scale, + sims[0].num_substeps, + sims[0].block_size[0], sims[0].block_size[1]) + sims[0].context.autotuner = autotuner + + self.sims = sims + self.grid = grid + + self.east = [None] * len(self.sims) + self.west = [None] * len(self.sims) + self.north = [None] * len(self.sims) + self.south = [None] * len(self.sims) + + self.nvars = [None] * len(self.sims) + + self.read_e = [None] * len(self.sims) + self.read_w = [None] * len(self.sims) + self.read_n = [None] * len(self.sims) + self.read_s = [None] * len(self.sims) + + self.write_e = [None] * len(self.sims) + self.write_w = [None] * len(self.sims) + self.write_n = [None] * len(self.sims) + self.write_s = [None] * len(self.sims) + + self.e = [None] * len(self.sims) + self.w = [None] * len(self.sims) + self.n = [None] * len(self.sims) + self.s = [None] * len(self.sims) + + for i, sim in enumerate(self.sims): + #Get neighbor subdomain ids + self.east[i] = grid.getEast(i) + self.west[i] = grid.getWest(i) + self.north[i] = grid.getNorth(i) + self.south[i] = grid.getSouth(i) + + #Get coordinate of this subdomain + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate(i) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars[i] = len(sim.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(sim.getOutput()[0].x_halo) + gc_y = int(sim.getOutput()[0].y_halo) + nx = int(sim.nx) + ny = int(sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) + self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) + self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) + self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) + self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) + + #Allocate host data + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) + self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) + self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) + self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) + + self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) + + + def substep(self, dt, step_number): + self.exchange() + + for i, sim in enumerate(self.sims): + sim.substep(dt, step_number) + + def getOutput(self): + # XXX: Does not return what we would expect. + # Returns first subdomain, but we want the whole domain. + return self.sims[0].getOutput() + + def synchronize(self): + for sim in self.sims: + sim.synchronize() + + def check(self): + # XXX: Does not return what we would expect. + # Checks only first subdomain, but we want to check the whole domain. + return self.sims[0].check() + + def computeDt(self): + global_dt = float("inf") + + for sim in self.sims: + sim.context.synchronize() + + for sim in self.sims: + local_dt = sim.computeDt() + if local_dt < global_dt: + global_dt = local_dt + self.logger.debug("Local dt: {:f}".format(local_dt)) + + self.logger.debug("Global dt: {:f}".format(global_dt)) + return global_dt + + def getExtent(self, index=0): + """ + Function which returns the extent of the subdomain with index + index in the grid + """ + width = self.sims[index].nx*self.sims[index].dx + height = self.sims[index].ny*self.sims[index].dy + i, j = self.grid.getCoordinate(index) + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def exchange(self): + #### + # First transfer internal cells north-south + #### + for i in range(len(self.sims)): + self.ns_download(i) + + for i in range(len(self.sims)): + self.ns_upload(i) + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + for i in range(len(self.sims)): + self.ew_download(i) + + for i in range(len(self.sims)): + self.ew_upload(i) + + def ns_download(self, i): + #Download from the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) + + def ns_upload(self, i): + #Upload to the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) + + def ew_download(self, i): + #Download from the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) + + def ew_upload(self, i): + #Upload to the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) + #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) + #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/SHMEMSimulatorGroup.py b/GPUSimulators/SHMEMSimulatorGroup.py new file mode 100644 index 0000000..c9dc30f --- /dev/null +++ b/GPUSimulators/SHMEMSimulatorGroup.py @@ -0,0 +1,413 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements SHMEM simulator group class + +Copyright (C) 2020 Norwegian Meteorological Institute + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import logging +from GPUSimulators import Simulator, CudaContext +import numpy as np + +#import pycuda.driver as cuda +from hip import hip, hiprtc + +import time + +class SHMEMGrid(object): + """ + Class which represents an SHMEM grid of GPUs. Facilitates easy communication between + neighboring subdomains in the grid. Contains one CUDA context per subdomain. + """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, ngpus=None, ndims=2): + self.logger = logging.getLogger(__name__) + + #cuda.init(flags=0) + self.logger.info("Initializing HIP") + #num_cuda_devices = cuda.Device.count() + num_cuda_devices = hip_check(hip.hipGetDeviceCount()) + + if ngpus is None: + ngpus = num_cuda_devices + + # XXX: disabled for testing on single-GPU system + #assert ngpus <= num_cuda_devices, "Trying to allocate more GPUs than are available in the system." + #assert ngpus >= 2, "Must have at least two GPUs available to run multi-GPU simulations." + + assert ndims == 2, "Unsupported number of dimensions. Must be two at the moment" + + self.ngpus = ngpus + self.ndims = ndims + + self.grid = SHMEMGrid.getGrid(self.ngpus, self.ndims) + + self.logger.debug("Created {:}-dimensional SHMEM grid, using {:} GPUs".format( + self.ndims, self.ngpus)) + + # XXX: Is this a natural place to store the contexts? Consider moving contexts out of this + # class, into notebook / calling script (shmemTesting.py) + self.cuda_contexts = [] + + for i in range(self.ngpus): + # XXX: disabled for testing on single-GPU system + #self.cuda_contexts.append(CudaContext.CudaContext(device=i, autotuning=False)) + self.cuda_contexts.append(CudaContext.CudaContext(device=0, autotuning=False)) + + def getCoordinate(self, index): + i = (index % self.grid[0]) + j = (index // self.grid[0]) + return i, j + + def getIndex(self, i, j): + return j*self.grid[0] + i + + def getEast(self, index): + i, j = self.getCoordinate(index) + i = (i+1) % self.grid[0] + return self.getIndex(i, j) + + def getWest(self, index): + i, j = self.getCoordinate(index) + i = (i+self.grid[0]-1) % self.grid[0] + return self.getIndex(i, j) + + def getNorth(self, index): + i, j = self.getCoordinate(index) + j = (j+1) % self.grid[1] + return self.getIndex(i, j) + + def getSouth(self, index): + i, j = self.getCoordinate(index) + j = (j+self.grid[1]-1) % self.grid[1] + return self.getIndex(i, j) + + def getGrid(num_gpus, num_dims): + assert(isinstance(num_gpus, int)) + assert(isinstance(num_dims, int)) + + # Adapted from https://stackoverflow.com/questions/28057307/factoring-a-number-into-roughly-equal-factors + # Original code by https://stackoverflow.com/users/3928385/ishamael + # Factorizes a number into n roughly equal factors + + #Dictionary to remember already computed permutations + memo = {} + def dp(n, left): # returns tuple (cost, [factors]) + """ + Recursively searches through all factorizations + """ + + #Already tried: return existing result + if (n, left) in memo: + return memo[(n, left)] + + #Spent all factors: return number itself + if left == 1: + return (n, [n]) + + #Find new factor + i = 2 + best = n + bestTuple = [n] + while i * i < n: + #If factor found + if n % i == 0: + #Factorize remainder + rem = dp(n // i, left - 1) + + #If new permutation better, save it + if rem[0] + i < best: + best = rem[0] + i + bestTuple = [i] + rem[1] + i += 1 + + #Store calculation + memo[(n, left)] = (best, bestTuple) + return memo[(n, left)] + + + grid = dp(num_gpus, num_dims)[1] + + if (len(grid) < num_dims): + #Split problematic 4 + if (4 in grid): + grid.remove(4) + grid.append(2) + grid.append(2) + + #Pad with ones to guarantee num_dims + grid = grid + [1]*(num_dims - len(grid)) + + #Sort in descending order + grid = np.sort(grid) + grid = grid[::-1] + + return grid + +class SHMEMSimulatorGroup(object): + """ + Class which handles communication and synchronization between simulators in different + contexts (typically on different GPUs) + """ + def __init__(self, sims, grid): + self.logger = logging.getLogger(__name__) + + assert(len(sims) > 1) + + self.sims = sims + + # XXX: This is not what was intended. Do we need extra wrapper class SHMEMSimulator? + # See also getOutput() and check(). + # + # SHMEMSimulatorGroup would then not have any superclass, but manage a collection of + # SHMEMSimulators that have BaseSimulator as a superclass. + # + # This would also eliminate the need for all the array bookkeeping in this class. + # + CONT HERE! Model shmemTesting after mpiTesting and divide existing functionality between SHMEMSimulatorGroup and SHMEMSimulator + + autotuner = sims[0].context.autotuner + sims[0].context.autotuner = None + boundary_conditions = sims[0].getBoundaryConditions() + super().__init__(sims[0].context, + sims[0].nx, sims[0].ny, + sims[0].dx, sims[0].dy, + boundary_conditions, + sims[0].cfl_scale, + sims[0].num_substeps, + sims[0].block_size[0], sims[0].block_size[1]) + sims[0].context.autotuner = autotuner + + self.sims = sims + self.grid = grid + + self.east = [None] * len(self.sims) + self.west = [None] * len(self.sims) + self.north = [None] * len(self.sims) + self.south = [None] * len(self.sims) + + self.nvars = [None] * len(self.sims) + + self.read_e = [None] * len(self.sims) + self.read_w = [None] * len(self.sims) + self.read_n = [None] * len(self.sims) + self.read_s = [None] * len(self.sims) + + self.write_e = [None] * len(self.sims) + self.write_w = [None] * len(self.sims) + self.write_n = [None] * len(self.sims) + self.write_s = [None] * len(self.sims) + + self.e = [None] * len(self.sims) + self.w = [None] * len(self.sims) + self.n = [None] * len(self.sims) + self.s = [None] * len(self.sims) + + for i, sim in enumerate(self.sims): + #Get neighbor subdomain ids + self.east[i] = grid.getEast(i) + self.west[i] = grid.getWest(i) + self.north[i] = grid.getNorth(i) + self.south[i] = grid.getSouth(i) + + #Get coordinate of this subdomain + #and handle global boundary conditions + new_boundary_conditions = Simulator.BoundaryCondition({ + 'north': Simulator.BoundaryCondition.Type.Dirichlet, + 'south': Simulator.BoundaryCondition.Type.Dirichlet, + 'east': Simulator.BoundaryCondition.Type.Dirichlet, + 'west': Simulator.BoundaryCondition.Type.Dirichlet + }) + gi, gj = grid.getCoordinate(i) + if (gi == 0 and boundary_conditions.west != Simulator.BoundaryCondition.Type.Periodic): + self.west = None + new_boundary_conditions.west = boundary_conditions.west; + if (gj == 0 and boundary_conditions.south != Simulator.BoundaryCondition.Type.Periodic): + self.south = None + new_boundary_conditions.south = boundary_conditions.south; + if (gi == grid.grid[0]-1 and boundary_conditions.east != Simulator.BoundaryCondition.Type.Periodic): + self.east = None + new_boundary_conditions.east = boundary_conditions.east; + if (gj == grid.grid[1]-1 and boundary_conditions.north != Simulator.BoundaryCondition.Type.Periodic): + self.north = None + new_boundary_conditions.north = boundary_conditions.north; + sim.setBoundaryConditions(new_boundary_conditions) + + #Get number of variables + self.nvars[i] = len(sim.getOutput().gpu_variables) + + #Shorthands for computing extents and sizes + gc_x = int(sim.getOutput()[0].x_halo) + gc_y = int(sim.getOutput()[0].y_halo) + nx = int(sim.nx) + ny = int(sim.ny) + + #Set regions for ghost cells to read from + #These have the format [x0, y0, width, height] + self.read_e[i] = np.array([ nx, 0, gc_x, ny + 2*gc_y]) + self.read_w[i] = np.array([gc_x, 0, gc_x, ny + 2*gc_y]) + self.read_n[i] = np.array([gc_x, ny, nx, gc_y]) + self.read_s[i] = np.array([gc_x, gc_y, nx, gc_y]) + + #Set regions for ghost cells to write to + self.write_e[i] = self.read_e[i] + np.array([gc_x, 0, 0, 0]) + self.write_w[i] = self.read_w[i] - np.array([gc_x, 0, 0, 0]) + self.write_n[i] = self.read_n[i] + np.array([0, gc_y, 0, 0]) + self.write_s[i] = self.read_s[i] - np.array([0, gc_y, 0, 0]) + + #Allocate host data + #Note that east and west also transfer ghost cells + #whilst north/south only transfer internal cells + #Reuses the width/height defined in the read-extets above + self.e[i] = np.empty((self.nvars[i], self.read_e[i][3], self.read_e[i][2]), dtype=np.float32) + self.w[i] = np.empty((self.nvars[i], self.read_w[i][3], self.read_w[i][2]), dtype=np.float32) + self.n[i] = np.empty((self.nvars[i], self.read_n[i][3], self.read_n[i][2]), dtype=np.float32) + self.s[i] = np.empty((self.nvars[i], self.read_s[i][3], self.read_s[i][2]), dtype=np.float32) + + self.logger.debug("Initialized {:d} subdomains".format(len(self.sims))) + + + def substep(self, dt, step_number): + self.exchange() + + for i, sim in enumerate(self.sims): + sim.substep(dt, step_number) + + def getOutput(self): + # XXX: Does not return what we would expect. + # Returns first subdomain, but we want the whole domain. + return self.sims[0].getOutput() + + def synchronize(self): + for sim in self.sims: + sim.synchronize() + + def check(self): + # XXX: Does not return what we would expect. + # Checks only first subdomain, but we want to check the whole domain. + return self.sims[0].check() + + def computeDt(self): + global_dt = float("inf") + + for sim in self.sims: + sim.context.synchronize() + + for sim in self.sims: + local_dt = sim.computeDt() + if local_dt < global_dt: + global_dt = local_dt + self.logger.debug("Local dt: {:f}".format(local_dt)) + + self.logger.debug("Global dt: {:f}".format(global_dt)) + return global_dt + + def getExtent(self, index=0): + """ + Function which returns the extent of the subdomain with index + index in the grid + """ + width = self.sims[index].nx*self.sims[index].dx + height = self.sims[index].ny*self.sims[index].dy + i, j = self.grid.getCoordinate(index) + x0 = i * width + y0 = j * height + x1 = x0 + width + y1 = y0 + height + return [x0, x1, y0, y1] + + def exchange(self): + #### + # First transfer internal cells north-south + #### + for i in range(len(self.sims)): + self.ns_download(i) + + for i in range(len(self.sims)): + self.ns_upload(i) + + #### + # Then transfer east-west including ghost cells that have been filled in by north-south transfer above + #### + for i in range(len(self.sims)): + self.ew_download(i) + + for i in range(len(self.sims)): + self.ew_upload(i) + + def ns_download(self, i): + #Download from the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the north) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.n[i][k,:,:], extent=self.read_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the south) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.s[i][k,:,:], extent=self.read_s[i]) + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) + + + def ns_upload(self, i): + #Upload to the GPU + if self.north[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.s[self.north[i]][k,:,:], extent=self.write_n[i]) + if self.south[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.n[self.south[i]][k,:,:], extent=self.write_s[i]) + + def ew_download(self, i): + #Download from the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the east) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.e[i][k,:,:], extent=self.read_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + # XXX: Unnecessary global sync (only need to sync with neighboring subdomain to the west) + self.sims[i].u0[k].download(self.sims[i].stream, cpu_data=self.w[i][k,:,:], extent=self.read_w[i]) + #self.sims[i].stream.synchronize() + hip_check(hip.hipStreamSynchronize(self.sims[i].stream)) + + def ew_upload(self, i): + #Upload to the GPU + if self.east[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.w[self.east[i]][k,:,:], extent=self.write_e[i]) + #test_east = np.ones_like(self.e[self.east[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_east, extent=self.write_e[i]) + if self.west[i] is not None: + for k in range(self.nvars[i]): + self.sims[i].u0[k].upload(self.sims[i].stream, self.e[self.west[i]][k,:,:], extent=self.write_w[i]) + #test_west = np.ones_like(self.e[self.west[i]][k,:,:]) + #self.sims[i].u0[k].upload(self.sims[i].stream, test_west, extent=self.write_w[i]) diff --git a/GPUSimulators/Simulator.py b/GPUSimulators/Simulator.py new file mode 100644 index 0000000..b804d79 --- /dev/null +++ b/GPUSimulators/Simulator.py @@ -0,0 +1,286 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the classical Lax-Friedrichs numerical +scheme for the shallow water equations + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +import numpy as np +import logging +from enum import IntEnum + +#import pycuda.compiler as cuda_compiler +#import pycuda.gpuarray +#import pycuda.driver as cuda + +from hip import hip, hiprtc + +from GPUSimulators import Common + + +class BoundaryCondition(object): + """ + Class for holding boundary conditions for global boundaries + """ + + + class Type(IntEnum): + """ + Enum that describes the different types of boundary conditions + WARNING: MUST MATCH THAT OF common.h IN CUDA + """ + Dirichlet = 0, + Neumann = 1, + Periodic = 2, + Reflective = 3 + + def __init__(self, types={ + 'north': Type.Reflective, + 'south': Type.Reflective, + 'east': Type.Reflective, + 'west': Type.Reflective + }): + """ + Constructor + """ + self.north = types['north'] + self.south = types['south'] + self.east = types['east'] + self.west = types['west'] + + if (self.north == BoundaryCondition.Type.Neumann \ + or self.south == BoundaryCondition.Type.Neumann \ + or self.east == BoundaryCondition.Type.Neumann \ + or self.west == BoundaryCondition.Type.Neumann): + raise(NotImplementedError("Neumann boundary condition not supported")) + + def __str__(self): + return '[north={:s}, south={:s}, east={:s}, west={:s}]'.format(str(self.north), str(self.south), str(self.east), str(self.west)) + + + def asCodedInt(self): + """ + Helper function which packs four boundary conditions into one integer + """ + bc = 0 + bc = bc | (self.north & 0x0000000F) << 24 + bc = bc | (self.south & 0x0000000F) << 16 + bc = bc | (self.east & 0x0000000F) << 8 + bc = bc | (self.west & 0x0000000F) << 0 + + #for t in types: + # print("{0:s}, {1:d}, {1:032b}, {1:08b}".format(t, types[t])) + #print("bc: {0:032b}".format(bc)) + + return np.int32(bc) + + def getTypes(bc): + types = {} + types['north'] = BoundaryCondition.Type((bc >> 24) & 0x0000000F) + types['south'] = BoundaryCondition.Type((bc >> 16) & 0x0000000F) + types['east'] = BoundaryCondition.Type((bc >> 8) & 0x0000000F) + types['west'] = BoundaryCondition.Type((bc >> 0) & 0x0000000F) + return types + + + +class BaseSimulator(object): + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + num_substeps, + block_width, block_height): + """ + Initialization routine + context: GPU context to use + kernel_wrapper: wrapper function of GPU kernel + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + cfl_scale: Courant number + num_substeps: Number of substeps to perform for a full step + """ + #Get logger + self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__) + + #Save input parameters + #Notice that we need to specify them in the correct dataformat for the + #GPU kernel + self.context = context + self.nx = np.int32(nx) + self.ny = np.int32(ny) + self.dx = np.float32(dx) + self.dy = np.float32(dy) + self.setBoundaryConditions(boundary_conditions) + self.cfl_scale = cfl_scale + self.num_substeps = num_substeps + + #Handle autotuning block size + if (self.context.autotuner): + peak_configuration = self.context.autotuner.get_peak_performance(self.__class__) + block_width = int(peak_configuration["block_width"]) + block_height = int(peak_configuration["block_height"]) + self.logger.debug("Used autotuning to get block size [%d x %d]", block_width, block_height) + + #Compute kernel launch parameters + self.block_size = (block_width, block_height, 1) + self.grid_size = ( + int(np.ceil(self.nx / float(self.block_size[0]))), + int(np.ceil(self.ny / float(self.block_size[1]))) + ) + + #Create a CUDA stream + #self.stream = cuda.Stream() + #self.internal_stream = cuda.Stream() + self.stream = hip_check(hip.hipStreamCreate()) + self.internal_stream = hip_check(hip.hipStreamCreate()) + + #Keep track of simulation time and number of timesteps + self.t = 0.0 + self.nt = 0 + + + def __str__(self): + return "{:s} [{:d}x{:d}]".format(self.__class__.__name__, self.nx, self.ny) + + + def simulate(self, t, dt=None): + """ + Function which simulates t_end seconds using the step function + Requires that the step() function is implemented in the subclasses + """ + + printer = Common.ProgressPrinter(t) + + t_start = self.simTime() + t_end = t_start + t + + update_dt = True + if (dt is not None): + update_dt = False + self.dt = dt + + while(self.simTime() < t_end): + # Update dt every 100 timesteps and cross your fingers it works + # for the next 100 + if (update_dt and (self.simSteps() % 100 == 0)): + self.dt = self.computeDt()*self.cfl_scale + + # Compute timestep for "this" iteration (i.e., shorten last timestep) + current_dt = np.float32(min(self.dt, t_end-self.simTime())) + + # Stop if end reached (should not happen) + if (current_dt <= 0.0): + self.logger.warning("Timestep size {:d} is less than or equal to zero!".format(self.simSteps())) + break + + # Step forward in time + self.step(current_dt) + + #Print info + print_string = printer.getPrintString(self.simTime() - t_start) + if (print_string): + self.logger.info("%s: %s", self, print_string) + try: + self.check() + except AssertionError as e: + e.args += ("Step={:d}, time={:f}".format(self.simSteps(), self.simTime()),) + raise + + + def step(self, dt): + """ + Function which performs one single timestep of size dt + """ + for i in range(self.num_substeps): + self.substep(dt, i) + + self.t += dt + self.nt += 1 + + def download(self, variables=None): + return self.getOutput().download(self.stream, variables) + + def synchronize(self): + #self.stream.synchronize() + #Synchronize the stream to ensure operations in the stream is complete + hip_check(hip.hipStreamSynchronize(self.stream)) + + def simTime(self): + return self.t + + def simSteps(self): + return self.nt + + def getExtent(self): + return [0, 0, self.nx*self.dx, self.ny*self.dy] + + def setBoundaryConditions(self, boundary_conditions): + self.logger.debug("Boundary conditions set to {:s}".format(str(boundary_conditions))) + self.boundary_conditions = boundary_conditions.asCodedInt() + + def getBoundaryConditions(self): + return BoundaryCondition(BoundaryCondition.getTypes(self.boundary_conditions)) + + def substep(self, dt, step_number): + """ + Function which performs one single substep with stepsize dt + """ + raise(NotImplementedError("Needs to be implemented in subclass")) + + def getOutput(self): + raise(NotImplementedError("Needs to be implemented in subclass")) + + def check(self): + self.logger.warning("check() is not implemented - please implement") + #raise(NotImplementedError("Needs to be implemented in subclass")) + + def computeDt(self): + raise(NotImplementedError("Needs to be implemented in subclass")) + + + + +def stepOrderToCodedInt(step, order): + """ + Helper function which packs the step and order into a single integer + """ + step_order = (step << 16) | (order & 0x0000ffff) + #print("Step: {0:032b}".format(step)) + #print("Order: {0:032b}".format(order)) + #print("Mix: {0:032b}".format(step_order)) + return np.int32(step_order) diff --git a/GPUSimulators/WAF.py b/GPUSimulators/WAF.py new file mode 100644 index 0000000..7e2763c --- /dev/null +++ b/GPUSimulators/WAF.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements the Weighted average flux (WAF) described in +E. Toro, Shock-Capturing methods for free-surface shallow flows, 2001 + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +#Import packages we need +from GPUSimulators import Simulator, Common +from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition +import numpy as np +import ctypes + +#from pycuda import gpuarray +from hip import hip,hiprtc + + + +""" +Class that solves the SW equations using the Forward-Backward linear scheme +""" +class WAF (Simulator.BaseSimulator): + + """ + Initialization routine + h0: Water depth incl ghost cells, (nx+1)*(ny+1) cells + hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+1) cells + hv0: Initial momentum along y-axis incl ghost cells, (nx+1)*(ny+1) cells + nx: Number of cells along x-axis + ny: Number of cells along y-axis + dx: Grid cell spacing along x-axis (20 000 m) + dy: Grid cell spacing along y-axis (20 000 m) + dt: Size of each timestep (90 s) + g: Gravitational accelleration (9.81 m/s^2) + """ + + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + + def __init__(self, + context, + h0, hu0, hv0, + nx, ny, + dx, dy, + g, + cfl_scale=0.9, + boundary_conditions=BoundaryCondition(), + block_width=16, block_height=16): + + # Call super constructor + super().__init__(context, + nx, ny, + dx, dy, + boundary_conditions, + cfl_scale, + 2, + block_width, block_height); + self.g = np.float32(g) + + #Get kernels +# module = context.get_module("cuda/SWE2D_WAF.cu", +# defines={ +# 'BLOCK_WIDTH': self.block_size[0], +# 'BLOCK_HEIGHT': self.block_size[1] +# }, +# compile_args={ +# 'no_extern_c': True, +# 'options': ["--use_fast_math"], +# }, +# jit_compile_args={}) +# self.kernel = module.get_function("WAFKernel") +# self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") + + kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_WAF.cu.hip')) + with open(kernel_file_path, 'r') as file: + kernel_source = file.read() + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 0, [], [])) + + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,0)) + arch = props.gcnArchName + + print(f"Compiling kernel .WAFKernel. for {arch}") + + cflags = [b"--offload-arch="+arch] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log = bytearray(log_size) + hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code = bytearray(code_size) + hip_check(hiprtc.hiprtcGetCode(prog, code)) + module = hip_check(hip.hipModuleLoadData(code)) + + kernel = hip_check(hip.hipModuleGetFunction(module, b"WAFKernel")) + + #Create data by uploading to device + self.u0 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [h0, hu0, hv0]) + self.u1 = Common.ArakawaA2D(self.stream, + nx, ny, + 2, 2, + [None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) + data_h = np.empty(self.grid_size, dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=self.grid_size) + + dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) + dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) + dt = min(dt_x, dt_y) + self.cfl_data.fill(dt, stream=self.stream) + + def substep(self, dt, step_number): + self.substepDimsplit(dt*0.5, step_number) + + def substepDimsplit(self, dt, substep): +# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, +# self.nx, self.ny, +# self.dx, self.dy, dt, +# self.g, +# substep, +# self.boundary_conditions, +# self.u0[0].data.gpudata, self.u0[0].data.strides[0], +# self.u0[1].data.gpudata, self.u0[1].data.strides[0], +# self.u0[2].data.gpudata, self.u0[2].data.strides[0], +# self.u1[0].data.gpudata, self.u1[0].data.strides[0], +# self.u1[1].data.gpudata, self.u1[1].data.strides[0], +# self.u1[2].data.gpudata, self.u1[2].data.strides[0], +# self.cfl_data.gpudata) + + #launch kernel + hip_check( + hip.hipModuleLaunchKernel( + kernel, + *self.grid_size, + *self.block_size, + sharedMemBytes=0, + stream=self.stream, + kernelParams=None, + extra=( # pass kernel's arguments + ctypes.c_int(self.nx), ctypes.c_int(self.ny), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.g), + ctypes.c_int(substep), + ctypes.c_int(self.boundary_conditions), + ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), + ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), + ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), + ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), + ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), + ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), + self.cfl_data + ) + ) + ) + + hip_check(hip.hipDeviceSynchronize()) + + self.u0, self.u1 = self.u1, self.u0 + + hip_check(hip.hipModuleUnload(module)) + + hip_check(hip.hipFree(cfl_data)) + + print("--Launching Kernel .WAFKernel. is ok") + + def getOutput(self): + return self.u0 + + def check(self): + self.u0.check() + self.u1.check() + + # computing min with hipblas: the output is an index + def min_hipblas(self, num_elements, cfl_data, stream): + num_bytes = num_elements * np.dtype(np.float32).itemsize + num_bytes_i = np.dtype(np.int32).itemsize + indx_d = hip_check(hip.hipMalloc(num_bytes_i)) + indx_h = np.zeros(1, dtype=np.int32) + x_temp = np.zeros(num_elements, dtype=np.float32) + + #print("--size.data:", cfl_data.size) + handle = hip_check(hipblas.hipblasCreate()) + + #hip_check(hipblas.hipblasGetStream(handle, stream)) + #"incx" [int] specifies the increment for the elements of x. incx must be > 0. + hip_check(hipblas.hipblasIsamin(handle, num_elements, cfl_data, 1, indx_d)) + + # destruction of handle + hip_check(hipblas.hipblasDestroy(handle)) + + # copy result (stored in indx_d) back to the host (store in indx_h) + hip_check(hip.hipMemcpyAsync(indx_h,indx_d,num_bytes_i,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + hip_check(hip.hipMemcpyAsync(x_temp,cfl_data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) + #hip_check(hip.hipMemsetAsync(cfl_data,0,num_bytes,self.stream)) + hip_check(hip.hipStreamSynchronize(stream)) + + min_value = x_temp.flatten()[indx_h[0]-1] + + # clean up + hip_check(hip.hipStreamDestroy(stream)) + hip_check(hip.hipFree(cfl_data)) + return min_value + + def computeDt(self): + #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) + return max_dt*0.5 diff --git a/GPUSimulators/__init__.py b/GPUSimulators/__init__.py new file mode 100644 index 0000000..4e04e36 --- /dev/null +++ b/GPUSimulators/__init__.py @@ -0,0 +1,5 @@ +#!/bin/env python +# -*- coding: utf-8 -*- + + +# Nothing general to do diff --git a/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc b/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da4daacc64c69c69b772807cf18375b7c8cf127b GIT binary patch literal 11058 zcmb_iOK=-UdY%`6!4RYS*uv09;PGHib$-8;*6=r=nJH)=_{+;GFt zvX)xyE$LTlk$rZlXuop)!W;JL^2*wcoA!0T>PK$FN@ta=s&tz+7N|(iv%^*;+H|FN z$=+&p?6Mo!(i46dNq@Z)VcE!b1F?v80Eb`MvQW_pgeUE&=Gl=ao1xvRaN*X$jf zMA^RBUm%_BoZXVvSvMkprQK@Npdz5$vVoAjaSI1Gytgl5`vC)8YqhbB8fL@BHvLA! zUia)y=v6w6a~3dQzrVcp_Ivl%?5isu*zaGxd-v+f+6QlTqmMeD9iq6>~AiG?PqV}c5#m&vlg=)}QXvyj#PMS_RT)eVi zeFa`oR*GtzU+q&U3OY!(OfADd9fh~>GJkmr_(~~hLM`dyw9tj|SS=Z1PGm(6xr{JH z{;^WZ3QLUPn-c{wj;|>u#4&vHVp1H(*Ai3W1ioY9j5sM~FtQ*{iCNT)iwyW{KF$G@ zThbTXp=IN@MBHO<*8Fk}IBQEUWEKM(5SLZakx&=3gcmN@H{G(|053&eXnStB1=Kd1 zoxm?M;ev9aP0tH>-oOJduD7HQobqU{{R+&jlxgGrALUKFOL)T?l1S;Pk+!RL6-Vy^ z9rv|x{vAbVkJOLUu39&Qem@f~pvIt@DAUVQZCBgXFe2Bbab{Pk>%F|l^ep6Zc+L7) zSLqdYm0f*Tg@96kitLKaV7=`(?*(C}-EPSU)Dm=>>tG>lS@;k(#DCcLyPYs1zKAwk zHaLa2x7i|D^p#38oZ)sRJG3@{naFMh9`Aem;*#`$FfqXACHohbb{6crZg3wtu>*Py zfykB=et?eDS{#pajaIb^1$`PT?rQ( z*gZ(&!Xh}L?zYdG)m)pU3YfuPqY&EZ+13Kk|OoYIK_3tvmOen%`c;cfbhY zB4}z*T4-<4j%bYI{1D59lSmY`piQb%c(a)KObfGZ@fz6%)EP~&_L;8x$VjLsyX@-%p4})@^+RspO2)H-EDnCVANV6#F z6rK@~sr_h>X`_dM%-{lxryw?F6X28MKL9*oBiX?-lsgK{_q{N>h7JQT=SRV$_M^c( z0<)mW6tjKW9KtM7BxCkSxJ(e%3WQee)-Yhe-+*EI9gc?V2m}%t>O_V>pgMv;wCtU} zvU`6{> z1u(Opf+VSQ8q7|-c3+b(p+>2qZWZK<1xZe!yaVvN8U%xa&HM^|v>64uP?MkYmbEz^ zKdU^_b~4=zZNIC5zI62de6LJ058H#ePNz$A3iWktaN~`x-ZcoS1fI)EKSz)ijqPRD z-()e+@2*$wz&L&2HMU?Cxw2e?MG)0wt5dDnE{w>ETZV1!Z?j>LSQ^Pl+fL}3j+n&; zi-lq~R)g4Rc$G+!e2a~Buv95e=~@Sdf2>Qd`BaNF(Thu0oC92qK*uR$UU10#wJ?|W?tef&@tQOmayvp z_{e=IdvHwqWIqBWY3G4VARS)?d>>AdweqQg6EcMI7utpa7}Oyq0r@YSj}5W`pQ<1) ztjPx99xue!WkNn|zxI`X#P`^(bNmZ z3)=SdVZZ_nbf`rYo>jt-6EcsA*m9gCJ^-V*;5Zu{w~?C2ImZ#Lvg628=$BX~Y>GTf z#pft_o{|?RAqJFhQbOD^M2T#&Fptt{QQweCrfL{g!OEGsnMt1|E5~+t8OXQkfW*e9T(QUPQ#6&HZF zfo;8gkLhmMxvL?J(SVdb)MeWXry!aU!1t+CJPH4XofZ~hJl}|vA*7LwOdAZRaXdaA zp?(RC4p!iKG9>e(G#klf2S@FIJ;U1-F}e5Ca9REO8XJBGJ_cf3nuCh#7^J4yC1f5%QM zyB5hvlz1M@ew<8sX9@sqZFXo3YIw{n z)nm*o@cn$3jj!khSe4(~RqEe|sirtTh+YKk9H$zBqW(k3V{>2KS=8l|y1IqDmFCBg zAB&2;m%0{c$Lx-Y@ot_>xGHdmxqrlM=AmuplD6(x*Xrie{x7F>`JuW6KsT62XhNgD zWXU^=Xh~jy@>JT&6vLq@!DV8p`Eh=oH&{N%uhZ`;SE z10%j|A8#HQ@ooFKd|<@4?c=97DWi8qO!j^cxvBbF^{dgf-V$O;a12zOa_{=Cf-j^% z?*`>K%5;3CsKFM3eFKXT7Gk2#mmxDtCnBfW_MI96kB;}S3;~Uh&O2y{=MQOtHrosu zEmy>Ihqb~;rnct}YePs+r18+$N;-s**NzK;6CujwP$6X7w`d>&L{23@Cf6-CD3}!+ zo3O(ZlbEEvyohA`MQEEO0?(EZhq5S`;;&3BZVI$-kptoi2M!2-H`6aWCa4eN z+@?fK4jt0WWqbq zyO(@6lTVO*LV1jEHc1H?j!8(3iRUVc@i^`m8Uw12$HJ|k4EHg>(G&|woa>Ki7#SGy zeI%t5>nRmDppujfN~Z#E(>bg`tNJCaS}z&Zvh$GBE%^b>RYC$kChs_WZv^{r&&DH5 z@n7+Vze0jQBuDZnnm7Dda49LuN1hp_8r)1no&4I!nOauO!cXJJFt`;x{>#i3w0XFq zrkbQLaz07nlIqjyG}bB@^Jqt2pHunCzM}G@r(f+^tjlr5ftfdmiH;a%gk=gkEa45m zL;`WeQAMF;l+$Dc!6AAbcH3oYAy*J?A>BBL%oBqI*+^ZsU)M8OywO}aDEdR@>SfX% zT0Kj~R63VA$ir|W2t+bIy@M}9@vA~JE9br5%&*+6%r?W zLL~a=gy>WdiiV4$@jj%BhNo)645P7AZdxM{Xck4?o*6d&DYB6Wm7^CWp6oB0Xe#+j zRP~o8{K!9}WZ%Nn@cz)kvm*-+aq-bhkM4JP**JqZQu+R&3x9-#OG67!jV#RLMwTQd zARWUfUPp%Sm6RUrB-{;wly}vt%CP{Yj%Zi}q8A!sAJm?$=js&w(4u_Z68aOJ33;qn z5Jqnt@dEfGU>KyS=I1<|Fx)6{%$C$6F8#Dtd;2-|bZrsQtxsj=oLz0S)^R`Fujo6o zB+_M>k|rfg$9Zz&aR+@pl0T+OmW6b1asFzffm`ZM*-PAY{oUmiR-t=Rx9^mjkmzp1 z0ZHeRrIFo#sE;w{hlE=?8bzH&jAaJ0XdhJ#pc%n4dEw+6GoDGf5>+A2L@A<`h-gqQ zjW(f7_(d?4Js2JFM+hI(X5d>uXv&ylMZD$Zpx!DQIWC{$f zF89JSBtPJ~rq`CqjV_xggTY)%FTm-xY|v25C9KD)f?|rSFXYcD$LJ@PhsdFLN1WO8 z1x%!DjjlSAxFz4q#M;C2C2i|Gz87L`>%u;Cv(mr8FnWY^(veDL5=fuutBn1q2SX=?5wf5d^#xEKp_&4_hskD;x6)B%Euy2q)#-KR!#DVF-~@LuhObF8AbX__=a*vK zA9JMpW356?HI1@_LAqdr9H23t?bqL7Hn`4*&NiOT*2QdCw>L4+F68bXqFvIxW%ui_ zDu@a%2k{6hIIJadzw5{9ll9~H=J6s*k92Ap8VHvf`6r0+Q`j5im-IV<@AODN3;o8@ ze$R1k0Ui$QmRsnI z3;Xea7Cz%IG`Vu*xtv0;e$n*BoGAcy{$bjII& zhZ&zeV#enNGk$~_|G)Sk*{dPfM`-m7R+t;`!NifYYLE0gHbSfYT^sT@BXd8?_~`Qi zjX9hTo*(H)FdWSX#>PLP_y2|ulKp4o&xTHBRNC$Ffqn2~u<8pb-DWoC8Df$v8TmKg zVZP@No$q^t`5J>g{a^SXv+?hPc@FRa+trh)UH$ve5No)NGRyEU>7u2S#Xoo8ZN=F{ zQ6>iUyXYogq69h#b|5Zk;Pc0N}+u0!mtx)F|Wc|7857sr#3%MIGdO-gQ2a*7giZP>X;a8E|Jq&>;c zDPb3CJoTK*9vt}8duAu$A$4H~>22ix1ur{DxGT{M74sR~mC$3F6O!7ClgYAVqsQYT z{}M_fEg3jp_(vxh#&H%AqgPSTQDEXKjkmWbXt5w9E}DFvsTQM!QWO%hgX7W#!bN>yAR3TRY literal 0 HcmV?d00001 diff --git a/GPUSimulators/__pycache__/Simulator.cpython-39.pyc b/GPUSimulators/__pycache__/Simulator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc16706a34d26a4641268277e3f0d257f351d864 GIT binary patch literal 8433 zcmbVR+ix3JdY>DI_|HP)7M^Ga1=ngC*SX>AHAB74nu)jRTkTver>A3f zJ6|hg^NKbI- zHCho1x^01)>pM}Sg$1k`O0^rMuY3^%3QW`L_HgJxTX1VRFrN8JtIYq0xXLj;jvE{H%qQA=>P z(Q|0+3Z1@|oz7N5?5YsC%7lk;$zBN=0WtR3@PKT$t1OhxoEsCsl7$_@z5*zF%vTi2 z*3BPB_y8}?yg&in3Ohifh24PIw&(k7L$GcnnqB{@0}8OGtLyh3K3Qi=)o1MK(&NWV z)%9n8g_&R}M#U?U?n7J!>%bH#-5~A(X6M13$IJIHf9b2$`>X5E05)4$U9a9*TVpE^ zAG0O)XzB6#>hhEOOOM&3CyyUJT)Q(5j)5$|-}w{dBuQw8QZO!J*YhKw`V7pA0FuvH z%Y7xl#D?%*0U(z(AQnRszI{{2^}}FOA%^2gYEwZ0M5|2}gz;4t0kWH|IPTn7SlHg) zp5F|*^P${a0H=w|(ZZMW&KoG5R;}nsel>{iK!#DZm&0~D3>q{v{>!0p3s3Yj6hf=% z&{k+^&4gwd+=OPC!h)vdL~hF-f4KE)tEnq()NA=v3u}c;|B8Hqm!O9ZpM)MxC8xep zq9uFFVZc2Smi`@w;kQhhr?jdS`rHHZ8`(^dESs8+Z~EZ|ggP7cM5NXq!$%+D!;av? zjhe-Q2e)cA&jDf^kmdVlk3lxS-s_0|Kd9{qL?U1+om@l>sH8}Z1~joLBy0$au|R3y z39yg0f!tPCt9Ng(2T#`4*@LC^<$G-X-qJdIxB^s(H}fsFT4l>mZZC}%Rdh)RRrEL6 z3-(NVW1^^53Q3_}58Sq>*Au5+Pc0kjrFtFa$j?Ud^*Rq5=q}#&$fo%sPV!aJZM#8` z6du94!K5@2=doz|q7g$}ztdzH2dLurKVRrHqXoYUTO>Ny7a)&YZl~j`3Z~b8`1y|( z@%!TQg%&nnSn&cc7C#Gp*v-3-p43}jX94duuib@R2xYV|s?K+MiAAKoio+suCk5TI zES>)344M@y$pxW|TS+d0nzs^5xKW%~+aih^M2F#z>hvt*iF_0To$D90fmWxQp=vs6 zrm7jJS*m6Zv@I>RwjkIqjOZJ*^g}J^aIpEL4J0{OMI||jT5`G?#;Zf0CiopG zLs>BrD-wP)$tn3fV5&StOCUVNta|-(w4(DUw4$S{KVwQSeQy}0KN@D~e+;XC@wko* z7dJ>Qhd-+4@hC>jp~Ce&E!II;;}+Ic;anRG}dt&FUT&!`JP~O;8^33g#ki`AQqcqOd5$tZ>Fgr+Poo6e^WqF zu}>mlVg#KehfO~Fv|=b~CdNkNC`BDNS`NA3@HSO!6_rqEQ01BQ&+cD3PTqqh$0%%J zwW9DhsCb}ftnGouT>ooD<$a~gH?^$S8tcuCdhM~^{HV7u)?2&@>i}9iRmpIUe4h%X zs~=EJDeQ+-Q(8-8Q>qIx5PYeHG9-duOsI6-=e+bG}V1rV_4{uXjHGA^s|0G{i6eHHaH z39IV`Vl<2pEEYTQ4F(@h9*PZ5S#+>{qv_Wp z+>dc}U4}Elt&8r=z03v9c<_MYOGfj|}X{#TgfScUG(@XMAKXltnHC@GHUmXyW`loT^8gsja1l6ptDFHk7*xIsfCCY*7S<6@(`nK&sG z2*Jd{Rnt#$>ik-96LM0|`n%k%*=va3-F9LlrlI#V#RW2O!MwCXj{$#NDWwt9bwcb@ zDn6s)M^sRln4BI48+Fhiy~suKFKPG-Dn6#-6BM->1vn-(y-kP(MJ{9BNjWBRU?HA} zE&$s3^Eq2DWq*!t>$X|Y%c@;AOU5Pr(jRRb(Wzfcdw^qQq+D3@GNNSQJK8 z--5B+qR`*kH(?MFd6p^UWMBW{2H)MG;&~+}-@yvGK!tL}2ef#X`VRc8!i2o=Z}1GG z$umdcQ;}q~K|Xc(Bfn0Mg74$ZSyA}63)uFIeQnnoz)_vkc5?_a?E!+vt}Yk3Np+a> zY}O$6Li?32KN{F_1M94Pgq&xQBYR<}c8(mvmOV>u!Hn~Jg}A7&&!uOa*IvT`?&b%$ z4ZqOkKVi(-o8b99a(38jAMohd|K&$$PY&|R`I&oD>FR-X9y4|egMwUCE2lYJ;-Elo ztxr3Y_cdL6s=ZvawD{Do^xlWMwp+y5{Ge!QTgcT7&q?>&H=6p7ag(dh{1U64A=uk3 z{$26qV)|ax7O|7c>pA^jvyp9BIUE>A=~a{N=yefr`fnIt#`pP6}wyYO*(ByMt8X?Ksue%LLvJQ6y&D3*;iz@<^MJ^;EKt zt0NZHreK5A*03LB9W5_thDi>&(8h~oYAK3@B-K{;&BSu$CR{^mLP)OVJruQkCxxw+ zk6~?G8P{@(HML^5gKVRy^SI_Tx{^{UR4=LdWSHngT^!Lig%xEJ#BZ^CG><~F9iyNZ z3`<9RtJ}tuIjfh=i!dN(b<1?j8T6KQ1IDDRvt#CCC^3g}3PjzJ`M8Njtc=HN{kHbJ zybA@~LmmOqqoHdr5z|4O%|R->P)xKep0+F{ux9_>NYIYRYi8S#KEiE{WE9!yk+X&O zRr12)(Q@U#s+C-lgR_M>C+bxoSLqwq)h=qqvi6}(_GHCniat5oa?<42_onEb&%85Sg#*nzAz{BkTPLazc zEXXdAwc;yRdhQ0~|0rb7FzerFe>zM%#;oCPhc1nDIiu+ct|?StjQ|x{AFi!w@+rD- zG69fGM7^NVl3@U+DNi!yHO!&}>JWenM~HH;{1^-bhl0UIgA1U?`X6J* z9q}ERI{!IG5&KWXY!w6lId;?$H##&H$Icq*!6hulhZ&Xc0D3yPmDHWiYMW^qCrwvc zY4D)p1{Fb2@Ng1LN>c9Z#8Ca;jX`$qFwJ3v(mBVZkqE6Q{sUA}uEWn8F_iH&e3$Z9 zI(ahqzZ}T|`TO^W0og&)ErOp=Hya8CkP0D#C`s+pK!6ZWui1A{%Y;}`{iyi<7=8bR zCn7!3V1N3T594xTJa~-cBP}aqmpDsxY{6H*M#>_UXEV)1sC&PH)nSLDs@5K3s0v8jDN6J=G;BkP3Y$$zlGy4xlrt1G-=`y>Oe(@lCbJX1Z{&H0a zxNo>cd=on2f3aHm-YPPE=K#qeqc=g@)ziz!Q7%2fv>{dB z8CCfl`$O2SDJh7wM4J(T8a${8b2K|DdJ<8c~Y zWBAO8Soxq6va4kK{@Vb3hyi?)9RYg!u=uLOjsf{r!pLvvoVQZtrx-;*I6CYIRYpNe z1zrXJGe`4c2RX1cI)}WPBnKkNh)Xi9l8m1GnhH`)`Ex3W{fZ-!u$2VGn1m3aRSr+i zq1MI|QGj1C3O^}W&eekBn1$h=g_@1WQgaWPCt}=)c?2kBFuzn^ByLfb*iiXPsI=0W zs?wT2mFpdbr1U8m;UazgMaUO2cNjHL060wb$`5d$_x<8q z={0ZzohM#;K5oy2VXZbv1Gq@01EZOnoi2U9?bYmbX)>8@fp7Boten{CTi&9RCJ%jG zQbBQIhZ|e?HkH~(;-La|48;+LlfupPv)Px#sfezL))_>SddVoIrCq*T{(J4}{{j@a BH8B7H literal 0 HcmV?d00001 diff --git a/GPUSimulators/__pycache__/__init__.cpython-39.pyc b/GPUSimulators/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a966589566cf0036cb97a40facf080e9dc3106f1 GIT binary patch literal 182 zcmYe~<>g`kg7xe&sV+eJF^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5w0=Qav3^cz zaY<2XfuVjuQGQlpK|v0fk(yi*Z(?R@00fq1`WczY8TxLSd6^}tVfi_wxvA~}q461+ s1^PfbI5W32C$S{Is8~Nf9;75bUaz3?7Kcr4eoARhsvXGE&p^xo01!1T>Hq)$ literal 0 HcmV?d00001 diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu new file mode 100644 index 0000000..238718c --- /dev/null +++ b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu @@ -0,0 +1,250 @@ + /* +This kernel implements the Central Upwind flux function to +solve the Euler equations + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "EulerCommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float gamma_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + + + // Write to main memory for all internal cells + writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); + } +} + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip new file mode 100644 index 0000000..67b701b --- /dev/null +++ b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip @@ -0,0 +1,251 @@ +#include "hip/hip_runtime.h" + /* +This kernel implements the Central Upwind flux function to +solve the Euler equations + +Copyright (C) 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "EulerCommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float gamma_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( rho0_ptr_, rho0_pitch_, Q[0], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_u0_ptr_, rho_u0_pitch_, Q[1], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock(rho_v0_ptr_, rho_v0_pitch_, Q[2], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + readBlock( E0_ptr_, E0_pitch_, Q[3], nx_, ny_, boundary_conditions_, x0, y0, x1, y1); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, gamma_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, gamma_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Gravity source term + if (g_ > 0.0f) { + const int i = threadIdx.x + gc_x; + const int j = threadIdx.y + gc_y; + const float rho_v = Q[2][j][i]; + Q[2][j][i] -= g_*Q[0][j][i]*dt_; + Q[3][j][i] -= g_*rho_v*dt_; + __syncthreads(); + } + } + + + // Write to main memory for all internal cells + writeBlock( rho1_ptr_, rho1_pitch_, Q[0], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_u1_ptr_, rho_u1_pitch_, Q[1], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock(rho_v1_ptr_, rho_v1_pitch_, Q[2], nx_, ny_, 0, 1, x0, y0, x1, y1); + writeBlock( E1_ptr_, E1_pitch_, Q[3], nx_, ny_, 0, 1, x0, y0, x1, y1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, gamma_, cfl_); + } +} + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/EulerCommon.h b/GPUSimulators/cuda/EulerCommon.h new file mode 100644 index 0000000..cb22a53 --- /dev/null +++ b/GPUSimulators/cuda/EulerCommon.h @@ -0,0 +1,187 @@ +/* +These CUDA functions implement different types of numerical flux +functions for the shallow water equations + +Copyright (C) 2016, 2017, 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once +#include "limiters.h" + + + +template +__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const float dx_, const float dy_, const float gamma_, + float* output_) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + tx; + const int tj = blockDim.y*blockIdx.y + ty; + + //Only internal cells + if (ti < nx_+gc_x && tj < ny_+gc_y) { + const float rho = Q[0][ty][tx]; + const float u = Q[1][ty][tx] / rho; + const float v = Q[2][ty][tx] / rho; + + const float max_u = dx_ / (fabsf(u) + sqrtf(gamma_*rho)); + const float max_v = dy_ / (fabsf(v) + sqrtf(gamma_*rho)); + + shmem[ty][tx] = fminf(max_u, max_v); + } + __syncthreads(); + + //One row of threads loop over all rows + if (ti < nx_+gc_x && tj < ny_+gc_y) { + if (ty == gc_y) { + float min_val = shmem[ty][tx]; + const int max_y = min(h, ny_+gc_y - tj); + for (int j=gc_y; j h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + + //Upwind selection + if (S_l >= 0.0f) { + return F_func(Q_l, P_l); + } + else if (S_r <= 0.0f) { + return F_func(Q_r, P_r); + } + //Or estimate flux in the star region + else { + const float4 F_l = F_func(Q_l, P_l); + const float4 F_r = F_func(Q_r, P_r); + const float4 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); + return flux; + } +} + + + + + + + +/** + * Central upwind flux function + */ +__device__ float4 CentralUpwindFlux(const float4 Qm, const float4 Qp, const float gamma) { + + const float Pp = pressure(Qp, gamma); + const float4 Fp = F_func(Qp, Pp); + const float up = Qp.y / Qp.x; // rho*u / rho + const float cp = sqrt(gamma*Pp/Qp.x); // sqrt(gamma*P/rho) + + const float Pm = pressure(Qm, gamma); + const float4 Fm = F_func(Qm, Pm); + const float um = Qm.y / Qm.x; // rho*u / rho + const float cm = sqrt(gamma*Pm/Qm.x); // sqrt(gamma*P/rho) + + const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed + const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed + + return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); +} \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu b/GPUSimulators/cuda/SWE2D_FORCE.cu new file mode 100644 index 0000000..dac46be --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_FORCE.cu @@ -0,0 +1,143 @@ +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_, const float dx_, const float dt_) { + //Compute fluxes along the x axis + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + //Compute flux along x, and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute flux along y, and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_FORCE.cu.hip b/GPUSimulators/cuda/SWE2D_FORCE.cu.hip new file mode 100644 index 0000000..aa4e968 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_FORCE.cu.hip @@ -0,0 +1,144 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_, const float dx_, const float dt_) { + //Compute fluxes along the x axis + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + //Compute flux along x, and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute flux along y, and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu b/GPUSimulators/cuda/SWE2D_HLL.cu new file mode 100644 index 0000000..3ed6b35 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL.cu @@ -0,0 +1,161 @@ +/* +This GPU kernel implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute F flux + computeFluxF(Q, F, g_); + __syncthreads(); + + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute G flux + computeFluxG(Q, F, g_); + __syncthreads(); + + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL.cu.hip b/GPUSimulators/cuda/SWE2D_HLL.cu.hip new file mode 100644 index 0000000..c2f449d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL.cu.hip @@ -0,0 +1,162 @@ +#include "hip/hip_runtime.h" +/* +This GPU kernel implements the HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + const float g_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute F flux + computeFluxF(Q, F, g_); + __syncthreads(); + + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute G flux + computeFluxG(Q, F, g_); + __syncthreads(); + + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu b/GPUSimulators/cuda/SWE2D_HLL2.cu new file mode 100644 index 0000000..94f92b5 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL2.cu @@ -0,0 +1,216 @@ +/* +This OpenCL kernel implements the second order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_HLL2.cu.hip b/GPUSimulators/cuda/SWE2D_HLL2.cu.hip new file mode 100644 index 0000000..c0bc9d1 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_HLL2.cu.hip @@ -0,0 +1,217 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the second order HLL flux + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + + + + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu b/GPUSimulators/cuda/SWE2D_KP07.cu new file mode 100644 index 0000000..6fa6154 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07.cu @@ -0,0 +1,233 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], + const float g_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + int j=ty; + const int l = j + 2; //Skip ghost cells + for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + + //Reconstruct slopes along x and axis + minmodSlopeX(Q, Qx, theta_); + minmodSlopeY(Q, Qy, theta_); + __syncthreads(); + + + //Compute fluxes along the x and y axis + computeFluxF(Q, Qx, F, g_); + computeFluxG(Q, Qy, G, g_); + __syncthreads(); + + + //Sum fluxes and advance in time for all internal cells + if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { + const int i = tx + 2; //Skip local ghost cells, i.e., +2 + const int j = ty + 2; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + + float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); + float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); + float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); + + if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { + //Write to main memory + h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); + hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); + hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); + } + else { + h_row[ti] = Q[0][j][i]; + hu_row[ti] = Q[1][j][i]; + hv_row[ti] = Q[2][j][i]; + } + } + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} +} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07.cu.hip b/GPUSimulators/cuda/SWE2D_KP07.cu.hip new file mode 100644 index 0000000..fd9ef0d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07.cu.hip @@ -0,0 +1,234 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float Qx[3][BLOCK_HEIGHT+2][BLOCK_WIDTH+2], + float F[3][BLOCK_HEIGHT+1][BLOCK_WIDTH+1], + const float g_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + int j=ty; + const int l = j + 2; //Skip ghost cells + for (int i=tx; i( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + + //Reconstruct slopes along x and axis + minmodSlopeX(Q, Qx, theta_); + minmodSlopeY(Q, Qy, theta_); + __syncthreads(); + + + //Compute fluxes along the x and y axis + computeFluxF(Q, Qx, F, g_); + computeFluxG(Q, Qy, G, g_); + __syncthreads(); + + + //Sum fluxes and advance in time for all internal cells + if (ti > 1 && ti < nx_+2 && tj > 1 && tj < ny_+2) { + const int i = tx + 2; //Skip local ghost cells, i.e., +2 + const int j = ty + 2; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + + float* const h_row = (float*) ((char*) h1_ptr_ + h1_pitch_*tj); + float* const hu_row = (float*) ((char*) hu1_ptr_ + hu1_pitch_*tj); + float* const hv_row = (float*) ((char*) hv1_ptr_ + hv1_pitch_*tj); + + if (getOrder(step_order_) == 2 && getStep(step_order_) == 1) { + //Write to main memory + h_row[ti] = 0.5f*(h_row[ti] + Q[0][j][i]); + hu_row[ti] = 0.5f*(hu_row[ti] + Q[1][j][i]); + hv_row[ti] = 0.5f*(hv_row[ti] + Q[2][j][i]); + } + else { + h_row[ti] = Q[0][j][i]; + hu_row[ti] = Q[1][j][i]; + hv_row[ti] = Q[2][j][i]; + } + } + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} +} //extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu new file mode 100644 index 0000000..ac256e3 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu @@ -0,0 +1,216 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +template +__device__ +void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], + float Qx[3][h+2*gc_y][w+2*gc_x], + float F[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j +__device__ +void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], + float Qy[3][h+2*gc_y][w+2*gc_x], + float G[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dy_, const float dt_) { + for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + if (step_ == 0) { + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + else { + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + + + + + + + + + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip new file mode 100644 index 0000000..f366b0a --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_KP07_dimsplit.cu.hip @@ -0,0 +1,217 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" +#include "limiters.h" + + +template +__device__ +void computeFluxF(float Q[3][h+2*gc_y][w+2*gc_x], + float Qx[3][h+2*gc_y][w+2*gc_x], + float F[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j +__device__ +void computeFluxG(float Q[3][h+2*gc_y][w+2*gc_x], + float Qy[3][h+2*gc_y][w+2*gc_x], + float G[3][h+2*gc_y][w+2*gc_x], + const float g_, const float dy_, const float dt_) { + for (int j=threadIdx.y+1; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + if (step_ == 0) { + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + else { + //Along Y + minmodSlopeY(Q, Qx, theta_); + __syncthreads(); + computeFluxG(Q, Qx, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Along X + minmodSlopeX(Q, Qx, theta_); + __syncthreads(); + computeFluxF(Q, Qx, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, F[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + + + + + + + + + + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu b/GPUSimulators/cuda/SWE2D_LxF.cu new file mode 100644 index 0000000..1f197fd --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_LxF.cu @@ -0,0 +1,168 @@ +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +template +__device__ +void computeFluxF(float Q[3][block_height+2][block_width+2], + float F[3][block_height][block_width+1], + const float g_, const float dx_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + const int j=ty; + const int l = j + 1; //Skip ghost cells + for (int i=tx; i +__device__ +void computeFluxG(float Q[3][block_height+2][block_width+2], + float G[3][block_height+1][block_width], + const float g_, const float dy_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute fluxes along the x and y axis + computeFluxF(Q, F, g_, dx_, dt_); + computeFluxG(Q, G, g_, dy_, dt_); + __syncthreads(); + + //Evolve for all cells + const int tx = threadIdx.x; + const int ty = threadIdx.y; + const int i = tx + 1; //Skip local ghost cells, i.e., +1 + const int j = ty + 1; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" + diff --git a/GPUSimulators/cuda/SWE2D_LxF.cu.hip b/GPUSimulators/cuda/SWE2D_LxF.cu.hip new file mode 100644 index 0000000..588d691 --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_LxF.cu.hip @@ -0,0 +1,169 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the classical Lax-Friedrichs scheme +for the shallow water equations, with edge fluxes. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + +#include "common.h" +#include "SWECommon.h" + + +/** + * Computes the flux along the x axis for all faces + */ +template +__device__ +void computeFluxF(float Q[3][block_height+2][block_width+2], + float F[3][block_height][block_width+1], + const float g_, const float dx_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + { + const int j=ty; + const int l = j + 1; //Skip ghost cells + for (int i=tx; i +__device__ +void computeFluxG(float Q[3][block_height+2][block_width+2], + float G[3][block_height+1][block_width], + const float g_, const float dy_, const float dt_) { + //Index of thread within block + const int tx = threadIdx.x; + const int ty = threadIdx.y; + + for (int j=ty; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + + //Compute fluxes along the x and y axis + computeFluxF(Q, F, g_, dx_, dt_); + computeFluxG(Q, G, g_, dy_, dt_); + __syncthreads(); + + //Evolve for all cells + const int tx = threadIdx.x; + const int ty = threadIdx.y; + const int i = tx + 1; //Skip local ghost cells, i.e., +1 + const int j = ty + 1; + + Q[0][j][i] += (F[0][ty][tx] - F[0][ty ][tx+1]) * dt_ / dx_ + + (G[0][ty][tx] - G[0][ty+1][tx ]) * dt_ / dy_; + Q[1][j][i] += (F[1][ty][tx] - F[1][ty ][tx+1]) * dt_ / dx_ + + (G[1][ty][tx] - G[1][ty+1][tx ]) * dt_ / dy_; + Q[2][j][i] += (F[2][ty][tx] - F[2][ty ][tx+1]) * dt_ / dx_ + + (G[2][ty][tx] - G[2][ty+1][tx ]) * dt_ / dy_; + __syncthreads(); + + //Write to main memory + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); + + //Compute the CFL for this block + if (cfl_ != NULL) { + writeCfl(Q, Q[0], nx_, ny_, dx_, dy_, g_, cfl_); + } +} + +} // extern "C" + diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu b/GPUSimulators/cuda/SWE2D_WAF.cu new file mode 100644 index 0000000..2c38cdf --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_WAF.cu @@ -0,0 +1,178 @@ +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWE2D_WAF.cu.hip b/GPUSimulators/cuda/SWE2D_WAF.cu.hip new file mode 100644 index 0000000..ddfad9d --- /dev/null +++ b/GPUSimulators/cuda/SWE2D_WAF.cu.hip @@ -0,0 +1,179 @@ +#include "hip/hip_runtime.h" +/* +This OpenCL kernel implements the Kurganov-Petrova numerical scheme +for the shallow water equations, described in +A. Kurganov & Guergana Petrova +A Second-Order Well-Balanced Positivity Preserving Central-Upwind +Scheme for the Saint-Venant System Communications in Mathematical +Sciences, 5 (2007), 133-160. + +Copyright (C) 2016 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + + + +#include "common.h" +#include "SWECommon.h" + + + +/** + * Computes the flux along the x axis for all faces + */ +__device__ +void computeFluxF(float Q[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + float F[3][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], + const float g_, const float dx_, const float dt_) { + for (int j=threadIdx.y; j( h0_ptr_, h0_pitch_, Q[0], nx_, ny_, boundary_conditions_); + readBlock(hu0_ptr_, hu0_pitch_, Q[1], nx_, ny_, boundary_conditions_); + readBlock(hv0_ptr_, hv0_pitch_, Q[2], nx_, ny_, boundary_conditions_); + __syncthreads(); + + + + //Step 0 => evolve x first, then y + if (step_ == 0) { + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + } + //Step 1 => evolve y first, then x + else { + //Compute fluxes along the y axis and evolve + computeFluxG(Q, F, g_, dy_, dt_); + __syncthreads(); + evolveG(Q, F, dy_, dt_); + __syncthreads(); + + //Compute fluxes along the x axis and evolve + computeFluxF(Q, F, g_, dx_, dt_); + __syncthreads(); + evolveF(Q, F, dx_, dt_); + __syncthreads(); + } + + + + // Write to main memory for all internal cells + writeBlock( h1_ptr_, h1_pitch_, Q[0], nx_, ny_, 0, 1); + writeBlock(hu1_ptr_, hu1_pitch_, Q[1], nx_, ny_, 0, 1); + writeBlock(hv1_ptr_, hv1_pitch_, Q[2], nx_, ny_, 0, 1); +} + +} // extern "C" \ No newline at end of file diff --git a/GPUSimulators/cuda/SWECommon.h b/GPUSimulators/cuda/SWECommon.h new file mode 100644 index 0000000..52f8b31 --- /dev/null +++ b/GPUSimulators/cuda/SWECommon.h @@ -0,0 +1,533 @@ +/* +These CUDA functions implement different types of numerical flux +functions for the shallow water equations + +Copyright (C) 2016, 2017, 2018 SINTEF Digital + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once +#include "limiters.h" + + + + + + +__device__ float3 F_func(const float3 Q, const float g) { + float3 F; + + F.x = Q.y; //hu + F.y = Q.y*Q.y / Q.x + 0.5f*g*Q.x*Q.x; //hu*hu/h + 0.5f*g*h*h; + F.z = Q.y*Q.z / Q.x; //hu*hv/h; + + return F; +} + + + + + + + + + + + + + +/** + * Superbee flux limiter for WAF. + * Related to superbee limiter so that WAF_superbee(r, c) = 1 - (1-|c|)*superbee(r) + * @param r_ the ratio of upwind change (see Toro 2001, p. 203/204) + * @param c_ the courant number for wave k, dt*S_k/dx + */ +__device__ float WAF_superbee(float r_, float c_) { + // r <= 0.0 + if (r_ <= 0.0f) { + return 1.0f; + } + // 0.0 <= r <= 1/2 + else if (r_ <= 0.5f) { + return 1.0f - 2.0f*(1.0f - fabsf(c_))*r_; + } + // 1/2 <= r <= 1 + else if (r_ <= 1.0f) { + return fabs(c_); + } + // 1 <= r <= 2 + else if (r_ <= 2.0f) { + return 1.0f - (1.0f - fabsf(c_))*r_; + } + // r >= 2 + else { + return 2.0f*fabsf(c_) - 1.0f; + } +} + + + + +__device__ float WAF_albada(float r_, float c_) { + if (r_ <= 0.0f) { + return 1.0f; + } + else { + return 1.0f - (1.0f - fabsf(c_)) * r_ * (1.0f + r_) / (1.0f + r_*r_); + } +} + +__device__ float WAF_minbee(float r_, float c_) { + r_ = fmaxf(-1.0f, fminf(2.0f, r_)); + if (r_ <= 0.0f) { + return 1.0f; + } + if (r_ >= 0.0f && r_ <= 1.0f) { + return 1.0f - (1.0f - fabsf(c_)) * r_; + } + else { + return fabsf(c_); + } +} + +__device__ float WAF_minmod(float r_, float c_) { + return 1.0f - (1.0f - fabsf(c_)) * fmaxf(0.0f, fminf(1.0f, r_)); +} + +__device__ float limiterToWAFLimiter(float r_, float c_) { + return 1.0f - (1.0f - fabsf(c_))*r_; +} + +// Compute h in the "star region", h^dagger +__device__ __inline__ float computeHStar(float h_l, float h_r, float u_l, float u_r, float c_l, float c_r, float g_) { + + //This estimate for the h* gives rise to spurious oscillations. + //return 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float h_tmp = 0.5f * (c_l + c_r) + 0.25f * (u_l - u_r); + return h_tmp*h_tmp / g_; +} + +/** + * Weighted average flux (Toro 2001, p 200) for interface {i+1/2} + * @param r_ The flux limiter parameter (see Toro 2001, p. 203) + * @param Q_l2 Q_{i-1} + * @param Q_l1 Q_{i} + * @param Q_r1 Q_{i+1} + * @param Q_r2 Q_{i+2} + */ +__device__ float3 WAF_1D_flux(const float3 Q_l2, const float3 Q_l1, const float3 Q_r1, const float3 Q_r2, const float g_, const float dx_, const float dt_) { + const float h_l = Q_l1.x; + const float h_r = Q_r1.x; + + const float h_l2 = Q_l2.x; + const float h_r2 = Q_r2.x; + + // Calculate velocities + const float u_l = Q_l1.y / h_l; + const float u_r = Q_r1.y / h_r; + + const float u_l2 = Q_l2.y / h_l2; + const float u_r2 = Q_r2.y / h_r2; + + const float v_l = Q_l1.z / h_l; + const float v_r = Q_r1.z / h_r; + + const float v_l2 = Q_l2.z / h_l2; + const float v_r2 = Q_r2.z / h_r2; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + const float c_l2 = sqrt(g_*h_l2); + const float c_r2 = sqrt(g_*h_r2); + + // Compute h in the "star region", h^dagger + const float h_dag_l = computeHStar(h_l2, h_l, u_l2, u_l, c_l2, c_l, g_); + const float h_dag = computeHStar( h_l, h_r, u_l, u_r, c_l, c_r, g_); + const float h_dag_r = computeHStar( h_r, h_r2, u_r, u_r2, c_r, c_r2, g_); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag ) ) / h_l; + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag ) ) / h_r; + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); + + const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1.0, S_star, v_l); + const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1.0, S_star, v_r); + + // Estimate the fluxes in the four regions + const float3 F_1 = F_func(Q_l1, g_); + const float3 F_4 = F_func(Q_r1, g_); + + const float3 F_2 = F_1 + S_l*(Q_star_l - Q_l1); + const float3 F_3 = F_4 + S_r*(Q_star_r - Q_r1); + //const float3 F_2 = F_func(Q_star_l, g_); + //const float3 F_3 = F_func(Q_star_r, g_); + + // Compute the courant numbers for the waves + const float c_1 = S_l * dt_ / dx_; + const float c_2 = S_star * dt_ / dx_; + const float c_3 = S_r * dt_ / dx_; + + // Compute the "upwind change" vectors for the i-3/2 and i+3/2 interfaces + const float eps = 1.0e-6f; + const float r_1 = desingularize( (c_1 > 0.0f) ? (h_dag_l - h_l2) : (h_dag_r - h_r), eps) / desingularize((h_dag - h_l), eps); + const float r_2 = desingularize( (c_2 > 0.0f) ? (v_l - v_l2) : (v_r2 - v_r), eps ) / desingularize((v_r - v_l), eps); + const float r_3 = desingularize( (c_3 > 0.0f) ? (h_l - h_dag_l) : (h_r2 - h_dag_r), eps ) / desingularize((h_r - h_dag), eps); + + // Compute the limiter + // We use h for the nonlinear waves, and v for the middle shear wave + const float A_1 = copysign(1.0f, c_1) * limiterToWAFLimiter(generalized_minmod(r_1, 1.9f), c_1); + const float A_2 = copysign(1.0f, c_2) * limiterToWAFLimiter(generalized_minmod(r_2, 1.9f), c_2); + const float A_3 = copysign(1.0f, c_3) * limiterToWAFLimiter(generalized_minmod(r_3, 1.9f), c_3); + + //Average the fluxes + const float3 flux = 0.5f*( F_1 + F_4 ) + - 0.5f*( A_1 * (F_2 - F_1) + + A_2 * (F_3 - F_2) + + A_3 * (F_4 - F_3) ); + + return flux; +} + + + + + + + + + + + + + + +/** + * Central upwind flux function + */ +__device__ float3 CentralUpwindFlux(const float3 Qm, float3 Qp, const float g) { + const float3 Fp = F_func(Qp, g); + const float up = Qp.y / Qp.x; // hu / h + const float cp = sqrt(g*Qp.x); // sqrt(g*h) + + const float3 Fm = F_func(Qm, g); + const float um = Qm.y / Qm.x; // hu / h + const float cm = sqrt(g*Qm.x); // sqrt(g*h) + + const float am = min(min(um-cm, up-cp), 0.0f); // largest negative wave speed + const float ap = max(max(um+cm, up+cp), 0.0f); // largest positive wave speed + + return ((ap*Fm - am*Fp) + ap*am*(Qp-Qm))/(ap-am); +} + + + + + + + + + +/** + * Godunovs centered scheme (Toro 2001, p 165) + */ +__device__ float3 GodC_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + const float3 Q_godc = 0.5f*(Q_l + Q_r) + (dt_/dx_)*(F_l - F_r); + + return F_func(Q_godc, g_); +} + + + + + + + + + +/** + * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 180) + */ +__device__ float3 HLL_flux(const float3 Q_l, const float3 Q_r, const float g_) { + const float h_l = Q_l.x; + const float h_r = Q_r.x; + + // Calculate velocities + const float u_l = Q_l.y / h_l; + const float u_r = Q_r.y / h_r; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + // Compute h in the "star region", h^dagger + const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + + //Upwind selection + if (S_l >= 0.0f) { + return F_func(Q_l, g_); + } + else if (S_r <= 0.0f) { + return F_func(Q_r, g_); + } + //Or estimate flux in the star region + else { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + const float3 flux = (S_r*F_l - S_l*F_r + S_r*S_l*(Q_r - Q_l)) / (S_r-S_l); + return flux; + } +} + + + + + + + + + + + + + + + + + +/** + * Harten-Lax-van Leer with contact discontinuity (Toro 2001, p 181) + */ +__device__ float3 HLLC_flux(const float3 Q_l, const float3 Q_r, const float g_) { + const float h_l = Q_l.x; + const float h_r = Q_r.x; + + // Calculate velocities + const float u_l = Q_l.y / h_l; + const float u_r = Q_r.y / h_r; + + // Estimate the potential wave speeds + const float c_l = sqrt(g_*h_l); + const float c_r = sqrt(g_*h_r); + + // Compute h in the "star region", h^dagger + const float h_dag = 0.5f * (h_l+h_r) - 0.25f * (u_r-u_l)*(h_l+h_r)/(c_l+c_r); + + const float q_l_tmp = sqrt(0.5f * ( (h_dag+h_l)*h_dag / (h_l*h_l) ) ); + const float q_r_tmp = sqrt(0.5f * ( (h_dag+h_r)*h_dag / (h_r*h_r) ) ); + + const float q_l = (h_dag > h_l) ? q_l_tmp : 1.0f; + const float q_r = (h_dag > h_r) ? q_r_tmp : 1.0f; + + // Compute wave speed estimates + const float S_l = u_l - c_l*q_l; + const float S_r = u_r + c_r*q_r; + const float S_star = ( S_l*h_r*(u_r - S_r) - S_r*h_l*(u_l - S_l) ) / ( h_r*(u_r - S_r) - h_l*(u_l - S_l) ); + + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + //Upwind selection + if (S_l >= 0.0f) { + return F_l; + } + else if (S_r <= 0.0f) { + return F_r; + } + //Or estimate flux in the "left star" region + else if (S_l <= 0.0f && 0.0f <=S_star) { + const float v_l = Q_l.z / h_l; + const float3 Q_star_l = h_l * (S_l - u_l) / (S_l - S_star) * make_float3(1, S_star, v_l); + const float3 flux = F_l + S_l*(Q_star_l - Q_l); + return flux; + } + //Or estimate flux in the "righ star" region + else if (S_star <= 0.0f && 0.0f <=S_r) { + const float v_r = Q_r.z / h_r; + const float3 Q_star_r = h_r * (S_r - u_r) / (S_r - S_star) * make_float3(1, S_star, v_r); + const float3 flux = F_r + S_r*(Q_star_r - Q_r); + return flux; + } + else { + return make_float3(-99999.9f, -99999.9f, -99999.9f); //Something wrong here + } +} + + + + + + + + + + + + + +/** + * Lax-Friedrichs flux (Toro 2001, p 163) + */ +__device__ float3 LxF_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + return 0.5f*(F_l + F_r) + (dx_/(2.0f*dt_))*(Q_l - Q_r); +} + + + + + + + + +/** + * Lax-Friedrichs extended to 2D + */ +__device__ float3 LxF_2D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + //Note numerical diffusion for 2D here (0.25) + return 0.5f*(F_l + F_r) + (dx_/(4.0f*dt_))*(Q_l - Q_r); +} + + + + + + + + + + + + +/** + * Richtmeyer / Two-step Lax-Wendroff flux (Toro 2001, p 164) + */ +__device__ float3 LxW2_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_l = F_func(Q_l, g_); + const float3 F_r = F_func(Q_r, g_); + + const float3 Q_lw2 = 0.5f*(Q_l + Q_r) + (dt_/(2.0f*dx_))*(F_l - F_r); + + return F_func(Q_lw2, g_); +} + + + + + + + + + + + + +/** + * First Ordered Centered (Toro 2001, p.163) + */ +__device__ float3 FORCE_1D_flux(const float3 Q_l, const float3 Q_r, const float g_, const float dx_, const float dt_) { + const float3 F_lf = LxF_1D_flux(Q_l, Q_r, g_, dx_, dt_); + const float3 F_lw2 = LxW2_1D_flux(Q_l, Q_r, g_, dx_, dt_); + return 0.5f*(F_lf + F_lw2); +} + + + + + + + + + + +template +__device__ void writeCfl(float Q[vars][h+2*gc_y][w+2*gc_x], + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const float dx_, const float dy_, const float g_, + float* output_) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + tx; + const int tj = blockDim.y*blockIdx.y + ty; + + //Only internal cells + if (ti < nx_+gc_x && tj < ny_+gc_y) { + const float h = Q[0][ty][tx]; + const float u = Q[1][ty][tx] / h; + const float v = Q[2][ty][tx] / h; + + const float max_u = dx_ / (fabsf(u) + sqrtf(g_*h)); + const float max_v = dy_ / (fabsf(v) + sqrtf(g_*h)); + + shmem[ty][tx] = fminf(max_u, max_v); + } + __syncthreads(); + + //One row of threads loop over all rows + if (ti < nx_+gc_x && tj < ny_+gc_y) { + if (ty == gc_y) { + float min_val = shmem[ty][tx]; + const int max_y = min(h, ny_+gc_y - tj); + for (int j=gc_y; j. +*/ + +#pragma once + + +/** + * Float3 operators + */ +inline __device__ float3 operator*(const float a, const float3 b) { + return make_float3(a*b.x, a*b.y, a*b.z); +} + +inline __device__ float3 operator/(const float3 a, const float b) { + return make_float3(a.x/b, a.y/b, a.z/b); +} + +inline __device__ float3 operator-(const float3 a, const float3 b) { + return make_float3(a.x-b.x, a.y-b.y, a.z-b.z); +} + +inline __device__ float3 operator+(const float3 a, const float3 b) { + return make_float3(a.x+b.x, a.y+b.y, a.z+b.z); +} + +/** + * Float4 operators + */ +inline __device__ float4 operator*(const float a, const float4 b) { + return make_float4(a*b.x, a*b.y, a*b.z, a*b.w); +} + +inline __device__ float4 operator/(const float4 a, const float b) { + return make_float4(a.x/b, a.y/b, a.z/b, a.w/b); +} + +inline __device__ float4 operator-(const float4 a, const float4 b) { + return make_float4(a.x-b.x, a.y-b.y, a.z-b.z, a.w-b.w); +} + +inline __device__ float4 operator+(const float4 a, const float4 b) { + return make_float4(a.x+b.x, a.y+b.y, a.z+b.z, a.w+b.w); +} + + + + +inline __device__ __host__ float clamp(const float f, const float a, const float b) { + return fmaxf(a, fminf(f, b)); +} + +inline __device__ __host__ int clamp(const int f, const int a, const int b) { + return (f < b) ? ( (f > a) ? f : a) : b; +} + + + + + +__device__ float desingularize(float x_, float eps_) { + return copysign(1.0f, x_)*fmaxf(fabsf(x_), fminf(x_*x_/(2.0f*eps_)+0.5f*eps_, eps_)); +} + + + + + + + + +/** + * Returns the step stored in the leftmost 16 bits + * of the 32 bit step-order integer + */ +inline __device__ int getStep(int step_order_) { + return step_order_ >> 16; +} + +/** + * Returns the order stored in the rightmost 16 bits + * of the 32 bit step-order integer + */ +inline __device__ int getOrder(int step_order_) { + return step_order_ & 0x0000FFFF; +} + + +enum BoundaryCondition { + Dirichlet = 0, + Neumann = 1, + Periodic = 2, + Reflective = 3 +}; + +inline __device__ BoundaryCondition getBCNorth(int bc_) { + return static_cast((bc_ >> 24) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCSouth(int bc_) { + return static_cast((bc_ >> 16) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCEast(int bc_) { + return static_cast((bc_ >> 8) & 0x0000000F); +} + +inline __device__ BoundaryCondition getBCWest(int bc_) { + return static_cast((bc_ >> 0) & 0x0000000F); +} + + +// West boundary +template +__device__ void bcWestReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int j=threadIdx.y; j= 1 && ti == gc_x) { + Q[j][i-1] = sign*Q[j][i]; + } + if (gc_x >= 2 && ti == gc_x + 1) { + Q[j][i-3] = sign*Q[j][i]; + } + if (gc_x >= 3 && ti == gc_x + 2) { + Q[j][i-5] = sign*Q[j][i]; + } + if (gc_x >= 4 && ti == gc_x + 3) { + Q[j][i-7] = sign*Q[j][i]; + } + if (gc_x >= 5 && ti == gc_x + 4) { + Q[j][i-9] = sign*Q[j][i]; + } + } +} + + +// East boundary +template +__device__ void bcEastReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int j=threadIdx.y; j= 1 && ti == nx_ + gc_x - 1) { + Q[j][i+1] = sign*Q[j][i]; + } + if (gc_x >= 2 && ti == nx_ + gc_x - 2) { + Q[j][i+3] = sign*Q[j][i]; + } + if (gc_x >= 3 && ti == nx_ + gc_x - 3) { + Q[j][i+5] = sign*Q[j][i]; + } + if (gc_x >= 4 && ti == nx_ + gc_x - 4) { + Q[j][i+7] = sign*Q[j][i]; + } + if (gc_x >= 5 && ti == nx_ + gc_x - 5) { + Q[j][i+9] = sign*Q[j][i]; + } + } +} + + +// South boundary +template +__device__ void bcSouthReflective(float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_) { + for (int i=threadIdx.x; i= 1 && tj == gc_y) { + Q[j-1][i] = sign*Q[j][i]; + } + if (gc_y >= 2 && tj == gc_y + 1) { + Q[j-3][i] = sign*Q[j][i]; + } + if (gc_y >= 3 && tj == gc_y + 2) { + Q[j-5][i] = sign*Q[j][i]; + } + if (gc_y >= 4 && tj == gc_y + 3) { + Q[j-7][i] = sign*Q[j][i]; + } + if (gc_y >= 5 && tj == gc_y + 4) { + Q[j-9][i] = sign*Q[j][i]; + } + } +} + + + + +// North boundary +template +__device__ void bcNorthReflective(float Q[h+2*gc_y][w+2*gc_x], const int nx_, const int ny_) { + for (int i=threadIdx.x; i= 1 && tj == ny_ + gc_y - 1) { + Q[j+1][i] = sign*Q[j][i]; + } + if (gc_y >= 2 && tj == ny_ + gc_y - 2) { + Q[j+3][i] = sign*Q[j][i]; + } + if (gc_y >= 3 && tj == ny_ + gc_y - 3) { + Q[j+5][i] = sign*Q[j][i]; + } + if (gc_y >= 4 && tj == ny_ + gc_y - 4) { + Q[j+7][i] = sign*Q[j][i]; + } + if (gc_y >= 5 && tj == ny_ + gc_y - 5) { + Q[j+9][i] = sign*Q[j][i]; + } + } +} + + + + +/** + * Alter the index l so that it gives periodic boundary conditions when reading + */ +template +inline __device__ int handlePeriodicBoundaryX(int k, int nx_, int boundary_conditions_) { + const int gc_pad = gc_x; + + //West boundary: add an offset to read from east of domain + if (gc_x > 0) { + if ((k < gc_pad) + && getBCWest(boundary_conditions_) == Periodic) { + k += (nx_+2*gc_x - 2*gc_pad); + } + //East boundary: subtract an offset to read from west of domain + else if ((k >= nx_+2*gc_x-gc_pad) + && getBCEast(boundary_conditions_) == Periodic) { + k -= (nx_+2*gc_x - 2*gc_pad); + } + } + + return k; +} + +/** + * Alter the index l so that it gives periodic boundary conditions when reading + */ +template +inline __device__ int handlePeriodicBoundaryY(int l, int ny_, int boundary_conditions_) { + const int gc_pad = gc_y; + + //South boundary: add an offset to read from north of domain + if (gc_y > 0) { + if ((l < gc_pad) + && getBCSouth(boundary_conditions_) == Periodic) { + l += (ny_+2*gc_y - 2*gc_pad); + } + //North boundary: subtract an offset to read from south of domain + else if ((l >= ny_+2*gc_y-gc_pad) + && getBCNorth(boundary_conditions_) == Periodic) { + l -= (ny_+2*gc_y - 2*gc_pad); + } + } + + return l; +} + + +template +inline __device__ +void handleReflectiveBoundary( + float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const int boundary_conditions_) { + + //Handle reflective boundary conditions + if (getBCNorth(boundary_conditions_) == Reflective) { + bcNorthReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCSouth(boundary_conditions_) == Reflective) { + bcSouthReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCEast(boundary_conditions_) == Reflective) { + bcEastReflective(Q, nx_, ny_); + __syncthreads(); + } + if (getBCWest(boundary_conditions_) == Reflective) { + bcWestReflective(Q, nx_, ny_); + __syncthreads(); + } +} + +/** + * Reads a block of data with ghost cells + */ +template +inline __device__ void readBlock(float* ptr_, int pitch_, + float Q[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + const int boundary_conditions_, + int x0, int y0, + int x1, int y1) { + //Index of block within domain + const int bx = blockDim.x * blockIdx.x; + const int by = blockDim.y * blockIdx.y; + + //Read into shared memory + //Loop over all variables + for (int j=threadIdx.y; j(by + j + y0, ny_, boundary_conditions_); + l = min(l, min(ny_+2*gc_y-1, y1+2*gc_y-1)); + float* row = (float*) ((char*) ptr_ + pitch_*l); + + for (int i=threadIdx.x; i(bx + i + x0, nx_, boundary_conditions_); + k = min(k, min(nx_+2*gc_x-1, x1+2*gc_x-1)); + + //Read from global memory + Q[j][i] = row[k]; + } + } + __syncthreads(); + + handleReflectiveBoundary(Q, nx_, ny_, boundary_conditions_); +} + + + + +/** + * Writes a block of data to global memory for the shallow water equations. + */ +template +inline __device__ void writeBlock(float* ptr_, int pitch_, + float shmem[h+2*gc_y][w+2*gc_x], + const int nx_, const int ny_, + int rk_step_, int rk_order_, + int x0, int y0, + int x1, int y1) { + + //Index of cell within domain + const int ti = blockDim.x*blockIdx.x + threadIdx.x + gc_x + x0; + const int tj = blockDim.y*blockIdx.y + threadIdx.y + gc_y + y0; + + //In case we are writing only to a subarea given by (x0, y0) x (x1, y1) + const int max_ti = min(nx_+gc_x, x1+gc_x); + const int max_tj = min(ny_+gc_y, y1+gc_y); + + //Only write internal cells + if ((x0+gc_x <= ti) && (ti < max_ti) && (y0+gc_y <= tj) && (tj < max_tj)) { + //Index of thread within block + const int tx = threadIdx.x + gc_x; + const int ty = threadIdx.y + gc_y; + + float* const row = (float*) ((char*) ptr_ + pitch_*tj); + + //Handle runge-kutta timestepping here + row[ti] = shmem[ty][tx]; + + + + /** + * SSPRK1 (forward Euler) + * u^1 = u^n + dt*f(u^n) + */ + if (rk_order_ == 1) { + row[ti] = shmem[ty][tx]; + } + /** + * SSPRK2 + * u^1 = u^n + dt*f(u^n) + * u^n+1 = 1/2*u^n + 1/2*(u^1 + dt*f(u^1)) + */ + else if (rk_order_ == 2) { + if (rk_step_ == 0) { + row[ti] = shmem[ty][tx]; + } + else if (rk_step_ == 1) { + row[ti] = 0.5f*row[ti] + 0.5f*shmem[ty][tx]; + } + } + /** + * SSPRK3 + * u^1 = u^n + dt*f(u^n) + * u^2 = 3/4 * u^n + 1/4 * (u^1 + dt*f(u^1)) + * u^n+1 = 1/3 * u^n + 2/3 * (u^2 + dt*f(u^2)) + * FIXME: This is not correct now, need a temporary to hold intermediate step u^2 + */ + else if (rk_order_ == 3) { + if (rk_step_ == 0) { + row[ti] = shmem[ty][tx]; + } + else if (rk_step_ == 1) { + row[ti] = 0.75f*row[ti] + 0.25f*shmem[ty][tx]; + } + else if (rk_step_ == 2) { + const float t = 1.0f / 3.0f; //Not representable in base 2 + row[ti] = t*row[ti] + (1.0f-t)*shmem[ty][tx]; + } + } + + // DEBUG + //row[ti] = 99.0; + } +} + + + + + + + + + + + +template +__device__ void evolveF(float Q[vars][h+2*gc_y][w+2*gc_x], + float F[vars][h+2*gc_y][w+2*gc_x], + const float dx_, const float dt_) { + for (int var=0; var < vars; ++var) { + for (int j=threadIdx.y; j +__device__ void evolveG(float Q[vars][h+2*gc_y][w+2*gc_x], + float G[vars][h+2*gc_y][w+2*gc_x], + const float dy_, const float dt_) { + for (int var=0; var < vars; ++var) { + for (int j=threadIdx.y+gc_y; j +__device__ void memset(float Q[vars][shmem_height][shmem_width], float value) { + for (int k=0; k +__device__ void reduce_max(float* data, unsigned int n) { + __shared__ float sdata[threads]; + unsigned int tid = threadIdx.x; + + //Reduce to "threads" elements + sdata[tid] = FLT_MIN; + for (unsigned int i=tid; i= 512) { + if (tid < 256) { + sdata[tid] = max(sdata[tid], sdata[tid + 256]); + } + __syncthreads(); + } + if (threads >= 256) { + if (tid < 128) { + sdata[tid] = max(sdata[tid], sdata[tid + 128]); + } + __syncthreads(); + } + if (threads >= 128) { + if (tid < 64) { + sdata[tid] = max(sdata[tid], sdata[tid + 64]); + } + __syncthreads(); + } + if (tid < 32) { + volatile float* sdata_volatile = sdata; + if (threads >= 64) { + sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 32]); + } + if (tid < 16) { + if (threads >= 32) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 16]); + if (threads >= 16) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 8]); + if (threads >= 8) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 4]); + if (threads >= 4) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 2]); + if (threads >= 2) sdata_volatile[tid] = max(sdata_volatile[tid], sdata_volatile[tid + 1]); + } + + if (tid == 0) { + return sdata_volatile[0]; + } + } +} + + + + + + + + + + diff --git a/GPUSimulators/cuda/limiters.h b/GPUSimulators/cuda/limiters.h new file mode 100644 index 0000000..c2effa7 --- /dev/null +++ b/GPUSimulators/cuda/limiters.h @@ -0,0 +1,118 @@ +/* +This file implements different flux and slope limiters + +Copyright (C) 2016, 2017, 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +#pragma once + + + + + + +/** + * Reconstructs a slope using the generalized minmod limiter based on three + * consecutive values + */ +__device__ __inline__ float minmodSlope(float left, float center, float right, float theta) { + const float backward = (center - left) * theta; + const float central = (right - left) * 0.5f; + const float forward = (right - center) * theta; + + return 0.25f + *copysign(1.0f, backward) + *(copysign(1.0f, backward) + copysign(1.0f, central)) + *(copysign(1.0f, central) + copysign(1.0f, forward)) + *min( min(fabs(backward), fabs(central)), fabs(forward) ); +} + + + + +/** + * Reconstructs a minmod slope for a whole block along the abscissa + */ +template +__device__ void minmodSlopeX(float Q[vars][h+2*gc_y][w+2*gc_x], + float Qx[vars][h+2*gc_y][w+2*gc_x], + const float theta_) { + //Reconstruct slopes along x axis + for (int p=0; p +__device__ void minmodSlopeY(float Q[vars][h+2*gc_y][w+2*gc_x], + float Qy[vars][h+2*gc_y][w+2*gc_x], + const float theta_) { + //Reconstruct slopes along y axis + for (int p=0; p. +""" + + +from GPUSimulators.Simulator import BoundaryCondition +import numpy as np +import gc + + +def getExtent(width, height, nx, ny, grid, index=None): + if grid is not None: + gx = grid.grid[0] + gy = grid.grid[1] + if index is not None: + i, j = grid.getCoordinate(index) + else: + i, j = grid.getCoordinate() + + dx = (width / gx) / nx + dy = (height / gy) / ny + + x0 = width*i/gx + 0.5*dx + y0 = height*j/gy + 0.5*dy + x1 = width*(i+1)/gx - 0.5*dx + y1 = height*(j+1)/gy - 0.5*dx + + else: + dx = width / nx + dy = height / ny + + x0 = 0.5*dx + y0 = 0.5*dy + x1 = width-0.5*dx + y1 = height-0.5*dy + + return [x0, x1, y0, y1, dx, dy] + + +def downsample(highres_solution, x_factor, y_factor=None): + if (y_factor == None): + y_factor = x_factor + + assert(highres_solution.shape[1] % x_factor == 0) + assert(highres_solution.shape[0] % y_factor == 0) + + if (x_factor*y_factor == 1): + return highres_solution + + if (len(highres_solution.shape) == 1): + highres_solution = highres_solution.reshape((1, highres_solution.size)) + + nx = highres_solution.shape[1] / x_factor + ny = highres_solution.shape[0] / y_factor + + return highres_solution.reshape([int(ny), int(y_factor), int(nx), int(x_factor)]).mean(3).mean(1) + + + + + +def bump(nx, ny, width, height, + bump_size=None, + ref_nx=None, ref_ny=None, + x_center=0.5, y_center=0.5, + h_ref=0.5, h_amp=0.1, u_ref=0.0, u_amp=0.1, v_ref=0.0, v_amp=0.1): + + if (ref_nx == None): + ref_nx = nx + assert(ref_nx >= nx) + + if (ref_ny == None): + ref_ny = ny + assert(ref_ny >= ny) + + if (bump_size == None): + bump_size = width/5.0 + + ref_dx = width / float(ref_nx) + ref_dy = height / float(ref_ny) + + x_center = ref_dx*ref_nx*x_center + y_center = ref_dy*ref_ny*y_center + + x = ref_dx*(np.arange(0, ref_nx, dtype=np.float32)+0.5) - x_center + y = ref_dy*(np.arange(0, ref_ny, dtype=np.float32)+0.5) - y_center + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + r = np.sqrt(xv**2 + yv**2) + xv = None + yv = None + gc.collect() + + #Generate highres then downsample + #h_highres = 0.5 + 0.1*np.exp(-(xv**2/size + yv**2/size)) + h_highres = h_ref + h_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + h = downsample(h_highres, ref_nx/nx, ref_ny/ny) + h_highres = None + gc.collect() + + #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) + u_highres = u_ref + u_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + hu = downsample(u_highres, ref_nx/nx, ref_ny/ny)*h + u_highres = None + gc.collect() + + #hu_highres = 0.1*np.exp(-(xv**2/size + yv**2/size)) + v_highres = v_ref + v_amp*0.5*(1.0 + np.cos(np.pi*r/bump_size)) * (r < bump_size) + hv = downsample(v_highres, ref_nx/nx, ref_ny/ny)*h + v_highres = None + gc.collect() + + dx = width/nx + dy = height/ny + + return h, hu, hv, dx, dy + + +def genShockBubble(nx, ny, gamma, grid=None): + """ + Generate Shock-bubble interaction case for the Euler equations + """ + + width = 4.0 + height = 1.0 + g = 0.0 + + + rho = np.ones((ny, nx), dtype=np.float32) + u = np.zeros((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + E = np.zeros((ny, nx), dtype=np.float32) + p = np.ones((ny, nx), dtype=np.float32) + + + x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) + x = np.linspace(x0, x1, nx, dtype=np.float32) + y = np.linspace(y0, y1, ny, dtype=np.float32) + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + + #Bubble + radius = 0.25 + x_center = 0.5 + y_center = 0.5 + bubble = np.sqrt((xv-x_center)**2+(yv-y_center)**2) <= radius + rho = np.where(bubble, 0.1, rho) + + #Left boundary + left = (xv < 0.1) + rho = np.where(left, 3.81250, rho) + u = np.where(left, 2.57669, u) + + #Energy + p = np.where(left, 10.0, p) + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Reflective, + 'south': BoundaryCondition.Type.Reflective, + 'east': BoundaryCondition.Type.Periodic, + 'west': BoundaryCondition.Type.Periodic + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + return arguments + + + + + + + +def genKelvinHelmholtz(nx, ny, gamma, roughness=0.125, grid=None, index=None): + """ + Roughness parameter in (0, 1.0] determines how "squiggly" + the interface betweeen the zones is + """ + + def genZones(nx, ny, n): + """ + Generates the zones of the two fluids of K-H + """ + zone = np.zeros((ny, nx), dtype=np.int32) + + + def genSmoothRandom(nx, n): + n = max(1, min(n, nx)) + + if n == nx: + return np.random.random(nx)-0.5 + else: + from scipy.interpolate import interp1d + + #Control points and interpolator + xp = np.linspace(0.0, 1.0, n) + yp = np.random.random(n) - 0.5 + + if (n == 1): + kind = 'nearest' + elif (n == 2): + kind = 'linear' + elif (n == 3): + kind = 'quadratic' + else: + kind = 'cubic' + + f = interp1d(xp, yp, kind=kind) + + #Interpolation points + x = np.linspace(0.0, 1.0, nx) + return f(x) + + + + x0, x1, y0, y1, _, dy = getExtent(1.0, 1.0, nx, ny, grid, index) + x = np.linspace(x0, x1, nx) + y = np.linspace(y0, y1, ny) + _, y = np.meshgrid(x, y) + + #print(y+a[0]) + + a = genSmoothRandom(nx, n)*dy + zone = np.where(y > 0.25+a, zone, 1) + + a = genSmoothRandom(nx, n)*dy + zone = np.where(y < 0.75+a, zone, 1) + + return zone + + width = 2.0 + height = 1.0 + g = 0.0 + gamma = 1.4 + + rho = np.empty((ny, nx), dtype=np.float32) + u = np.empty((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + p = 2.5*np.ones((ny, nx), dtype=np.float32) + + #Generate the different zones + zones = genZones(nx, ny, max(1, min(nx, int(nx*roughness)))) + + #Zone 0 + zone0 = zones == 0 + rho = np.where(zone0, 1.0, rho) + u = np.where(zone0, 0.5, u) + + #Zone 1 + zone1 = zones == 1 + rho = np.where(zone1, 2.0, rho) + u = np.where(zone1, -0.5, u) + + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + _, _, _, _, dx, dy = getExtent(width, height, nx, ny, grid, index) + + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Periodic, + 'south': BoundaryCondition.Type.Periodic, + 'east': BoundaryCondition.Type.Periodic, + 'west': BoundaryCondition.Type.Periodic + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + + return arguments + + + +def genRayleighTaylor(nx, ny, gamma, version=0, grid=None): + """ + Generates Rayleigh-Taylor instability case + """ + width = 0.5 + height = 1.5 + g = 0.1 + + rho = np.zeros((ny, nx), dtype=np.float32) + u = np.zeros((ny, nx), dtype=np.float32) + v = np.zeros((ny, nx), dtype=np.float32) + p = np.zeros((ny, nx), dtype=np.float32) + + + x0, x1, y0, y1, dx, dy = getExtent(width, height, nx, ny, grid) + x = np.linspace(x0, x1, nx, dtype=np.float32)-width*0.5 + y = np.linspace(y0, y1, ny, dtype=np.float32)-height*0.5 + xv, yv = np.meshgrid(x, y, sparse=False, indexing='xy') + + #This gives a squigly interfact + if (version == 0): + y_threshold = 0.01*np.cos(2*np.pi*np.abs(x)/0.5) + rho = np.where(yv <= y_threshold, 1.0, rho) + rho = np.where(yv > y_threshold, 2.0, rho) + elif (version == 1): + rho = np.where(yv <= 0.0, 1.0, rho) + rho = np.where(yv > 0.0, 2.0, rho) + v = 0.01*(1.0 + np.cos(2*np.pi*xv/0.5))/4 + else: + assert False, "Invalid version" + + p = 2.5 - rho*g*yv + E = 0.5*rho*(u**2+v**2) + p/(gamma-1.0) + + bc = BoundaryCondition({ + 'north': BoundaryCondition.Type.Reflective, + 'south': BoundaryCondition.Type.Reflective, + 'east': BoundaryCondition.Type.Reflective, + 'west': BoundaryCondition.Type.Reflective + }) + + #Construct simulator + arguments = { + 'rho': rho, 'rho_u': rho*u, 'rho_v': rho*v, 'E': E, + 'nx': nx, 'ny': ny, + 'dx': dx, 'dy': dy, + 'g': g, + 'gamma': gamma, + 'boundary_conditions': bc + } + + return arguments \ No newline at end of file diff --git a/GPUSimulators/helpers/Visualization.py b/GPUSimulators/helpers/Visualization.py new file mode 100644 index 0000000..a2ff8f1 --- /dev/null +++ b/GPUSimulators/helpers/Visualization.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements visualization techniques/modes + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + + +import numpy as np + +from matplotlib.colors import Normalize + + + +def genSchlieren(rho): + #Compute length of z-component of normalized gradient vector + normal = np.gradient(rho) #[x, y, 1] + length = 1.0 / np.sqrt(normal[0]**2 + normal[1]**2 + 1.0) + schlieren = np.power(length, 128) + return schlieren + + +def genVorticity(rho, rho_u, rho_v): + u = rho_u / rho + v = rho_v / rho + u = np.sqrt(u**2 + v**2) + u_max = u.max() + + du_dy, _ = np.gradient(u) + _, dv_dx = np.gradient(v) + + #Length of curl + curl = dv_dx - du_dy + return curl + + +def genColors(rho, rho_u, rho_v, cmap, vmax, vmin): + schlieren = genSchlieren(rho) + curl = genVorticity(rho, rho_u, rho_v) + + colors = Normalize(vmin, vmax, clip=True)(curl) + colors = cmap(colors) + for k in range(3): + colors[:,:,k] = colors[:,:,k]*schlieren + + return colors \ No newline at end of file From b312923ce69d357cd33b434666ecb526deb559e7 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 27 Feb 2024 15:55:22 +0100 Subject: [PATCH 45/55] Recover deleted files from commit e2b1281 --- Jobs/job_lumi.slrum | 34 + LICENSE | 674 +++++ README.md | 45 + SYSTEMS.md | 62 + conda_environment_lumi.yml | 36 + mpiTesting.py | 230 ++ reference/swashes_1_nx=1024.csv | 1042 ++++++++ reference/swashes_1_nx=128.csv | 146 ++ reference/swashes_1_nx=2048.csv | 2066 ++++++++++++++++ reference/swashes_1_nx=256.csv | 274 ++ reference/swashes_1_nx=4096.csv | 4114 +++++++++++++++++++++++++++++++ reference/swashes_1_nx=512.csv | 530 ++++ 12 files changed, 9253 insertions(+) create mode 100644 Jobs/job_lumi.slrum create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SYSTEMS.md create mode 100644 conda_environment_lumi.yml create mode 100644 mpiTesting.py create mode 100644 reference/swashes_1_nx=1024.csv create mode 100644 reference/swashes_1_nx=128.csv create mode 100644 reference/swashes_1_nx=2048.csv create mode 100644 reference/swashes_1_nx=256.csv create mode 100644 reference/swashes_1_nx=4096.csv create mode 100644 reference/swashes_1_nx=512.csv diff --git a/Jobs/job_lumi.slrum b/Jobs/job_lumi.slrum new file mode 100644 index 0000000..b946bbf --- /dev/null +++ b/Jobs/job_lumi.slrum @@ -0,0 +1,34 @@ +#!/bin/bash -e +#SBATCH --job-name=lumi +#SBATCH --account=project_4650000xx +#SBATCH --time=00:10:00 +#SBATCH --partition=dev-g +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=2 +#SBATCH --gpus=2 +#SBATCH --gpus-per-node=2 +#SBATCH -o %x-%j.out +# + +N=$SLURM_JOB_NUM_NODES +echo "--nbr of nodes:", $N +echo "--total nbr of gpus:", $SLURM_NTASKS + +Mydir=/project/project_4650000xx +Myapplication=${Mydir}/FiniteVolumeGPU_hip/mpiTesting.py + +#modules +ml LUMI/23.03 partition/G +ml lumi-container-wrapper +ml cray-python/3.9.13.1 +ml rocm/5.2.3 + +ml craype-accel-amd-gfx90a +ml cray-mpich/8.1.27 + +#Enable GPU-aware MPI +export MPICH_GPU_SUPPORT_ENABLED=1 + +export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH" + +srun python ${Myapplication} -nx 1024 -ny 1024 --profile diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5f68f36 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# FiniteVolumeGPU + +This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. + +## Setup +A good place to start exploring this codebase is the notebooks. Complete the following steps to run the notebooks: + +1. Install conda (see e.g. Miniconda or Anaconda) +2. Change directory to the repository root and run the following commands +3. conda env create -f conda_environment.yml +4. conda activate ShallowWaterGPU +5. jupyter notebook + +Make sure you are running the correct kernel ("conda:ShallowWaterGPU"). If not, change kernel using the "Kernel"-menu in the notebook. + +If you do not need to run notebooks you may use the conda environment found in conda_environment_hpc.yml + +## Troubleshooting +Have a look at the conda documentation and https://towardsdatascience.com/how-to-set-up-anaconda-and-jupyter-notebook-the-right-way-de3b7623ea4a + +## Setup on LUMI-G +Here is a step-by-step guide on installing packages on LUMI-G + +### Step 0: load modules +``` +ml LUMI/23.03 +ml lumi-container-wrapper +ml cray-python/3.9.13.1 +``` + +### Step 1: run conda-container +Installation via conda can be done as: +``` +conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml +``` +where the file `conda_environment_lumi.yml` contains packages to be installed. + +### Step 2: Set the env. variable to search for binaries +``` +export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" +``` +### An alternative: Convert to a singularity container with cotainr +``` +cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml +``` diff --git a/SYSTEMS.md b/SYSTEMS.md new file mode 100644 index 0000000..5b0e770 --- /dev/null +++ b/SYSTEMS.md @@ -0,0 +1,62 @@ +$ANSIBLE_VAULT;1.1;AES256 +61316265663939333638336466323036663861343233316466646432313138653633623662353937 +3232313165656633346432376237383566363537366534310a303231343936663438653835373161 +35616161323432653062323164623861353065333761663136313137333732313230626665386336 +6166656538396463370a356166316363326133313864386536323236346634323537393639653038 +66336337336132613061353964613638326233356336323962366531333932366539366339623563 +36333365326463616634323939333062363263636663373635653064626138363464666233316561 +63393735393233616437386537393739393433663631313864646535636262616336333631396166 +38643636323530386565396338623366393232313838356536303537393338393634666632656234 +65353930303762333639376638336364303439306132626531326132376264623063376464636430 +32333536386134333136313139313861306364333037323363393463333664633764653937623866 +34313064346261313330373132353563343761323435393930303136353865303163373937623831 +64343038373162333039653161643233353764633337366434396638376530636261323362373434 +38393630613065356632663533333331633039663935663732353234643131306665343339373265 +64356563653838613337663132663234356462343333623139626662363764656239326637653832 +35396636643937336431623531306133643137623831333936313839333738333730373136666336 +35623965643664343164373630313362656663386638376237616134343631386366313336626138 +62376436383837376539663438346431633138383363633862356366376537393932626262383637 +31323365333139653736623233636233323162343039663035346135326638633430303134396337 +36353264313835346130643736663665386364343835643166383361316631373338663731373335 +30313530326662663937666330643565363565616566633333363535656539656531666266613638 +30306264613438363265646332386535383238373433396337633636616532626161343236336533 +36366362653137333739353737386563613136653164383437316237643533633133313735633363 +64326433356266363133343339626333633063326533383632353639613163663966376465396231 +36663034363534396430316463386564663465323036613636343136643262666566303533346439 +63396466656639623836613130363835346435633437666463363333356231343038356434343861 +66643636633739336666316566653136363862346336353862653130346335363334616430366435 +30376365383262326438306266366265363030353764633630333034663037643037343132303631 +39316364366234363339363130333765616432306331373566393530653963356539636437383062 +34633938643563656363633864656361643539663833356638356365373061663964363530393535 +37646533386235613763396638393539303062326239633238373763326561313634313265613135 +64646138313562313732393732303133343234323438616165326530333234626363393735636530 +62353735313231353662353533636134306530623339383730306332613636663366653566313935 +32343935353566656130393533323639353863666436333839386463396337336635356663373136 +61323734613239396236393266363631313465363630306565636663396235626132336339623938 +62383435643661623938393662363262376566613365613465323432343534356433323330666133 +30303963656635303734316539333038663962626331313366666337663165323230646564623935 +61316630353739386365323339626166323562616630383538393733353864396565353039656333 +30343038636231363531383061613836653038373937616163643963393231356235626531366239 +62343333326434636665363931376235313535343135626261336439636663323233383565633964 +65333830613131396630336337646230393038386536336365313738316335386261393838383961 +64656331363738616539346663613261386639353437316231636533353031336464383432623939 +65386164396231393735643563663337643563633233373338643630313739373861356166616463 +35306263333963663434376263396464323135346663376334356134393066653439376263376231 +30333730383163366636323533393334336331633234306536376634313735613263366537346536 +62366564383861656662353738366665396639313833323038356661306135393338333466333563 +32653861346166663163383036386432343833333137663462343030363762663139366534326466 +66313864623438336164333430613766373430656536323964633863333931643036656563353639 +30313835666366383035343031643265386263316165323537613636656533376239633964393866 +61646163343032313036303738643763383364663134356634373262633361383035306231636364 +39333232636538643033313438396332383962656131363365666566633239366532326336363133 +38393064643030333538333562643435663434343863383834663266373337336433313663646164 +36343334343965623830613736393231666361643239663062393239613233376335383362666161 +66383035653330373736613234303631386163656561383138613363613539396332376162316131 +61313532653531653836343731636535623066383231613635316432323331623761383833623333 +39343632623961613561373261653939636363366531303839336237383166363733303538363237 +36373362636263666334316163633766303334373033636539353464393536356466636664333665 +32643135626366666137626464393961366165383334343063356334373534633764326162363837 +38643662326266313464343464646166643235663663303761313639376537306337353863336264 +66376335333738366265343636376363366365306137336665623466626261653937656461303332 +32613561616662383032393562613831626666373134303032626134313262363830326530643632 +61366133663564313933366430396430353762386133396436633839303766653765 diff --git a/conda_environment_lumi.yml b/conda_environment_lumi.yml new file mode 100644 index 0000000..661ebc5 --- /dev/null +++ b/conda_environment_lumi.yml @@ -0,0 +1,36 @@ +# Assumes that conda, pip, build-essentials and cuda are installed +--- +name: ShallowWaterGPU_HPC +channels: +- conda-forge + +dependencies: +- python=3.9 +- numpy +- mpi4py +- six +- pytools +- netcdf4 +- scipy +- pip: + - hip-python + - hip-python-as-cuda + - -i https://test.pypi.org/simple/ + + +#On LUMI-G +#module load LUMI/23.03 +#module load lumi-container-wrapper +#ml cray-python/3.9.13.1 +#conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml +# export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" +# +# +# +# Install conda environment (one-time operation): +# $ conda env create -f conda_environment_hpc.yml +# Activate environment and install the following packages using pip: +# $ conda activate ShallowWaterGPU_HPC +# - pycuda: $ pip3 install --no-deps -U pycuda +# on Windows: make sure your visual studio c++ compiler is available in PATH +# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\ diff --git a/mpiTesting.py b/mpiTesting.py new file mode 100644 index 0000000..5869fdd --- /dev/null +++ b/mpiTesting.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +""" +This python module implements MPI simulations for benchmarking + +Copyright (C) 2018 SINTEF ICT + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + + +import numpy as np +import gc +import time +import json +import logging +import os + +#GPU-aware MPI +from os import environ +if environ.get("MPICH_GPU_SUPPORT_ENABLED", False): + from ctypes import CDLL, RTLD_GLOBAL + CDLL(f"{environ.get('CRAY_MPICH_ROOTDIR')}/gtl/lib/libmpi_gtl_hsa.so", mode=RTLD_GLOBAL) + +# MPI +from mpi4py import MPI + +# CUDA +#import pycuda.driver as cuda +from hip import hip + +# Simulator engine etc +from GPUSimulators import MPISimulator, Common, CudaContext +from GPUSimulators import EE2D_KP07_dimsplit +from GPUSimulators.helpers import InitialConditions as IC +from GPUSimulators.Simulator import BoundaryCondition as BC + +import argparse +parser = argparse.ArgumentParser(description='Strong and weak scaling experiments.') +parser.add_argument('-nx', type=int, default=128) +parser.add_argument('-ny', type=int, default=128) +parser.add_argument('--profile', action='store_true') # default: False + +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + +args = parser.parse_args() + +if(args.profile): + profiling_data = {} + # profiling: total run time + t_total_start = time.time() + t_init_start = time.time() + + +# Get MPI COMM to use +comm = MPI.COMM_WORLD + + +#### +# Initialize logging +#### +log_level_console = 20 +log_level_file = 10 +log_filename = 'mpi_' + str(comm.rank) + '.log' +logger = logging.getLogger('GPUSimulators') +logger.setLevel(min(log_level_console, log_level_file)) + +ch = logging.StreamHandler() +ch.setLevel(log_level_console) +logger.addHandler(ch) +logger.info("Console logger using level %s", + logging.getLevelName(log_level_console)) + +fh = logging.FileHandler(log_filename) +formatter = logging.Formatter( + '%(asctime)s:%(name)s:%(levelname)s: %(message)s') +fh.setFormatter(formatter) +fh.setLevel(log_level_file) +logger.addHandler(fh) +logger.info("File logger using level %s to %s", + logging.getLevelName(log_level_file), log_filename) + + +#### +# Initialize MPI grid etc +#### +logger.info("Creating MPI grid") +grid = MPISimulator.MPIGrid(MPI.COMM_WORLD) + + +#### +# Initialize CUDA +#### +#cuda.init(flags=0) +#logger.info("Initializing CUDA") +local_rank = grid.getLocalRank() +#num_cuda_devices = cuda.Device.count() +num_cuda_devices = hip_check(hip.hipGetDeviceCount()) +cuda_device = local_rank % num_cuda_devices +logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device)) +cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False) + + +#### +# Set initial conditions +#### + +# DEBUGGING - setting random seed +np.random.seed(42) + +logger.info("Generating initial conditions") +nx = args.nx +ny = args.ny + +dt = 0.000001 + +gamma = 1.4 +#save_times = np.linspace(0, 0.000009, 2) +#save_times = np.linspace(0, 0.000099, 11) +#save_times = np.linspace(0, 0.000099, 2) +save_times = np.linspace(0, 0.0000999, 2) +outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc" +save_var_names = ['rho', 'rho_u', 'rho_v', 'E'] + +arguments = IC.genKelvinHelmholtz(nx, ny, gamma, grid=grid) +arguments['context'] = cuda_context +arguments['theta'] = 1.2 +arguments['grid'] = grid + +if(args.profile): + t_init_end = time.time() + t_init = t_init_end - t_init_start + profiling_data["t_init"] = t_init + +#### +# Run simulation +#### +logger.info("Running simulation") +# Helper function to create MPI simulator + + +def genSim(grid, **kwargs): + local_sim = EE2D_KP07_dimsplit.EE2D_KP07_dimsplit(**kwargs) + sim = MPISimulator.MPISimulator(local_sim, grid) + return sim + + +outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation( + genSim, arguments, outfile, save_times, save_var_names, dt) + +if(args.profile): + t_total_end = time.time() + t_total = t_total_end - t_total_start + profiling_data["t_total"] = t_total + print("Total run time on rank " + str(MPI.COMM_WORLD.rank) + " is " + str(t_total) + " s") + +# write profiling to json file +if(args.profile and MPI.COMM_WORLD.rank == 0): + job_id = "" + if "SLURM_JOB_ID" in os.environ: + job_id = int(os.environ["SLURM_JOB_ID"]) + allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"]) + allocated_gpus = int(os.environ["HIP_VISIBLE_DEVICES"].count(",") + 1) + # allocated_gpus = int(os.environ["CUDA_VISIBLE_DEVICES"].count(",") + 1) + profiling_file = "MPI_jobid_" + \ + str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json" + profiling_data["outfile"] = outfile + else: + profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json" + + for stage in sim_runner_profiling_data["start"].keys(): + profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage] + + for stage in sim_profiling_data["start"].keys(): + profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage] + + profiling_data["nx"] = nx + profiling_data["ny"] = ny + profiling_data["dt"] = dt + profiling_data["n_time_steps"] = sim_profiling_data["n_time_steps"] + + profiling_data["slurm_job_id"] = job_id + profiling_data["n_cuda_devices"] = str(num_cuda_devices) + profiling_data["n_processes"] = str(MPI.COMM_WORLD.size) + profiling_data["git_hash"] = Common.getGitHash() + profiling_data["git_status"] = Common.getGitStatus() + + with open(profiling_file, "w") as write_file: + json.dump(profiling_data, write_file) + +#### +# Clean shutdown +#### +sim = None +local_sim = None +cuda_context = None +arguments = None +logging.shutdown() +gc.collect() + + + +#### +# Print completion and exit +#### +print("Completed!") +exit(0) diff --git a/reference/swashes_1_nx=1024.csv b/reference/swashes_1_nx=1024.csv new file mode 100644 index 0000000..bf7bda7 --- /dev/null +++ b/reference/swashes_1_nx=1024.csv @@ -0,0 +1,1042 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00976562 meters +# Number of cells: 1024 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00488281 0.005 0 0 0 0.005 0 0 +0.0146484 0.005 0 0 0 0.005 0 0 +0.0244141 0.005 0 0 0 0.005 0 0 +0.0341797 0.005 0 0 0 0.005 0 0 +0.0439453 0.005 0 0 0 0.005 0 0 +0.0537109 0.005 0 0 0 0.005 0 0 +0.0634766 0.005 0 0 0 0.005 0 0 +0.0732422 0.005 0 0 0 0.005 0 0 +0.0830078 0.005 0 0 0 0.005 0 0 +0.0927734 0.005 0 0 0 0.005 0 0 +0.102539 0.005 0 0 0 0.005 0 0 +0.112305 0.005 0 0 0 0.005 0 0 +0.12207 0.005 0 0 0 0.005 0 0 +0.131836 0.005 0 0 0 0.005 0 0 +0.141602 0.005 0 0 0 0.005 0 0 +0.151367 0.005 0 0 0 0.005 0 0 +0.161133 0.005 0 0 0 0.005 0 0 +0.170898 0.005 0 0 0 0.005 0 0 +0.180664 0.005 0 0 0 0.005 0 0 +0.19043 0.005 0 0 0 0.005 0 0 +0.200195 0.005 0 0 0 0.005 0 0 +0.209961 0.005 0 0 0 0.005 0 0 +0.219727 0.005 0 0 0 0.005 0 0 +0.229492 0.005 0 0 0 0.005 0 0 +0.239258 0.005 0 0 0 0.005 0 0 +0.249023 0.005 0 0 0 0.005 0 0 +0.258789 0.005 0 0 0 0.005 0 0 +0.268555 0.005 0 0 0 0.005 0 0 +0.27832 0.005 0 0 0 0.005 0 0 +0.288086 0.005 0 0 0 0.005 0 0 +0.297852 0.005 0 0 0 0.005 0 0 +0.307617 0.005 0 0 0 0.005 0 0 +0.317383 0.005 0 0 0 0.005 0 0 +0.327148 0.005 0 0 0 0.005 0 0 +0.336914 0.005 0 0 0 0.005 0 0 +0.34668 0.005 0 0 0 0.005 0 0 +0.356445 0.005 0 0 0 0.005 0 0 +0.366211 0.005 0 0 0 0.005 0 0 +0.375977 0.005 0 0 0 0.005 0 0 +0.385742 0.005 0 0 0 0.005 0 0 +0.395508 0.005 0 0 0 0.005 0 0 +0.405273 0.005 0 0 0 0.005 0 0 +0.415039 0.005 0 0 0 0.005 0 0 +0.424805 0.005 0 0 0 0.005 0 0 +0.43457 0.005 0 0 0 0.005 0 0 +0.444336 0.005 0 0 0 0.005 0 0 +0.454102 0.005 0 0 0 0.005 0 0 +0.463867 0.005 0 0 0 0.005 0 0 +0.473633 0.005 0 0 0 0.005 0 0 +0.483398 0.005 0 0 0 0.005 0 0 +0.493164 0.005 0 0 0 0.005 0 0 +0.50293 0.005 0 0 0 0.005 0 0 +0.512695 0.005 0 0 0 0.005 0 0 +0.522461 0.005 0 0 0 0.005 0 0 +0.532227 0.005 0 0 0 0.005 0 0 +0.541992 0.005 0 0 0 0.005 0 0 +0.551758 0.005 0 0 0 0.005 0 0 +0.561523 0.005 0 0 0 0.005 0 0 +0.571289 0.005 0 0 0 0.005 0 0 +0.581055 0.005 0 0 0 0.005 0 0 +0.59082 0.005 0 0 0 0.005 0 0 +0.600586 0.005 0 0 0 0.005 0 0 +0.610352 0.005 0 0 0 0.005 0 0 +0.620117 0.005 0 0 0 0.005 0 0 +0.629883 0.005 0 0 0 0.005 0 0 +0.639648 0.005 0 0 0 0.005 0 0 +0.649414 0.005 0 0 0 0.005 0 0 +0.65918 0.005 0 0 0 0.005 0 0 +0.668945 0.005 0 0 0 0.005 0 0 +0.678711 0.005 0 0 0 0.005 0 0 +0.688477 0.005 0 0 0 0.005 0 0 +0.698242 0.005 0 0 0 0.005 0 0 +0.708008 0.005 0 0 0 0.005 0 0 +0.717773 0.005 0 0 0 0.005 0 0 +0.727539 0.005 0 0 0 0.005 0 0 +0.737305 0.005 0 0 0 0.005 0 0 +0.74707 0.005 0 0 0 0.005 0 0 +0.756836 0.005 0 0 0 0.005 0 0 +0.766602 0.005 0 0 0 0.005 0 0 +0.776367 0.005 0 0 0 0.005 0 0 +0.786133 0.005 0 0 0 0.005 0 0 +0.795898 0.005 0 0 0 0.005 0 0 +0.805664 0.005 0 0 0 0.005 0 0 +0.81543 0.005 0 0 0 0.005 0 0 +0.825195 0.005 0 0 0 0.005 0 0 +0.834961 0.005 0 0 0 0.005 0 0 +0.844727 0.005 0 0 0 0.005 0 0 +0.854492 0.005 0 0 0 0.005 0 0 +0.864258 0.005 0 0 0 0.005 0 0 +0.874023 0.005 0 0 0 0.005 0 0 +0.883789 0.005 0 0 0 0.005 0 0 +0.893555 0.005 0 0 0 0.005 0 0 +0.90332 0.005 0 0 0 0.005 0 0 +0.913086 0.005 0 0 0 0.005 0 0 +0.922852 0.005 0 0 0 0.005 0 0 +0.932617 0.005 0 0 0 0.005 0 0 +0.942383 0.005 0 0 0 0.005 0 0 +0.952148 0.005 0 0 0 0.005 0 0 +0.961914 0.005 0 0 0 0.005 0 0 +0.97168 0.005 0 0 0 0.005 0 0 +0.981445 0.005 0 0 0 0.005 0 0 +0.991211 0.005 0 0 0 0.005 0 0 +1.00098 0.005 0 0 0 0.005 0 0 +1.01074 0.005 0 0 0 0.005 0 0 +1.02051 0.005 0 0 0 0.005 0 0 +1.03027 0.005 0 0 0 0.005 0 0 +1.04004 0.005 0 0 0 0.005 0 0 +1.0498 0.005 0 0 0 0.005 0 0 +1.05957 0.005 0 0 0 0.005 0 0 +1.06934 0.005 0 0 0 0.005 0 0 +1.0791 0.005 0 0 0 0.005 0 0 +1.08887 0.005 0 0 0 0.005 0 0 +1.09863 0.005 0 0 0 0.005 0 0 +1.1084 0.005 0 0 0 0.005 0 0 +1.11816 0.005 0 0 0 0.005 0 0 +1.12793 0.005 0 0 0 0.005 0 0 +1.1377 0.005 0 0 0 0.005 0 0 +1.14746 0.005 0 0 0 0.005 0 0 +1.15723 0.005 0 0 0 0.005 0 0 +1.16699 0.005 0 0 0 0.005 0 0 +1.17676 0.005 0 0 0 0.005 0 0 +1.18652 0.005 0 0 0 0.005 0 0 +1.19629 0.005 0 0 0 0.005 0 0 +1.20605 0.005 0 0 0 0.005 0 0 +1.21582 0.005 0 0 0 0.005 0 0 +1.22559 0.005 0 0 0 0.005 0 0 +1.23535 0.005 0 0 0 0.005 0 0 +1.24512 0.005 0 0 0 0.005 0 0 +1.25488 0.005 0 0 0 0.005 0 0 +1.26465 0.005 0 0 0 0.005 0 0 +1.27441 0.005 0 0 0 0.005 0 0 +1.28418 0.005 0 0 0 0.005 0 0 +1.29395 0.005 0 0 0 0.005 0 0 +1.30371 0.005 0 0 0 0.005 0 0 +1.31348 0.005 0 0 0 0.005 0 0 +1.32324 0.005 0 0 0 0.005 0 0 +1.33301 0.005 0 0 0 0.005 0 0 +1.34277 0.005 0 0 0 0.005 0 0 +1.35254 0.005 0 0 0 0.005 0 0 +1.3623 0.005 0 0 0 0.005 0 0 +1.37207 0.005 0 0 0 0.005 0 0 +1.38184 0.005 0 0 0 0.005 0 0 +1.3916 0.005 0 0 0 0.005 0 0 +1.40137 0.005 0 0 0 0.005 0 0 +1.41113 0.005 0 0 0 0.005 0 0 +1.4209 0.005 0 0 0 0.005 0 0 +1.43066 0.005 0 0 0 0.005 0 0 +1.44043 0.005 0 0 0 0.005 0 0 +1.4502 0.005 0 0 0 0.005 0 0 +1.45996 0.005 0 0 0 0.005 0 0 +1.46973 0.005 0 0 0 0.005 0 0 +1.47949 0.005 0 0 0 0.005 0 0 +1.48926 0.005 0 0 0 0.005 0 0 +1.49902 0.005 0 0 0 0.005 0 0 +1.50879 0.005 0 0 0 0.005 0 0 +1.51855 0.005 0 0 0 0.005 0 0 +1.52832 0.005 0 0 0 0.005 0 0 +1.53809 0.005 0 0 0 0.005 0 0 +1.54785 0.005 0 0 0 0.005 0 0 +1.55762 0.005 0 0 0 0.005 0 0 +1.56738 0.005 0 0 0 0.005 0 0 +1.57715 0.005 0 0 0 0.005 0 0 +1.58691 0.005 0 0 0 0.005 0 0 +1.59668 0.005 0 0 0 0.005 0 0 +1.60645 0.005 0 0 0 0.005 0 0 +1.61621 0.005 0 0 0 0.005 0 0 +1.62598 0.005 0 0 0 0.005 0 0 +1.63574 0.005 0 0 0 0.005 0 0 +1.64551 0.005 0 0 0 0.005 0 0 +1.65527 0.005 0 0 0 0.005 0 0 +1.66504 0.005 0 0 0 0.005 0 0 +1.6748 0.005 0 0 0 0.005 0 0 +1.68457 0.005 0 0 0 0.005 0 0 +1.69434 0.005 0 0 0 0.005 0 0 +1.7041 0.005 0 0 0 0.005 0 0 +1.71387 0.005 0 0 0 0.005 0 0 +1.72363 0.005 0 0 0 0.005 0 0 +1.7334 0.005 0 0 0 0.005 0 0 +1.74316 0.005 0 0 0 0.005 0 0 +1.75293 0.005 0 0 0 0.005 0 0 +1.7627 0.005 0 0 0 0.005 0 0 +1.77246 0.005 0 0 0 0.005 0 0 +1.78223 0.005 0 0 0 0.005 0 0 +1.79199 0.005 0 0 0 0.005 0 0 +1.80176 0.005 0 0 0 0.005 0 0 +1.81152 0.005 0 0 0 0.005 0 0 +1.82129 0.005 0 0 0 0.005 0 0 +1.83105 0.005 0 0 0 0.005 0 0 +1.84082 0.005 0 0 0 0.005 0 0 +1.85059 0.005 0 0 0 0.005 0 0 +1.86035 0.005 0 0 0 0.005 0 0 +1.87012 0.005 0 0 0 0.005 0 0 +1.87988 0.005 0 0 0 0.005 0 0 +1.88965 0.005 0 0 0 0.005 0 0 +1.89941 0.005 0 0 0 0.005 0 0 +1.90918 0.005 0 0 0 0.005 0 0 +1.91895 0.005 0 0 0 0.005 0 0 +1.92871 0.005 0 0 0 0.005 0 0 +1.93848 0.005 0 0 0 0.005 0 0 +1.94824 0.005 0 0 0 0.005 0 0 +1.95801 0.005 0 0 0 0.005 0 0 +1.96777 0.005 0 0 0 0.005 0 0 +1.97754 0.005 0 0 0 0.005 0 0 +1.9873 0.005 0 0 0 0.005 0 0 +1.99707 0.005 0 0 0 0.005 0 0 +2.00684 0.005 0 0 0 0.005 0 0 +2.0166 0.005 0 0 0 0.005 0 0 +2.02637 0.005 0 0 0 0.005 0 0 +2.03613 0.005 0 0 0 0.005 0 0 +2.0459 0.005 0 0 0 0.005 0 0 +2.05566 0.005 0 0 0 0.005 0 0 +2.06543 0.005 0 0 0 0.005 0 0 +2.0752 0.005 0 0 0 0.005 0 0 +2.08496 0.005 0 0 0 0.005 0 0 +2.09473 0.005 0 0 0 0.005 0 0 +2.10449 0.005 0 0 0 0.005 0 0 +2.11426 0.005 0 0 0 0.005 0 0 +2.12402 0.005 0 0 0 0.005 0 0 +2.13379 0.005 0 0 0 0.005 0 0 +2.14355 0.005 0 0 0 0.005 0 0 +2.15332 0.005 0 0 0 0.005 0 0 +2.16309 0.005 0 0 0 0.005 0 0 +2.17285 0.005 0 0 0 0.005 0 0 +2.18262 0.005 0 0 0 0.005 0 0 +2.19238 0.005 0 0 0 0.005 0 0 +2.20215 0.005 0 0 0 0.005 0 0 +2.21191 0.005 0 0 0 0.005 0 0 +2.22168 0.005 0 0 0 0.005 0 0 +2.23145 0.005 0 0 0 0.005 0 0 +2.24121 0.005 0 0 0 0.005 0 0 +2.25098 0.005 0 0 0 0.005 0 0 +2.26074 0.005 0 0 0 0.005 0 0 +2.27051 0.005 0 0 0 0.005 0 0 +2.28027 0.005 0 0 0 0.005 0 0 +2.29004 0.005 0 0 0 0.005 0 0 +2.2998 0.005 0 0 0 0.005 0 0 +2.30957 0.005 0 0 0 0.005 0 0 +2.31934 0.005 0 0 0 0.005 0 0 +2.3291 0.005 0 0 0 0.005 0 0 +2.33887 0.005 0 0 0 0.005 0 0 +2.34863 0.005 0 0 0 0.005 0 0 +2.3584 0.005 0 0 0 0.005 0 0 +2.36816 0.005 0 0 0 0.005 0 0 +2.37793 0.005 0 0 0 0.005 0 0 +2.3877 0.005 0 0 0 0.005 0 0 +2.39746 0.005 0 0 0 0.005 0 0 +2.40723 0.005 0 0 0 0.005 0 0 +2.41699 0.005 0 0 0 0.005 0 0 +2.42676 0.005 0 0 0 0.005 0 0 +2.43652 0.005 0 0 0 0.005 0 0 +2.44629 0.005 0 0 0 0.005 0 0 +2.45605 0.005 0 0 0 0.005 0 0 +2.46582 0.005 0 0 0 0.005 0 0 +2.47559 0.005 0 0 0 0.005 0 0 +2.48535 0.005 0 0 0 0.005 0 0 +2.49512 0.005 0 0 0 0.005 0 0 +2.50488 0.005 0 0 0 0.005 0 0 +2.51465 0.005 0 0 0 0.005 0 0 +2.52441 0.005 0 0 0 0.005 0 0 +2.53418 0.005 0 0 0 0.005 0 0 +2.54395 0.005 0 0 0 0.005 0 0 +2.55371 0.005 0 0 0 0.005 0 0 +2.56348 0.005 0 0 0 0.005 0 0 +2.57324 0.005 0 0 0 0.005 0 0 +2.58301 0.005 0 0 0 0.005 0 0 +2.59277 0.005 0 0 0 0.005 0 0 +2.60254 0.005 0 0 0 0.005 0 0 +2.6123 0.005 0 0 0 0.005 0 0 +2.62207 0.005 0 0 0 0.005 0 0 +2.63184 0.005 0 0 0 0.005 0 0 +2.6416 0.005 0 0 0 0.005 0 0 +2.65137 0.005 0 0 0 0.005 0 0 +2.66113 0.005 0 0 0 0.005 0 0 +2.6709 0.005 0 0 0 0.005 0 0 +2.68066 0.005 0 0 0 0.005 0 0 +2.69043 0.005 0 0 0 0.005 0 0 +2.7002 0.005 0 0 0 0.005 0 0 +2.70996 0.005 0 0 0 0.005 0 0 +2.71973 0.005 0 0 0 0.005 0 0 +2.72949 0.005 0 0 0 0.005 0 0 +2.73926 0.005 0 0 0 0.005 0 0 +2.74902 0.005 0 0 0 0.005 0 0 +2.75879 0.005 0 0 0 0.005 0 0 +2.76855 0.005 0 0 0 0.005 0 0 +2.77832 0.005 0 0 0 0.005 0 0 +2.78809 0.005 0 0 0 0.005 0 0 +2.79785 0.005 0 0 0 0.005 0 0 +2.80762 0.005 0 0 0 0.005 0 0 +2.81738 0.005 0 0 0 0.005 0 0 +2.82715 0.005 0 0 0 0.005 0 0 +2.83691 0.005 0 0 0 0.005 0 0 +2.84668 0.005 0 0 0 0.005 0 0 +2.85645 0.005 0 0 0 0.005 0 0 +2.86621 0.005 0 0 0 0.005 0 0 +2.87598 0.005 0 0 0 0.005 0 0 +2.88574 0.005 0 0 0 0.005 0 0 +2.89551 0.005 0 0 0 0.005 0 0 +2.90527 0.005 0 0 0 0.005 0 0 +2.91504 0.005 0 0 0 0.005 0 0 +2.9248 0.005 0 0 0 0.005 0 0 +2.93457 0.005 0 0 0 0.005 0 0 +2.94434 0.005 0 0 0 0.005 0 0 +2.9541 0.005 0 0 0 0.005 0 0 +2.96387 0.005 0 0 0 0.005 0 0 +2.97363 0.005 0 0 0 0.005 0 0 +2.9834 0.005 0 0 0 0.005 0 0 +2.99316 0.005 0 0 0 0.005 0 0 +3.00293 0.005 0 0 0 0.005 0 0 +3.0127 0.005 0 0 0 0.005 0 0 +3.02246 0.005 0 0 0 0.005 0 0 +3.03223 0.005 0 0 0 0.005 0 0 +3.04199 0.005 0 0 0 0.005 0 0 +3.05176 0.005 0 0 0 0.005 0 0 +3.06152 0.005 0 0 0 0.005 0 0 +3.07129 0.005 0 0 0 0.005 0 0 +3.08105 0.005 0 0 0 0.005 0 0 +3.09082 0.005 0 0 0 0.005 0 0 +3.10059 0.005 0 0 0 0.005 0 0 +3.11035 0.005 0 0 0 0.005 0 0 +3.12012 0.005 0 0 0 0.005 0 0 +3.12988 0.005 0 0 0 0.005 0 0 +3.13965 0.005 0 0 0 0.005 0 0 +3.14941 0.005 0 0 0 0.005 0 0 +3.15918 0.005 0 0 0 0.005 0 0 +3.16895 0.005 0 0 0 0.005 0 0 +3.17871 0.005 0 0 0 0.005 0 0 +3.18848 0.005 0 0 0 0.005 0 0 +3.19824 0.005 0 0 0 0.005 0 0 +3.20801 0.005 0 0 0 0.005 0 0 +3.21777 0.005 0 0 0 0.005 0 0 +3.22754 0.005 0 0 0 0.005 0 0 +3.2373 0.005 0 0 0 0.005 0 0 +3.24707 0.005 0 0 0 0.005 0 0 +3.25684 0.005 0 0 0 0.005 0 0 +3.2666 0.005 0 0 0 0.005 0 0 +3.27637 0.005 0 0 0 0.005 0 0 +3.28613 0.005 0 0 0 0.005 0 0 +3.2959 0.005 0 0 0 0.005 0 0 +3.30566 0.005 0 0 0 0.005 0 0 +3.31543 0.005 0 0 0 0.005 0 0 +3.3252 0.005 0 0 0 0.005 0 0 +3.33496 0.005 0 0 0 0.005 0 0 +3.34473 0.005 0 0 0 0.005 0 0 +3.35449 0.005 0 0 0 0.005 0 0 +3.36426 0.005 0 0 0 0.005 0 0 +3.37402 0.005 0 0 0 0.005 0 0 +3.38379 0.005 0 0 0 0.005 0 0 +3.39355 0.005 0 0 0 0.005 0 0 +3.40332 0.005 0 0 0 0.005 0 0 +3.41309 0.005 0 0 0 0.005 0 0 +3.42285 0.005 0 0 0 0.005 0 0 +3.43262 0.005 0 0 0 0.005 0 0 +3.44238 0.005 0 0 0 0.005 0 0 +3.45215 0.005 0 0 0 0.005 0 0 +3.46191 0.005 0 0 0 0.005 0 0 +3.47168 0.005 0 0 0 0.005 0 0 +3.48145 0.005 0 0 0 0.005 0 0 +3.49121 0.005 0 0 0 0.005 0 0 +3.50098 0.005 0 0 0 0.005 0 0 +3.51074 0.005 0 0 0 0.005 0 0 +3.52051 0.005 0 0 0 0.005 0 0 +3.53027 0.005 0 0 0 0.005 0 0 +3.54004 0.005 0 0 0 0.005 0 0 +3.5498 0.005 0 0 0 0.005 0 0 +3.55957 0.005 0 0 0 0.005 0 0 +3.56934 0.005 0 0 0 0.005 0 0 +3.5791 0.005 0 0 0 0.005 0 0 +3.58887 0.005 0 0 0 0.005 0 0 +3.59863 0.005 0 0 0 0.005 0 0 +3.6084 0.005 0 0 0 0.005 0 0 +3.61816 0.005 0 0 0 0.005 0 0 +3.62793 0.005 0 0 0 0.005 0 0 +3.6377 0.005 0 0 0 0.005 0 0 +3.64746 0.005 0 0 0 0.005 0 0 +3.65723 0.005 0 0 0 0.005 0 0 +3.66699 0.005 0 0 0 0.005 0 0 +3.67676 0.00498598 0.000621321 0 3.0979e-006 0.00498598 0.00280935 9.92708e-005 +3.68652 0.00496155 0.00170639 0 8.46634e-006 0.00496155 0.00773455 0.000194047 +3.69629 0.00493718 0.00279146 0 1.37819e-005 0.00493718 0.012684 0.000268524 +3.70605 0.00491287 0.00387653 0 1.90449e-005 0.00491287 0.017658 0.000333142 +3.71582 0.00488861 0.0049616 0 2.42553e-005 0.00488861 0.0226566 0.000391425 +3.72559 0.00486442 0.00604667 0 2.94135e-005 0.00486442 0.02768 0.000445118 +3.73535 0.00484029 0.00713174 0 3.45197e-005 0.00484029 0.0327284 0.000495246 +3.74512 0.00481622 0.00821681 0 3.95739e-005 0.00481622 0.0378021 0.000542479 +3.75488 0.0047922 0.00930188 0 4.45765e-005 0.0047922 0.0429011 0.000587283 +3.76465 0.00476825 0.0103869 0 4.95276e-005 0.00476825 0.0480257 0.000630002 +3.77441 0.00474436 0.011472 0 5.44274e-005 0.00474436 0.0531761 0.000670896 +3.78418 0.00472053 0.0125571 0 5.92761e-005 0.00472053 0.0583524 0.000710171 +3.79395 0.00469676 0.0136422 0 6.40739e-005 0.00469676 0.063555 0.000747993 +3.80371 0.00467304 0.0147272 0 6.88209e-005 0.00467304 0.0687839 0.000784496 +3.81348 0.00464939 0.0158123 0 7.35175e-005 0.00464939 0.0740393 0.000819793 +3.82324 0.0046258 0.0168974 0 7.81638e-005 0.0046258 0.0793215 0.000853979 +3.83301 0.00460227 0.0179824 0 8.27599e-005 0.00460227 0.0846307 0.000887136 +3.84277 0.00457879 0.0190675 0 8.73062e-005 0.00457879 0.0899671 0.000919335 +3.85254 0.00455538 0.0201526 0 9.18027e-005 0.00455538 0.0953309 0.000950635 +3.8623 0.00453203 0.0212376 0 9.62496e-005 0.00453203 0.100722 0.000981092 +3.87207 0.00450874 0.0223227 0 0.000100647 0.00450874 0.106141 0.00101075 +3.88184 0.00448551 0.0234078 0 0.000104996 0.00448551 0.111589 0.00103966 +3.8916 0.00446233 0.0244928 0 0.000109295 0.00446233 0.117064 0.00106785 +3.90137 0.00443922 0.0255779 0 0.000113546 0.00443922 0.122568 0.00109536 +3.91113 0.00441617 0.026663 0 0.000117748 0.00441617 0.128101 0.00112223 +3.9209 0.00439318 0.0277481 0 0.000121902 0.00439318 0.133662 0.00114847 +3.93066 0.00437024 0.0288331 0 0.000126008 0.00437024 0.139253 0.00117411 +3.94043 0.00434737 0.0299182 0 0.000130066 0.00434737 0.144873 0.00119918 +3.9502 0.00432456 0.0310033 0 0.000134075 0.00432456 0.150523 0.0012237 +3.95996 0.00430181 0.0320883 0 0.000138038 0.00430181 0.156202 0.0012477 +3.96973 0.00427912 0.0331734 0 0.000141953 0.00427912 0.161912 0.00127118 +3.97949 0.00425648 0.0342585 0 0.000145821 0.00425648 0.167652 0.00129417 +3.98926 0.00423391 0.0353435 0 0.000149641 0.00423391 0.173422 0.00131667 +3.99902 0.0042114 0.0364286 0 0.000153415 0.0042114 0.179223 0.00133872 +4.00879 0.00418895 0.0375137 0 0.000157143 0.00418895 0.185056 0.00136032 +4.01855 0.00416656 0.0385988 0 0.000160824 0.00416656 0.190919 0.00138148 +4.02832 0.00414422 0.0396838 0 0.000164459 0.00414422 0.196815 0.00140222 +4.03809 0.00412195 0.0407689 0 0.000168047 0.00412195 0.202742 0.00142254 +4.04785 0.00409974 0.041854 0 0.00017159 0.00409974 0.208701 0.00144247 +4.05762 0.00407759 0.042939 0 0.000175088 0.00407759 0.214692 0.001462 +4.06738 0.0040555 0.0440241 0 0.00017854 0.0040555 0.220716 0.00148115 +4.07715 0.00403346 0.0451092 0 0.000181946 0.00403346 0.226773 0.00149993 +4.08691 0.00401149 0.0461942 0 0.000185308 0.00401149 0.232863 0.00151835 +4.09668 0.00398958 0.0472793 0 0.000188625 0.00398958 0.238986 0.00153642 +4.10645 0.00396773 0.0483644 0 0.000191897 0.00396773 0.245143 0.00155413 +4.11621 0.00394594 0.0494494 0 0.000195124 0.00394594 0.251334 0.00157151 +4.12598 0.0039242 0.0505345 0 0.000198308 0.0039242 0.25756 0.00158856 +4.13574 0.00390253 0.0516196 0 0.000201447 0.00390253 0.263819 0.00160528 +4.14551 0.00388092 0.0527047 0 0.000204543 0.00388092 0.270114 0.00162168 +4.15527 0.00385937 0.0537897 0 0.000207594 0.00385937 0.276444 0.00163777 +4.16504 0.00383788 0.0548748 0 0.000210603 0.00383788 0.282809 0.00165356 +4.1748 0.00381644 0.0559599 0 0.000213568 0.00381644 0.28921 0.00166904 +4.18457 0.00379507 0.0570449 0 0.00021649 0.00379507 0.295646 0.00168423 +4.19434 0.00377376 0.05813 0 0.000219369 0.00377376 0.302119 0.00169913 +4.2041 0.00375251 0.0592151 0 0.000222205 0.00375251 0.308629 0.00171375 +4.21387 0.00373132 0.0603001 0 0.000224999 0.00373132 0.315176 0.00172808 +4.22363 0.00371018 0.0613852 0 0.00022775 0.00371018 0.32176 0.00174214 +4.2334 0.00368911 0.0624703 0 0.00023046 0.00368911 0.328381 0.00175593 +4.24316 0.0036681 0.0635553 0 0.000233127 0.0036681 0.33504 0.00176945 +4.25293 0.00364715 0.0646404 0 0.000235753 0.00364715 0.341738 0.00178272 +4.2627 0.00362626 0.0657255 0 0.000238338 0.00362626 0.348474 0.00179572 +4.27246 0.00360543 0.0668106 0 0.00024088 0.00360543 0.355249 0.00180847 +4.28223 0.00358465 0.0678956 0 0.000243382 0.00358465 0.362063 0.00182097 +4.29199 0.00356394 0.0689807 0 0.000245843 0.00356394 0.368916 0.00183322 +4.30176 0.00354329 0.0700658 0 0.000248263 0.00354329 0.37581 0.00184524 +4.31152 0.0035227 0.0711508 0 0.000250643 0.0035227 0.382743 0.00185701 +4.32129 0.00350217 0.0722359 0 0.000252982 0.00350217 0.389718 0.00186855 +4.33105 0.00348169 0.073321 0 0.000255281 0.00348169 0.396733 0.00187985 +4.34082 0.00346128 0.074406 0 0.00025754 0.00346128 0.40379 0.00189092 +4.35059 0.00344093 0.0754911 0 0.00025976 0.00344093 0.410888 0.00190177 +4.36035 0.00342064 0.0765762 0 0.000261939 0.00342064 0.418028 0.0019124 +4.37012 0.00340041 0.0776613 0 0.00026408 0.00340041 0.425211 0.0019228 +4.37988 0.00338024 0.0787463 0 0.000266181 0.00338024 0.432436 0.00193299 +4.38965 0.00336012 0.0798314 0 0.000268243 0.00336012 0.439705 0.00194296 +4.39941 0.00334007 0.0809165 0 0.000270267 0.00334007 0.447017 0.00195271 +4.40918 0.00332008 0.0820015 0 0.000272252 0.00332008 0.454374 0.00196226 +4.41895 0.00330015 0.0830866 0 0.000274198 0.00330015 0.461774 0.00197161 +4.42871 0.00328028 0.0841717 0 0.000276106 0.00328028 0.46922 0.00198074 +4.43848 0.00326047 0.0852567 0 0.000277977 0.00326047 0.47671 0.00198968 +4.44824 0.00324071 0.0863418 0 0.000279809 0.00324071 0.484246 0.00199841 +4.45801 0.00322102 0.0874269 0 0.000281604 0.00322102 0.491828 0.00200695 +4.46777 0.00320139 0.0885119 0 0.000283361 0.00320139 0.499457 0.00201529 +4.47754 0.00318182 0.089597 0 0.000285081 0.00318182 0.507132 0.00202344 +4.4873 0.00316231 0.0906821 0 0.000286765 0.00316231 0.514855 0.00203139 +4.49707 0.00314286 0.0917672 0 0.000288411 0.00314286 0.522625 0.00203916 +4.50684 0.00312346 0.0928522 0 0.000290021 0.00312346 0.530444 0.00204674 +4.5166 0.00310413 0.0939373 0 0.000291594 0.00310413 0.538311 0.00205414 +4.52637 0.00308486 0.0950224 0 0.000293131 0.00308486 0.546227 0.00206135 +4.53613 0.00306565 0.0961074 0 0.000294632 0.00306565 0.554193 0.00206838 +4.5459 0.0030465 0.0971925 0 0.000296097 0.0030465 0.562209 0.00207523 +4.55566 0.00302741 0.0982776 0 0.000297526 0.00302741 0.570275 0.0020819 +4.56543 0.00300837 0.0993626 0 0.00029892 0.00300837 0.578392 0.0020884 +4.5752 0.0029894 0.100448 0 0.000300279 0.0029894 0.586561 0.00209472 +4.58496 0.00297049 0.101533 0 0.000301602 0.00297049 0.594782 0.00210087 +4.59473 0.00295164 0.102618 0 0.000302891 0.00295164 0.603055 0.00210685 +4.60449 0.00293285 0.103703 0 0.000304145 0.00293285 0.611381 0.00211267 +4.61426 0.00291412 0.104788 0 0.000305364 0.00291412 0.61976 0.00211831 +4.62402 0.00289545 0.105873 0 0.00030655 0.00289545 0.628193 0.00212379 +4.63379 0.00287683 0.106958 0 0.000307701 0.00287683 0.636681 0.0021291 +4.64355 0.00285828 0.108043 0 0.000308818 0.00285828 0.645224 0.00213425 +4.65332 0.00283979 0.109128 0 0.000309901 0.00283979 0.653822 0.00213924 +4.66309 0.00282136 0.110213 0 0.000310951 0.00282136 0.662476 0.00214407 +4.67285 0.00280299 0.111298 0 0.000311968 0.00280299 0.671187 0.00214874 +4.68262 0.00278468 0.112383 0 0.000312952 0.00278468 0.679956 0.00215325 +4.69238 0.00276643 0.113469 0 0.000313902 0.00276643 0.688782 0.00215761 +4.70215 0.00274823 0.114554 0 0.00031482 0.00274823 0.697666 0.00216182 +4.71191 0.0027301 0.115639 0 0.000315705 0.0027301 0.706609 0.00216587 +4.72168 0.00271203 0.116724 0 0.000316558 0.00271203 0.715612 0.00216977 +4.73145 0.00269402 0.117809 0 0.000317379 0.00269402 0.724674 0.00217352 +4.74121 0.00267607 0.118894 0 0.000318168 0.00267607 0.733798 0.00217712 +4.75098 0.00265818 0.119979 0 0.000318925 0.00265818 0.742983 0.00218057 +4.76074 0.00264035 0.121064 0 0.000319651 0.00264035 0.752229 0.00218387 +4.77051 0.00262257 0.122149 0 0.000320345 0.00262257 0.761539 0.00218703 +4.78027 0.00260486 0.123234 0 0.000321008 0.00260486 0.770911 0.00219005 +4.79004 0.00258721 0.124319 0 0.00032164 0.00258721 0.780347 0.00219293 +4.7998 0.00256962 0.125404 0 0.000322241 0.00256962 0.789848 0.00219566 +4.80957 0.00255209 0.126489 0 0.000322812 0.00255209 0.799414 0.00219825 +4.81934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01465 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02441 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04395 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05371 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06348 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07324 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08301 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09277 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10254 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13184 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15137 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16113 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18066 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19043 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20996 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21973 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22949 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23926 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24902 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25879 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26855 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28809 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29785 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30762 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31738 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32715 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33691 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36621 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37598 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38574 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39551 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40527 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41504 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43457 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44434 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46387 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47363 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49316 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52246 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53223 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54199 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55176 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56152 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57129 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58105 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60059 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61035 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62012 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62988 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63965 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64941 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65918 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67871 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68848 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69824 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70801 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71777 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72754 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75684 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77637 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78613 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80566 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83496 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84473 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85449 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86426 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87402 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88379 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90332 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91309 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92285 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93262 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94238 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95215 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96191 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97168 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99121 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00098 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01074 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02051 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03027 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04004 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06934 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08887 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09863 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11816 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12793 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14746 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15723 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16699 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17676 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18652 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19629 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21582 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22559 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23535 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24512 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25488 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26465 0.001 0 0 0 0.001 0 0 +6.27441 0.001 0 0 0 0.001 0 0 +6.28418 0.001 0 0 0 0.001 0 0 +6.29395 0.001 0 0 0 0.001 0 0 +6.30371 0.001 0 0 0 0.001 0 0 +6.31348 0.001 0 0 0 0.001 0 0 +6.32324 0.001 0 0 0 0.001 0 0 +6.33301 0.001 0 0 0 0.001 0 0 +6.34277 0.001 0 0 0 0.001 0 0 +6.35254 0.001 0 0 0 0.001 0 0 +6.3623 0.001 0 0 0 0.001 0 0 +6.37207 0.001 0 0 0 0.001 0 0 +6.38184 0.001 0 0 0 0.001 0 0 +6.3916 0.001 0 0 0 0.001 0 0 +6.40137 0.001 0 0 0 0.001 0 0 +6.41113 0.001 0 0 0 0.001 0 0 +6.4209 0.001 0 0 0 0.001 0 0 +6.43066 0.001 0 0 0 0.001 0 0 +6.44043 0.001 0 0 0 0.001 0 0 +6.4502 0.001 0 0 0 0.001 0 0 +6.45996 0.001 0 0 0 0.001 0 0 +6.46973 0.001 0 0 0 0.001 0 0 +6.47949 0.001 0 0 0 0.001 0 0 +6.48926 0.001 0 0 0 0.001 0 0 +6.49902 0.001 0 0 0 0.001 0 0 +6.50879 0.001 0 0 0 0.001 0 0 +6.51855 0.001 0 0 0 0.001 0 0 +6.52832 0.001 0 0 0 0.001 0 0 +6.53809 0.001 0 0 0 0.001 0 0 +6.54785 0.001 0 0 0 0.001 0 0 +6.55762 0.001 0 0 0 0.001 0 0 +6.56738 0.001 0 0 0 0.001 0 0 +6.57715 0.001 0 0 0 0.001 0 0 +6.58691 0.001 0 0 0 0.001 0 0 +6.59668 0.001 0 0 0 0.001 0 0 +6.60645 0.001 0 0 0 0.001 0 0 +6.61621 0.001 0 0 0 0.001 0 0 +6.62598 0.001 0 0 0 0.001 0 0 +6.63574 0.001 0 0 0 0.001 0 0 +6.64551 0.001 0 0 0 0.001 0 0 +6.65527 0.001 0 0 0 0.001 0 0 +6.66504 0.001 0 0 0 0.001 0 0 +6.6748 0.001 0 0 0 0.001 0 0 +6.68457 0.001 0 0 0 0.001 0 0 +6.69434 0.001 0 0 0 0.001 0 0 +6.7041 0.001 0 0 0 0.001 0 0 +6.71387 0.001 0 0 0 0.001 0 0 +6.72363 0.001 0 0 0 0.001 0 0 +6.7334 0.001 0 0 0 0.001 0 0 +6.74316 0.001 0 0 0 0.001 0 0 +6.75293 0.001 0 0 0 0.001 0 0 +6.7627 0.001 0 0 0 0.001 0 0 +6.77246 0.001 0 0 0 0.001 0 0 +6.78223 0.001 0 0 0 0.001 0 0 +6.79199 0.001 0 0 0 0.001 0 0 +6.80176 0.001 0 0 0 0.001 0 0 +6.81152 0.001 0 0 0 0.001 0 0 +6.82129 0.001 0 0 0 0.001 0 0 +6.83105 0.001 0 0 0 0.001 0 0 +6.84082 0.001 0 0 0 0.001 0 0 +6.85059 0.001 0 0 0 0.001 0 0 +6.86035 0.001 0 0 0 0.001 0 0 +6.87012 0.001 0 0 0 0.001 0 0 +6.87988 0.001 0 0 0 0.001 0 0 +6.88965 0.001 0 0 0 0.001 0 0 +6.89941 0.001 0 0 0 0.001 0 0 +6.90918 0.001 0 0 0 0.001 0 0 +6.91895 0.001 0 0 0 0.001 0 0 +6.92871 0.001 0 0 0 0.001 0 0 +6.93848 0.001 0 0 0 0.001 0 0 +6.94824 0.001 0 0 0 0.001 0 0 +6.95801 0.001 0 0 0 0.001 0 0 +6.96777 0.001 0 0 0 0.001 0 0 +6.97754 0.001 0 0 0 0.001 0 0 +6.9873 0.001 0 0 0 0.001 0 0 +6.99707 0.001 0 0 0 0.001 0 0 +7.00684 0.001 0 0 0 0.001 0 0 +7.0166 0.001 0 0 0 0.001 0 0 +7.02637 0.001 0 0 0 0.001 0 0 +7.03613 0.001 0 0 0 0.001 0 0 +7.0459 0.001 0 0 0 0.001 0 0 +7.05566 0.001 0 0 0 0.001 0 0 +7.06543 0.001 0 0 0 0.001 0 0 +7.0752 0.001 0 0 0 0.001 0 0 +7.08496 0.001 0 0 0 0.001 0 0 +7.09473 0.001 0 0 0 0.001 0 0 +7.10449 0.001 0 0 0 0.001 0 0 +7.11426 0.001 0 0 0 0.001 0 0 +7.12402 0.001 0 0 0 0.001 0 0 +7.13379 0.001 0 0 0 0.001 0 0 +7.14355 0.001 0 0 0 0.001 0 0 +7.15332 0.001 0 0 0 0.001 0 0 +7.16309 0.001 0 0 0 0.001 0 0 +7.17285 0.001 0 0 0 0.001 0 0 +7.18262 0.001 0 0 0 0.001 0 0 +7.19238 0.001 0 0 0 0.001 0 0 +7.20215 0.001 0 0 0 0.001 0 0 +7.21191 0.001 0 0 0 0.001 0 0 +7.22168 0.001 0 0 0 0.001 0 0 +7.23145 0.001 0 0 0 0.001 0 0 +7.24121 0.001 0 0 0 0.001 0 0 +7.25098 0.001 0 0 0 0.001 0 0 +7.26074 0.001 0 0 0 0.001 0 0 +7.27051 0.001 0 0 0 0.001 0 0 +7.28027 0.001 0 0 0 0.001 0 0 +7.29004 0.001 0 0 0 0.001 0 0 +7.2998 0.001 0 0 0 0.001 0 0 +7.30957 0.001 0 0 0 0.001 0 0 +7.31934 0.001 0 0 0 0.001 0 0 +7.3291 0.001 0 0 0 0.001 0 0 +7.33887 0.001 0 0 0 0.001 0 0 +7.34863 0.001 0 0 0 0.001 0 0 +7.3584 0.001 0 0 0 0.001 0 0 +7.36816 0.001 0 0 0 0.001 0 0 +7.37793 0.001 0 0 0 0.001 0 0 +7.3877 0.001 0 0 0 0.001 0 0 +7.39746 0.001 0 0 0 0.001 0 0 +7.40723 0.001 0 0 0 0.001 0 0 +7.41699 0.001 0 0 0 0.001 0 0 +7.42676 0.001 0 0 0 0.001 0 0 +7.43652 0.001 0 0 0 0.001 0 0 +7.44629 0.001 0 0 0 0.001 0 0 +7.45605 0.001 0 0 0 0.001 0 0 +7.46582 0.001 0 0 0 0.001 0 0 +7.47559 0.001 0 0 0 0.001 0 0 +7.48535 0.001 0 0 0 0.001 0 0 +7.49512 0.001 0 0 0 0.001 0 0 +7.50488 0.001 0 0 0 0.001 0 0 +7.51465 0.001 0 0 0 0.001 0 0 +7.52441 0.001 0 0 0 0.001 0 0 +7.53418 0.001 0 0 0 0.001 0 0 +7.54395 0.001 0 0 0 0.001 0 0 +7.55371 0.001 0 0 0 0.001 0 0 +7.56348 0.001 0 0 0 0.001 0 0 +7.57324 0.001 0 0 0 0.001 0 0 +7.58301 0.001 0 0 0 0.001 0 0 +7.59277 0.001 0 0 0 0.001 0 0 +7.60254 0.001 0 0 0 0.001 0 0 +7.6123 0.001 0 0 0 0.001 0 0 +7.62207 0.001 0 0 0 0.001 0 0 +7.63184 0.001 0 0 0 0.001 0 0 +7.6416 0.001 0 0 0 0.001 0 0 +7.65137 0.001 0 0 0 0.001 0 0 +7.66113 0.001 0 0 0 0.001 0 0 +7.6709 0.001 0 0 0 0.001 0 0 +7.68066 0.001 0 0 0 0.001 0 0 +7.69043 0.001 0 0 0 0.001 0 0 +7.7002 0.001 0 0 0 0.001 0 0 +7.70996 0.001 0 0 0 0.001 0 0 +7.71973 0.001 0 0 0 0.001 0 0 +7.72949 0.001 0 0 0 0.001 0 0 +7.73926 0.001 0 0 0 0.001 0 0 +7.74902 0.001 0 0 0 0.001 0 0 +7.75879 0.001 0 0 0 0.001 0 0 +7.76855 0.001 0 0 0 0.001 0 0 +7.77832 0.001 0 0 0 0.001 0 0 +7.78809 0.001 0 0 0 0.001 0 0 +7.79785 0.001 0 0 0 0.001 0 0 +7.80762 0.001 0 0 0 0.001 0 0 +7.81738 0.001 0 0 0 0.001 0 0 +7.82715 0.001 0 0 0 0.001 0 0 +7.83691 0.001 0 0 0 0.001 0 0 +7.84668 0.001 0 0 0 0.001 0 0 +7.85645 0.001 0 0 0 0.001 0 0 +7.86621 0.001 0 0 0 0.001 0 0 +7.87598 0.001 0 0 0 0.001 0 0 +7.88574 0.001 0 0 0 0.001 0 0 +7.89551 0.001 0 0 0 0.001 0 0 +7.90527 0.001 0 0 0 0.001 0 0 +7.91504 0.001 0 0 0 0.001 0 0 +7.9248 0.001 0 0 0 0.001 0 0 +7.93457 0.001 0 0 0 0.001 0 0 +7.94434 0.001 0 0 0 0.001 0 0 +7.9541 0.001 0 0 0 0.001 0 0 +7.96387 0.001 0 0 0 0.001 0 0 +7.97363 0.001 0 0 0 0.001 0 0 +7.9834 0.001 0 0 0 0.001 0 0 +7.99316 0.001 0 0 0 0.001 0 0 +8.00293 0.001 0 0 0 0.001 0 0 +8.0127 0.001 0 0 0 0.001 0 0 +8.02246 0.001 0 0 0 0.001 0 0 +8.03223 0.001 0 0 0 0.001 0 0 +8.04199 0.001 0 0 0 0.001 0 0 +8.05176 0.001 0 0 0 0.001 0 0 +8.06152 0.001 0 0 0 0.001 0 0 +8.07129 0.001 0 0 0 0.001 0 0 +8.08105 0.001 0 0 0 0.001 0 0 +8.09082 0.001 0 0 0 0.001 0 0 +8.10059 0.001 0 0 0 0.001 0 0 +8.11035 0.001 0 0 0 0.001 0 0 +8.12012 0.001 0 0 0 0.001 0 0 +8.12988 0.001 0 0 0 0.001 0 0 +8.13965 0.001 0 0 0 0.001 0 0 +8.14941 0.001 0 0 0 0.001 0 0 +8.15918 0.001 0 0 0 0.001 0 0 +8.16895 0.001 0 0 0 0.001 0 0 +8.17871 0.001 0 0 0 0.001 0 0 +8.18848 0.001 0 0 0 0.001 0 0 +8.19824 0.001 0 0 0 0.001 0 0 +8.20801 0.001 0 0 0 0.001 0 0 +8.21777 0.001 0 0 0 0.001 0 0 +8.22754 0.001 0 0 0 0.001 0 0 +8.2373 0.001 0 0 0 0.001 0 0 +8.24707 0.001 0 0 0 0.001 0 0 +8.25684 0.001 0 0 0 0.001 0 0 +8.2666 0.001 0 0 0 0.001 0 0 +8.27637 0.001 0 0 0 0.001 0 0 +8.28613 0.001 0 0 0 0.001 0 0 +8.2959 0.001 0 0 0 0.001 0 0 +8.30566 0.001 0 0 0 0.001 0 0 +8.31543 0.001 0 0 0 0.001 0 0 +8.3252 0.001 0 0 0 0.001 0 0 +8.33496 0.001 0 0 0 0.001 0 0 +8.34473 0.001 0 0 0 0.001 0 0 +8.35449 0.001 0 0 0 0.001 0 0 +8.36426 0.001 0 0 0 0.001 0 0 +8.37402 0.001 0 0 0 0.001 0 0 +8.38379 0.001 0 0 0 0.001 0 0 +8.39355 0.001 0 0 0 0.001 0 0 +8.40332 0.001 0 0 0 0.001 0 0 +8.41309 0.001 0 0 0 0.001 0 0 +8.42285 0.001 0 0 0 0.001 0 0 +8.43262 0.001 0 0 0 0.001 0 0 +8.44238 0.001 0 0 0 0.001 0 0 +8.45215 0.001 0 0 0 0.001 0 0 +8.46191 0.001 0 0 0 0.001 0 0 +8.47168 0.001 0 0 0 0.001 0 0 +8.48145 0.001 0 0 0 0.001 0 0 +8.49121 0.001 0 0 0 0.001 0 0 +8.50098 0.001 0 0 0 0.001 0 0 +8.51074 0.001 0 0 0 0.001 0 0 +8.52051 0.001 0 0 0 0.001 0 0 +8.53027 0.001 0 0 0 0.001 0 0 +8.54004 0.001 0 0 0 0.001 0 0 +8.5498 0.001 0 0 0 0.001 0 0 +8.55957 0.001 0 0 0 0.001 0 0 +8.56934 0.001 0 0 0 0.001 0 0 +8.5791 0.001 0 0 0 0.001 0 0 +8.58887 0.001 0 0 0 0.001 0 0 +8.59863 0.001 0 0 0 0.001 0 0 +8.6084 0.001 0 0 0 0.001 0 0 +8.61816 0.001 0 0 0 0.001 0 0 +8.62793 0.001 0 0 0 0.001 0 0 +8.6377 0.001 0 0 0 0.001 0 0 +8.64746 0.001 0 0 0 0.001 0 0 +8.65723 0.001 0 0 0 0.001 0 0 +8.66699 0.001 0 0 0 0.001 0 0 +8.67676 0.001 0 0 0 0.001 0 0 +8.68652 0.001 0 0 0 0.001 0 0 +8.69629 0.001 0 0 0 0.001 0 0 +8.70605 0.001 0 0 0 0.001 0 0 +8.71582 0.001 0 0 0 0.001 0 0 +8.72559 0.001 0 0 0 0.001 0 0 +8.73535 0.001 0 0 0 0.001 0 0 +8.74512 0.001 0 0 0 0.001 0 0 +8.75488 0.001 0 0 0 0.001 0 0 +8.76465 0.001 0 0 0 0.001 0 0 +8.77441 0.001 0 0 0 0.001 0 0 +8.78418 0.001 0 0 0 0.001 0 0 +8.79395 0.001 0 0 0 0.001 0 0 +8.80371 0.001 0 0 0 0.001 0 0 +8.81348 0.001 0 0 0 0.001 0 0 +8.82324 0.001 0 0 0 0.001 0 0 +8.83301 0.001 0 0 0 0.001 0 0 +8.84277 0.001 0 0 0 0.001 0 0 +8.85254 0.001 0 0 0 0.001 0 0 +8.8623 0.001 0 0 0 0.001 0 0 +8.87207 0.001 0 0 0 0.001 0 0 +8.88184 0.001 0 0 0 0.001 0 0 +8.8916 0.001 0 0 0 0.001 0 0 +8.90137 0.001 0 0 0 0.001 0 0 +8.91113 0.001 0 0 0 0.001 0 0 +8.9209 0.001 0 0 0 0.001 0 0 +8.93066 0.001 0 0 0 0.001 0 0 +8.94043 0.001 0 0 0 0.001 0 0 +8.9502 0.001 0 0 0 0.001 0 0 +8.95996 0.001 0 0 0 0.001 0 0 +8.96973 0.001 0 0 0 0.001 0 0 +8.97949 0.001 0 0 0 0.001 0 0 +8.98926 0.001 0 0 0 0.001 0 0 +8.99902 0.001 0 0 0 0.001 0 0 +9.00879 0.001 0 0 0 0.001 0 0 +9.01855 0.001 0 0 0 0.001 0 0 +9.02832 0.001 0 0 0 0.001 0 0 +9.03809 0.001 0 0 0 0.001 0 0 +9.04785 0.001 0 0 0 0.001 0 0 +9.05762 0.001 0 0 0 0.001 0 0 +9.06738 0.001 0 0 0 0.001 0 0 +9.07715 0.001 0 0 0 0.001 0 0 +9.08691 0.001 0 0 0 0.001 0 0 +9.09668 0.001 0 0 0 0.001 0 0 +9.10645 0.001 0 0 0 0.001 0 0 +9.11621 0.001 0 0 0 0.001 0 0 +9.12598 0.001 0 0 0 0.001 0 0 +9.13574 0.001 0 0 0 0.001 0 0 +9.14551 0.001 0 0 0 0.001 0 0 +9.15527 0.001 0 0 0 0.001 0 0 +9.16504 0.001 0 0 0 0.001 0 0 +9.1748 0.001 0 0 0 0.001 0 0 +9.18457 0.001 0 0 0 0.001 0 0 +9.19434 0.001 0 0 0 0.001 0 0 +9.2041 0.001 0 0 0 0.001 0 0 +9.21387 0.001 0 0 0 0.001 0 0 +9.22363 0.001 0 0 0 0.001 0 0 +9.2334 0.001 0 0 0 0.001 0 0 +9.24316 0.001 0 0 0 0.001 0 0 +9.25293 0.001 0 0 0 0.001 0 0 +9.2627 0.001 0 0 0 0.001 0 0 +9.27246 0.001 0 0 0 0.001 0 0 +9.28223 0.001 0 0 0 0.001 0 0 +9.29199 0.001 0 0 0 0.001 0 0 +9.30176 0.001 0 0 0 0.001 0 0 +9.31152 0.001 0 0 0 0.001 0 0 +9.32129 0.001 0 0 0 0.001 0 0 +9.33105 0.001 0 0 0 0.001 0 0 +9.34082 0.001 0 0 0 0.001 0 0 +9.35059 0.001 0 0 0 0.001 0 0 +9.36035 0.001 0 0 0 0.001 0 0 +9.37012 0.001 0 0 0 0.001 0 0 +9.37988 0.001 0 0 0 0.001 0 0 +9.38965 0.001 0 0 0 0.001 0 0 +9.39941 0.001 0 0 0 0.001 0 0 +9.40918 0.001 0 0 0 0.001 0 0 +9.41895 0.001 0 0 0 0.001 0 0 +9.42871 0.001 0 0 0 0.001 0 0 +9.43848 0.001 0 0 0 0.001 0 0 +9.44824 0.001 0 0 0 0.001 0 0 +9.45801 0.001 0 0 0 0.001 0 0 +9.46777 0.001 0 0 0 0.001 0 0 +9.47754 0.001 0 0 0 0.001 0 0 +9.4873 0.001 0 0 0 0.001 0 0 +9.49707 0.001 0 0 0 0.001 0 0 +9.50684 0.001 0 0 0 0.001 0 0 +9.5166 0.001 0 0 0 0.001 0 0 +9.52637 0.001 0 0 0 0.001 0 0 +9.53613 0.001 0 0 0 0.001 0 0 +9.5459 0.001 0 0 0 0.001 0 0 +9.55566 0.001 0 0 0 0.001 0 0 +9.56543 0.001 0 0 0 0.001 0 0 +9.5752 0.001 0 0 0 0.001 0 0 +9.58496 0.001 0 0 0 0.001 0 0 +9.59473 0.001 0 0 0 0.001 0 0 +9.60449 0.001 0 0 0 0.001 0 0 +9.61426 0.001 0 0 0 0.001 0 0 +9.62402 0.001 0 0 0 0.001 0 0 +9.63379 0.001 0 0 0 0.001 0 0 +9.64355 0.001 0 0 0 0.001 0 0 +9.65332 0.001 0 0 0 0.001 0 0 +9.66309 0.001 0 0 0 0.001 0 0 +9.67285 0.001 0 0 0 0.001 0 0 +9.68262 0.001 0 0 0 0.001 0 0 +9.69238 0.001 0 0 0 0.001 0 0 +9.70215 0.001 0 0 0 0.001 0 0 +9.71191 0.001 0 0 0 0.001 0 0 +9.72168 0.001 0 0 0 0.001 0 0 +9.73145 0.001 0 0 0 0.001 0 0 +9.74121 0.001 0 0 0 0.001 0 0 +9.75098 0.001 0 0 0 0.001 0 0 +9.76074 0.001 0 0 0 0.001 0 0 +9.77051 0.001 0 0 0 0.001 0 0 +9.78027 0.001 0 0 0 0.001 0 0 +9.79004 0.001 0 0 0 0.001 0 0 +9.7998 0.001 0 0 0 0.001 0 0 +9.80957 0.001 0 0 0 0.001 0 0 +9.81934 0.001 0 0 0 0.001 0 0 +9.8291 0.001 0 0 0 0.001 0 0 +9.83887 0.001 0 0 0 0.001 0 0 +9.84863 0.001 0 0 0 0.001 0 0 +9.8584 0.001 0 0 0 0.001 0 0 +9.86816 0.001 0 0 0 0.001 0 0 +9.87793 0.001 0 0 0 0.001 0 0 +9.8877 0.001 0 0 0 0.001 0 0 +9.89746 0.001 0 0 0 0.001 0 0 +9.90723 0.001 0 0 0 0.001 0 0 +9.91699 0.001 0 0 0 0.001 0 0 +9.92676 0.001 0 0 0 0.001 0 0 +9.93652 0.001 0 0 0 0.001 0 0 +9.94629 0.001 0 0 0 0.001 0 0 +9.95605 0.001 0 0 0 0.001 0 0 +9.96582 0.001 0 0 0 0.001 0 0 +9.97559 0.001 0 0 0 0.001 0 0 +9.98535 0.001 0 0 0 0.001 0 0 +9.99512 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=128.csv b/reference/swashes_1_nx=128.csv new file mode 100644 index 0000000..b5357d4 --- /dev/null +++ b/reference/swashes_1_nx=128.csv @@ -0,0 +1,146 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.078125 meters +# Number of cells: 128 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0390625 0.005 0 0 0 0.005 0 0 +0.117188 0.005 0 0 0 0.005 0 0 +0.195312 0.005 0 0 0 0.005 0 0 +0.273438 0.005 0 0 0 0.005 0 0 +0.351562 0.005 0 0 0 0.005 0 0 +0.429688 0.005 0 0 0 0.005 0 0 +0.507812 0.005 0 0 0 0.005 0 0 +0.585938 0.005 0 0 0 0.005 0 0 +0.664062 0.005 0 0 0 0.005 0 0 +0.742188 0.005 0 0 0 0.005 0 0 +0.820312 0.005 0 0 0 0.005 0 0 +0.898438 0.005 0 0 0 0.005 0 0 +0.976562 0.005 0 0 0 0.005 0 0 +1.05469 0.005 0 0 0 0.005 0 0 +1.13281 0.005 0 0 0 0.005 0 0 +1.21094 0.005 0 0 0 0.005 0 0 +1.28906 0.005 0 0 0 0.005 0 0 +1.36719 0.005 0 0 0 0.005 0 0 +1.44531 0.005 0 0 0 0.005 0 0 +1.52344 0.005 0 0 0 0.005 0 0 +1.60156 0.005 0 0 0 0.005 0 0 +1.67969 0.005 0 0 0 0.005 0 0 +1.75781 0.005 0 0 0 0.005 0 0 +1.83594 0.005 0 0 0 0.005 0 0 +1.91406 0.005 0 0 0 0.005 0 0 +1.99219 0.005 0 0 0 0.005 0 0 +2.07031 0.005 0 0 0 0.005 0 0 +2.14844 0.005 0 0 0 0.005 0 0 +2.22656 0.005 0 0 0 0.005 0 0 +2.30469 0.005 0 0 0 0.005 0 0 +2.38281 0.005 0 0 0 0.005 0 0 +2.46094 0.005 0 0 0 0.005 0 0 +2.53906 0.005 0 0 0 0.005 0 0 +2.61719 0.005 0 0 0 0.005 0 0 +2.69531 0.005 0 0 0 0.005 0 0 +2.77344 0.005 0 0 0 0.005 0 0 +2.85156 0.005 0 0 0 0.005 0 0 +2.92969 0.005 0 0 0 0.005 0 0 +3.00781 0.005 0 0 0 0.005 0 0 +3.08594 0.005 0 0 0 0.005 0 0 +3.16406 0.005 0 0 0 0.005 0 0 +3.24219 0.005 0 0 0 0.005 0 0 +3.32031 0.005 0 0 0 0.005 0 0 +3.39844 0.005 0 0 0 0.005 0 0 +3.47656 0.005 0 0 0 0.005 0 0 +3.55469 0.005 0 0 0 0.005 0 0 +3.63281 0.005 0 0 0 0.005 0 0 +3.71094 0.00490073 0.00441906 0 2.16566e-005 0.00490073 0.0201542 0.000362943 +3.78906 0.00470863 0.0130996 0 6.16813e-005 0.00470863 0.0609504 0.000729255 +3.86719 0.00452038 0.0217802 0 9.84546e-005 0.00452038 0.103428 0.000996019 +3.94531 0.00433596 0.0304607 0 0.000132076 0.00433596 0.147694 0.00121151 +4.02344 0.00415538 0.0391413 0 0.000162647 0.00415538 0.193863 0.0013919 +4.10156 0.00397865 0.0478218 0 0.000190266 0.00397865 0.242061 0.00154532 +4.17969 0.00380575 0.0565024 0 0.000215034 0.00380575 0.292423 0.00167667 +4.25781 0.0036367 0.065183 0 0.000237051 0.0036367 0.345101 0.00178925 +4.33594 0.00347148 0.0738635 0 0.000256416 0.00347148 0.400256 0.00188541 +4.41406 0.00331011 0.0825441 0 0.00027323 0.00331011 0.458068 0.00196696 +4.49219 0.00315257 0.0912246 0 0.000287592 0.00315257 0.518734 0.0020353 +4.57031 0.00299888 0.0999052 0 0.000299604 0.00299888 0.58247 0.00209158 +4.64844 0.00284903 0.108586 0 0.000309364 0.00284903 0.649516 0.00213677 +4.72656 0.00270302 0.117266 0 0.000316973 0.00270302 0.720135 0.00217166 +4.80469 0.00256085 0.125947 0 0.000322531 0.00256085 0.794623 0.00219697 +4.88281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03906 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11719 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19531 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27344 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35156 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42969 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50781 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58594 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66406 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74219 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82031 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89844 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97656 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05469 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13281 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21094 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.28906 0.001 0 0 0 0.001 0 0 +6.36719 0.001 0 0 0 0.001 0 0 +6.44531 0.001 0 0 0 0.001 0 0 +6.52344 0.001 0 0 0 0.001 0 0 +6.60156 0.001 0 0 0 0.001 0 0 +6.67969 0.001 0 0 0 0.001 0 0 +6.75781 0.001 0 0 0 0.001 0 0 +6.83594 0.001 0 0 0 0.001 0 0 +6.91406 0.001 0 0 0 0.001 0 0 +6.99219 0.001 0 0 0 0.001 0 0 +7.07031 0.001 0 0 0 0.001 0 0 +7.14844 0.001 0 0 0 0.001 0 0 +7.22656 0.001 0 0 0 0.001 0 0 +7.30469 0.001 0 0 0 0.001 0 0 +7.38281 0.001 0 0 0 0.001 0 0 +7.46094 0.001 0 0 0 0.001 0 0 +7.53906 0.001 0 0 0 0.001 0 0 +7.61719 0.001 0 0 0 0.001 0 0 +7.69531 0.001 0 0 0 0.001 0 0 +7.77344 0.001 0 0 0 0.001 0 0 +7.85156 0.001 0 0 0 0.001 0 0 +7.92969 0.001 0 0 0 0.001 0 0 +8.00781 0.001 0 0 0 0.001 0 0 +8.08594 0.001 0 0 0 0.001 0 0 +8.16406 0.001 0 0 0 0.001 0 0 +8.24219 0.001 0 0 0 0.001 0 0 +8.32031 0.001 0 0 0 0.001 0 0 +8.39844 0.001 0 0 0 0.001 0 0 +8.47656 0.001 0 0 0 0.001 0 0 +8.55469 0.001 0 0 0 0.001 0 0 +8.63281 0.001 0 0 0 0.001 0 0 +8.71094 0.001 0 0 0 0.001 0 0 +8.78906 0.001 0 0 0 0.001 0 0 +8.86719 0.001 0 0 0 0.001 0 0 +8.94531 0.001 0 0 0 0.001 0 0 +9.02344 0.001 0 0 0 0.001 0 0 +9.10156 0.001 0 0 0 0.001 0 0 +9.17969 0.001 0 0 0 0.001 0 0 +9.25781 0.001 0 0 0 0.001 0 0 +9.33594 0.001 0 0 0 0.001 0 0 +9.41406 0.001 0 0 0 0.001 0 0 +9.49219 0.001 0 0 0 0.001 0 0 +9.57031 0.001 0 0 0 0.001 0 0 +9.64844 0.001 0 0 0 0.001 0 0 +9.72656 0.001 0 0 0 0.001 0 0 +9.80469 0.001 0 0 0 0.001 0 0 +9.88281 0.001 0 0 0 0.001 0 0 +9.96094 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=2048.csv b/reference/swashes_1_nx=2048.csv new file mode 100644 index 0000000..184b240 --- /dev/null +++ b/reference/swashes_1_nx=2048.csv @@ -0,0 +1,2066 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00488281 meters +# Number of cells: 2048 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00244141 0.005 0 0 0 0.005 0 0 +0.00732422 0.005 0 0 0 0.005 0 0 +0.012207 0.005 0 0 0 0.005 0 0 +0.0170898 0.005 0 0 0 0.005 0 0 +0.0219727 0.005 0 0 0 0.005 0 0 +0.0268555 0.005 0 0 0 0.005 0 0 +0.0317383 0.005 0 0 0 0.005 0 0 +0.0366211 0.005 0 0 0 0.005 0 0 +0.0415039 0.005 0 0 0 0.005 0 0 +0.0463867 0.005 0 0 0 0.005 0 0 +0.0512695 0.005 0 0 0 0.005 0 0 +0.0561523 0.005 0 0 0 0.005 0 0 +0.0610352 0.005 0 0 0 0.005 0 0 +0.065918 0.005 0 0 0 0.005 0 0 +0.0708008 0.005 0 0 0 0.005 0 0 +0.0756836 0.005 0 0 0 0.005 0 0 +0.0805664 0.005 0 0 0 0.005 0 0 +0.0854492 0.005 0 0 0 0.005 0 0 +0.090332 0.005 0 0 0 0.005 0 0 +0.0952148 0.005 0 0 0 0.005 0 0 +0.100098 0.005 0 0 0 0.005 0 0 +0.10498 0.005 0 0 0 0.005 0 0 +0.109863 0.005 0 0 0 0.005 0 0 +0.114746 0.005 0 0 0 0.005 0 0 +0.119629 0.005 0 0 0 0.005 0 0 +0.124512 0.005 0 0 0 0.005 0 0 +0.129395 0.005 0 0 0 0.005 0 0 +0.134277 0.005 0 0 0 0.005 0 0 +0.13916 0.005 0 0 0 0.005 0 0 +0.144043 0.005 0 0 0 0.005 0 0 +0.148926 0.005 0 0 0 0.005 0 0 +0.153809 0.005 0 0 0 0.005 0 0 +0.158691 0.005 0 0 0 0.005 0 0 +0.163574 0.005 0 0 0 0.005 0 0 +0.168457 0.005 0 0 0 0.005 0 0 +0.17334 0.005 0 0 0 0.005 0 0 +0.178223 0.005 0 0 0 0.005 0 0 +0.183105 0.005 0 0 0 0.005 0 0 +0.187988 0.005 0 0 0 0.005 0 0 +0.192871 0.005 0 0 0 0.005 0 0 +0.197754 0.005 0 0 0 0.005 0 0 +0.202637 0.005 0 0 0 0.005 0 0 +0.20752 0.005 0 0 0 0.005 0 0 +0.212402 0.005 0 0 0 0.005 0 0 +0.217285 0.005 0 0 0 0.005 0 0 +0.222168 0.005 0 0 0 0.005 0 0 +0.227051 0.005 0 0 0 0.005 0 0 +0.231934 0.005 0 0 0 0.005 0 0 +0.236816 0.005 0 0 0 0.005 0 0 +0.241699 0.005 0 0 0 0.005 0 0 +0.246582 0.005 0 0 0 0.005 0 0 +0.251465 0.005 0 0 0 0.005 0 0 +0.256348 0.005 0 0 0 0.005 0 0 +0.26123 0.005 0 0 0 0.005 0 0 +0.266113 0.005 0 0 0 0.005 0 0 +0.270996 0.005 0 0 0 0.005 0 0 +0.275879 0.005 0 0 0 0.005 0 0 +0.280762 0.005 0 0 0 0.005 0 0 +0.285645 0.005 0 0 0 0.005 0 0 +0.290527 0.005 0 0 0 0.005 0 0 +0.29541 0.005 0 0 0 0.005 0 0 +0.300293 0.005 0 0 0 0.005 0 0 +0.305176 0.005 0 0 0 0.005 0 0 +0.310059 0.005 0 0 0 0.005 0 0 +0.314941 0.005 0 0 0 0.005 0 0 +0.319824 0.005 0 0 0 0.005 0 0 +0.324707 0.005 0 0 0 0.005 0 0 +0.32959 0.005 0 0 0 0.005 0 0 +0.334473 0.005 0 0 0 0.005 0 0 +0.339355 0.005 0 0 0 0.005 0 0 +0.344238 0.005 0 0 0 0.005 0 0 +0.349121 0.005 0 0 0 0.005 0 0 +0.354004 0.005 0 0 0 0.005 0 0 +0.358887 0.005 0 0 0 0.005 0 0 +0.36377 0.005 0 0 0 0.005 0 0 +0.368652 0.005 0 0 0 0.005 0 0 +0.373535 0.005 0 0 0 0.005 0 0 +0.378418 0.005 0 0 0 0.005 0 0 +0.383301 0.005 0 0 0 0.005 0 0 +0.388184 0.005 0 0 0 0.005 0 0 +0.393066 0.005 0 0 0 0.005 0 0 +0.397949 0.005 0 0 0 0.005 0 0 +0.402832 0.005 0 0 0 0.005 0 0 +0.407715 0.005 0 0 0 0.005 0 0 +0.412598 0.005 0 0 0 0.005 0 0 +0.41748 0.005 0 0 0 0.005 0 0 +0.422363 0.005 0 0 0 0.005 0 0 +0.427246 0.005 0 0 0 0.005 0 0 +0.432129 0.005 0 0 0 0.005 0 0 +0.437012 0.005 0 0 0 0.005 0 0 +0.441895 0.005 0 0 0 0.005 0 0 +0.446777 0.005 0 0 0 0.005 0 0 +0.45166 0.005 0 0 0 0.005 0 0 +0.456543 0.005 0 0 0 0.005 0 0 +0.461426 0.005 0 0 0 0.005 0 0 +0.466309 0.005 0 0 0 0.005 0 0 +0.471191 0.005 0 0 0 0.005 0 0 +0.476074 0.005 0 0 0 0.005 0 0 +0.480957 0.005 0 0 0 0.005 0 0 +0.48584 0.005 0 0 0 0.005 0 0 +0.490723 0.005 0 0 0 0.005 0 0 +0.495605 0.005 0 0 0 0.005 0 0 +0.500488 0.005 0 0 0 0.005 0 0 +0.505371 0.005 0 0 0 0.005 0 0 +0.510254 0.005 0 0 0 0.005 0 0 +0.515137 0.005 0 0 0 0.005 0 0 +0.52002 0.005 0 0 0 0.005 0 0 +0.524902 0.005 0 0 0 0.005 0 0 +0.529785 0.005 0 0 0 0.005 0 0 +0.534668 0.005 0 0 0 0.005 0 0 +0.539551 0.005 0 0 0 0.005 0 0 +0.544434 0.005 0 0 0 0.005 0 0 +0.549316 0.005 0 0 0 0.005 0 0 +0.554199 0.005 0 0 0 0.005 0 0 +0.559082 0.005 0 0 0 0.005 0 0 +0.563965 0.005 0 0 0 0.005 0 0 +0.568848 0.005 0 0 0 0.005 0 0 +0.57373 0.005 0 0 0 0.005 0 0 +0.578613 0.005 0 0 0 0.005 0 0 +0.583496 0.005 0 0 0 0.005 0 0 +0.588379 0.005 0 0 0 0.005 0 0 +0.593262 0.005 0 0 0 0.005 0 0 +0.598145 0.005 0 0 0 0.005 0 0 +0.603027 0.005 0 0 0 0.005 0 0 +0.60791 0.005 0 0 0 0.005 0 0 +0.612793 0.005 0 0 0 0.005 0 0 +0.617676 0.005 0 0 0 0.005 0 0 +0.622559 0.005 0 0 0 0.005 0 0 +0.627441 0.005 0 0 0 0.005 0 0 +0.632324 0.005 0 0 0 0.005 0 0 +0.637207 0.005 0 0 0 0.005 0 0 +0.64209 0.005 0 0 0 0.005 0 0 +0.646973 0.005 0 0 0 0.005 0 0 +0.651855 0.005 0 0 0 0.005 0 0 +0.656738 0.005 0 0 0 0.005 0 0 +0.661621 0.005 0 0 0 0.005 0 0 +0.666504 0.005 0 0 0 0.005 0 0 +0.671387 0.005 0 0 0 0.005 0 0 +0.67627 0.005 0 0 0 0.005 0 0 +0.681152 0.005 0 0 0 0.005 0 0 +0.686035 0.005 0 0 0 0.005 0 0 +0.690918 0.005 0 0 0 0.005 0 0 +0.695801 0.005 0 0 0 0.005 0 0 +0.700684 0.005 0 0 0 0.005 0 0 +0.705566 0.005 0 0 0 0.005 0 0 +0.710449 0.005 0 0 0 0.005 0 0 +0.715332 0.005 0 0 0 0.005 0 0 +0.720215 0.005 0 0 0 0.005 0 0 +0.725098 0.005 0 0 0 0.005 0 0 +0.72998 0.005 0 0 0 0.005 0 0 +0.734863 0.005 0 0 0 0.005 0 0 +0.739746 0.005 0 0 0 0.005 0 0 +0.744629 0.005 0 0 0 0.005 0 0 +0.749512 0.005 0 0 0 0.005 0 0 +0.754395 0.005 0 0 0 0.005 0 0 +0.759277 0.005 0 0 0 0.005 0 0 +0.76416 0.005 0 0 0 0.005 0 0 +0.769043 0.005 0 0 0 0.005 0 0 +0.773926 0.005 0 0 0 0.005 0 0 +0.778809 0.005 0 0 0 0.005 0 0 +0.783691 0.005 0 0 0 0.005 0 0 +0.788574 0.005 0 0 0 0.005 0 0 +0.793457 0.005 0 0 0 0.005 0 0 +0.79834 0.005 0 0 0 0.005 0 0 +0.803223 0.005 0 0 0 0.005 0 0 +0.808105 0.005 0 0 0 0.005 0 0 +0.812988 0.005 0 0 0 0.005 0 0 +0.817871 0.005 0 0 0 0.005 0 0 +0.822754 0.005 0 0 0 0.005 0 0 +0.827637 0.005 0 0 0 0.005 0 0 +0.83252 0.005 0 0 0 0.005 0 0 +0.837402 0.005 0 0 0 0.005 0 0 +0.842285 0.005 0 0 0 0.005 0 0 +0.847168 0.005 0 0 0 0.005 0 0 +0.852051 0.005 0 0 0 0.005 0 0 +0.856934 0.005 0 0 0 0.005 0 0 +0.861816 0.005 0 0 0 0.005 0 0 +0.866699 0.005 0 0 0 0.005 0 0 +0.871582 0.005 0 0 0 0.005 0 0 +0.876465 0.005 0 0 0 0.005 0 0 +0.881348 0.005 0 0 0 0.005 0 0 +0.88623 0.005 0 0 0 0.005 0 0 +0.891113 0.005 0 0 0 0.005 0 0 +0.895996 0.005 0 0 0 0.005 0 0 +0.900879 0.005 0 0 0 0.005 0 0 +0.905762 0.005 0 0 0 0.005 0 0 +0.910645 0.005 0 0 0 0.005 0 0 +0.915527 0.005 0 0 0 0.005 0 0 +0.92041 0.005 0 0 0 0.005 0 0 +0.925293 0.005 0 0 0 0.005 0 0 +0.930176 0.005 0 0 0 0.005 0 0 +0.935059 0.005 0 0 0 0.005 0 0 +0.939941 0.005 0 0 0 0.005 0 0 +0.944824 0.005 0 0 0 0.005 0 0 +0.949707 0.005 0 0 0 0.005 0 0 +0.95459 0.005 0 0 0 0.005 0 0 +0.959473 0.005 0 0 0 0.005 0 0 +0.964355 0.005 0 0 0 0.005 0 0 +0.969238 0.005 0 0 0 0.005 0 0 +0.974121 0.005 0 0 0 0.005 0 0 +0.979004 0.005 0 0 0 0.005 0 0 +0.983887 0.005 0 0 0 0.005 0 0 +0.98877 0.005 0 0 0 0.005 0 0 +0.993652 0.005 0 0 0 0.005 0 0 +0.998535 0.005 0 0 0 0.005 0 0 +1.00342 0.005 0 0 0 0.005 0 0 +1.0083 0.005 0 0 0 0.005 0 0 +1.01318 0.005 0 0 0 0.005 0 0 +1.01807 0.005 0 0 0 0.005 0 0 +1.02295 0.005 0 0 0 0.005 0 0 +1.02783 0.005 0 0 0 0.005 0 0 +1.03271 0.005 0 0 0 0.005 0 0 +1.0376 0.005 0 0 0 0.005 0 0 +1.04248 0.005 0 0 0 0.005 0 0 +1.04736 0.005 0 0 0 0.005 0 0 +1.05225 0.005 0 0 0 0.005 0 0 +1.05713 0.005 0 0 0 0.005 0 0 +1.06201 0.005 0 0 0 0.005 0 0 +1.06689 0.005 0 0 0 0.005 0 0 +1.07178 0.005 0 0 0 0.005 0 0 +1.07666 0.005 0 0 0 0.005 0 0 +1.08154 0.005 0 0 0 0.005 0 0 +1.08643 0.005 0 0 0 0.005 0 0 +1.09131 0.005 0 0 0 0.005 0 0 +1.09619 0.005 0 0 0 0.005 0 0 +1.10107 0.005 0 0 0 0.005 0 0 +1.10596 0.005 0 0 0 0.005 0 0 +1.11084 0.005 0 0 0 0.005 0 0 +1.11572 0.005 0 0 0 0.005 0 0 +1.12061 0.005 0 0 0 0.005 0 0 +1.12549 0.005 0 0 0 0.005 0 0 +1.13037 0.005 0 0 0 0.005 0 0 +1.13525 0.005 0 0 0 0.005 0 0 +1.14014 0.005 0 0 0 0.005 0 0 +1.14502 0.005 0 0 0 0.005 0 0 +1.1499 0.005 0 0 0 0.005 0 0 +1.15479 0.005 0 0 0 0.005 0 0 +1.15967 0.005 0 0 0 0.005 0 0 +1.16455 0.005 0 0 0 0.005 0 0 +1.16943 0.005 0 0 0 0.005 0 0 +1.17432 0.005 0 0 0 0.005 0 0 +1.1792 0.005 0 0 0 0.005 0 0 +1.18408 0.005 0 0 0 0.005 0 0 +1.18896 0.005 0 0 0 0.005 0 0 +1.19385 0.005 0 0 0 0.005 0 0 +1.19873 0.005 0 0 0 0.005 0 0 +1.20361 0.005 0 0 0 0.005 0 0 +1.2085 0.005 0 0 0 0.005 0 0 +1.21338 0.005 0 0 0 0.005 0 0 +1.21826 0.005 0 0 0 0.005 0 0 +1.22314 0.005 0 0 0 0.005 0 0 +1.22803 0.005 0 0 0 0.005 0 0 +1.23291 0.005 0 0 0 0.005 0 0 +1.23779 0.005 0 0 0 0.005 0 0 +1.24268 0.005 0 0 0 0.005 0 0 +1.24756 0.005 0 0 0 0.005 0 0 +1.25244 0.005 0 0 0 0.005 0 0 +1.25732 0.005 0 0 0 0.005 0 0 +1.26221 0.005 0 0 0 0.005 0 0 +1.26709 0.005 0 0 0 0.005 0 0 +1.27197 0.005 0 0 0 0.005 0 0 +1.27686 0.005 0 0 0 0.005 0 0 +1.28174 0.005 0 0 0 0.005 0 0 +1.28662 0.005 0 0 0 0.005 0 0 +1.2915 0.005 0 0 0 0.005 0 0 +1.29639 0.005 0 0 0 0.005 0 0 +1.30127 0.005 0 0 0 0.005 0 0 +1.30615 0.005 0 0 0 0.005 0 0 +1.31104 0.005 0 0 0 0.005 0 0 +1.31592 0.005 0 0 0 0.005 0 0 +1.3208 0.005 0 0 0 0.005 0 0 +1.32568 0.005 0 0 0 0.005 0 0 +1.33057 0.005 0 0 0 0.005 0 0 +1.33545 0.005 0 0 0 0.005 0 0 +1.34033 0.005 0 0 0 0.005 0 0 +1.34521 0.005 0 0 0 0.005 0 0 +1.3501 0.005 0 0 0 0.005 0 0 +1.35498 0.005 0 0 0 0.005 0 0 +1.35986 0.005 0 0 0 0.005 0 0 +1.36475 0.005 0 0 0 0.005 0 0 +1.36963 0.005 0 0 0 0.005 0 0 +1.37451 0.005 0 0 0 0.005 0 0 +1.37939 0.005 0 0 0 0.005 0 0 +1.38428 0.005 0 0 0 0.005 0 0 +1.38916 0.005 0 0 0 0.005 0 0 +1.39404 0.005 0 0 0 0.005 0 0 +1.39893 0.005 0 0 0 0.005 0 0 +1.40381 0.005 0 0 0 0.005 0 0 +1.40869 0.005 0 0 0 0.005 0 0 +1.41357 0.005 0 0 0 0.005 0 0 +1.41846 0.005 0 0 0 0.005 0 0 +1.42334 0.005 0 0 0 0.005 0 0 +1.42822 0.005 0 0 0 0.005 0 0 +1.43311 0.005 0 0 0 0.005 0 0 +1.43799 0.005 0 0 0 0.005 0 0 +1.44287 0.005 0 0 0 0.005 0 0 +1.44775 0.005 0 0 0 0.005 0 0 +1.45264 0.005 0 0 0 0.005 0 0 +1.45752 0.005 0 0 0 0.005 0 0 +1.4624 0.005 0 0 0 0.005 0 0 +1.46729 0.005 0 0 0 0.005 0 0 +1.47217 0.005 0 0 0 0.005 0 0 +1.47705 0.005 0 0 0 0.005 0 0 +1.48193 0.005 0 0 0 0.005 0 0 +1.48682 0.005 0 0 0 0.005 0 0 +1.4917 0.005 0 0 0 0.005 0 0 +1.49658 0.005 0 0 0 0.005 0 0 +1.50146 0.005 0 0 0 0.005 0 0 +1.50635 0.005 0 0 0 0.005 0 0 +1.51123 0.005 0 0 0 0.005 0 0 +1.51611 0.005 0 0 0 0.005 0 0 +1.521 0.005 0 0 0 0.005 0 0 +1.52588 0.005 0 0 0 0.005 0 0 +1.53076 0.005 0 0 0 0.005 0 0 +1.53564 0.005 0 0 0 0.005 0 0 +1.54053 0.005 0 0 0 0.005 0 0 +1.54541 0.005 0 0 0 0.005 0 0 +1.55029 0.005 0 0 0 0.005 0 0 +1.55518 0.005 0 0 0 0.005 0 0 +1.56006 0.005 0 0 0 0.005 0 0 +1.56494 0.005 0 0 0 0.005 0 0 +1.56982 0.005 0 0 0 0.005 0 0 +1.57471 0.005 0 0 0 0.005 0 0 +1.57959 0.005 0 0 0 0.005 0 0 +1.58447 0.005 0 0 0 0.005 0 0 +1.58936 0.005 0 0 0 0.005 0 0 +1.59424 0.005 0 0 0 0.005 0 0 +1.59912 0.005 0 0 0 0.005 0 0 +1.604 0.005 0 0 0 0.005 0 0 +1.60889 0.005 0 0 0 0.005 0 0 +1.61377 0.005 0 0 0 0.005 0 0 +1.61865 0.005 0 0 0 0.005 0 0 +1.62354 0.005 0 0 0 0.005 0 0 +1.62842 0.005 0 0 0 0.005 0 0 +1.6333 0.005 0 0 0 0.005 0 0 +1.63818 0.005 0 0 0 0.005 0 0 +1.64307 0.005 0 0 0 0.005 0 0 +1.64795 0.005 0 0 0 0.005 0 0 +1.65283 0.005 0 0 0 0.005 0 0 +1.65771 0.005 0 0 0 0.005 0 0 +1.6626 0.005 0 0 0 0.005 0 0 +1.66748 0.005 0 0 0 0.005 0 0 +1.67236 0.005 0 0 0 0.005 0 0 +1.67725 0.005 0 0 0 0.005 0 0 +1.68213 0.005 0 0 0 0.005 0 0 +1.68701 0.005 0 0 0 0.005 0 0 +1.69189 0.005 0 0 0 0.005 0 0 +1.69678 0.005 0 0 0 0.005 0 0 +1.70166 0.005 0 0 0 0.005 0 0 +1.70654 0.005 0 0 0 0.005 0 0 +1.71143 0.005 0 0 0 0.005 0 0 +1.71631 0.005 0 0 0 0.005 0 0 +1.72119 0.005 0 0 0 0.005 0 0 +1.72607 0.005 0 0 0 0.005 0 0 +1.73096 0.005 0 0 0 0.005 0 0 +1.73584 0.005 0 0 0 0.005 0 0 +1.74072 0.005 0 0 0 0.005 0 0 +1.74561 0.005 0 0 0 0.005 0 0 +1.75049 0.005 0 0 0 0.005 0 0 +1.75537 0.005 0 0 0 0.005 0 0 +1.76025 0.005 0 0 0 0.005 0 0 +1.76514 0.005 0 0 0 0.005 0 0 +1.77002 0.005 0 0 0 0.005 0 0 +1.7749 0.005 0 0 0 0.005 0 0 +1.77979 0.005 0 0 0 0.005 0 0 +1.78467 0.005 0 0 0 0.005 0 0 +1.78955 0.005 0 0 0 0.005 0 0 +1.79443 0.005 0 0 0 0.005 0 0 +1.79932 0.005 0 0 0 0.005 0 0 +1.8042 0.005 0 0 0 0.005 0 0 +1.80908 0.005 0 0 0 0.005 0 0 +1.81396 0.005 0 0 0 0.005 0 0 +1.81885 0.005 0 0 0 0.005 0 0 +1.82373 0.005 0 0 0 0.005 0 0 +1.82861 0.005 0 0 0 0.005 0 0 +1.8335 0.005 0 0 0 0.005 0 0 +1.83838 0.005 0 0 0 0.005 0 0 +1.84326 0.005 0 0 0 0.005 0 0 +1.84814 0.005 0 0 0 0.005 0 0 +1.85303 0.005 0 0 0 0.005 0 0 +1.85791 0.005 0 0 0 0.005 0 0 +1.86279 0.005 0 0 0 0.005 0 0 +1.86768 0.005 0 0 0 0.005 0 0 +1.87256 0.005 0 0 0 0.005 0 0 +1.87744 0.005 0 0 0 0.005 0 0 +1.88232 0.005 0 0 0 0.005 0 0 +1.88721 0.005 0 0 0 0.005 0 0 +1.89209 0.005 0 0 0 0.005 0 0 +1.89697 0.005 0 0 0 0.005 0 0 +1.90186 0.005 0 0 0 0.005 0 0 +1.90674 0.005 0 0 0 0.005 0 0 +1.91162 0.005 0 0 0 0.005 0 0 +1.9165 0.005 0 0 0 0.005 0 0 +1.92139 0.005 0 0 0 0.005 0 0 +1.92627 0.005 0 0 0 0.005 0 0 +1.93115 0.005 0 0 0 0.005 0 0 +1.93604 0.005 0 0 0 0.005 0 0 +1.94092 0.005 0 0 0 0.005 0 0 +1.9458 0.005 0 0 0 0.005 0 0 +1.95068 0.005 0 0 0 0.005 0 0 +1.95557 0.005 0 0 0 0.005 0 0 +1.96045 0.005 0 0 0 0.005 0 0 +1.96533 0.005 0 0 0 0.005 0 0 +1.97021 0.005 0 0 0 0.005 0 0 +1.9751 0.005 0 0 0 0.005 0 0 +1.97998 0.005 0 0 0 0.005 0 0 +1.98486 0.005 0 0 0 0.005 0 0 +1.98975 0.005 0 0 0 0.005 0 0 +1.99463 0.005 0 0 0 0.005 0 0 +1.99951 0.005 0 0 0 0.005 0 0 +2.00439 0.005 0 0 0 0.005 0 0 +2.00928 0.005 0 0 0 0.005 0 0 +2.01416 0.005 0 0 0 0.005 0 0 +2.01904 0.005 0 0 0 0.005 0 0 +2.02393 0.005 0 0 0 0.005 0 0 +2.02881 0.005 0 0 0 0.005 0 0 +2.03369 0.005 0 0 0 0.005 0 0 +2.03857 0.005 0 0 0 0.005 0 0 +2.04346 0.005 0 0 0 0.005 0 0 +2.04834 0.005 0 0 0 0.005 0 0 +2.05322 0.005 0 0 0 0.005 0 0 +2.05811 0.005 0 0 0 0.005 0 0 +2.06299 0.005 0 0 0 0.005 0 0 +2.06787 0.005 0 0 0 0.005 0 0 +2.07275 0.005 0 0 0 0.005 0 0 +2.07764 0.005 0 0 0 0.005 0 0 +2.08252 0.005 0 0 0 0.005 0 0 +2.0874 0.005 0 0 0 0.005 0 0 +2.09229 0.005 0 0 0 0.005 0 0 +2.09717 0.005 0 0 0 0.005 0 0 +2.10205 0.005 0 0 0 0.005 0 0 +2.10693 0.005 0 0 0 0.005 0 0 +2.11182 0.005 0 0 0 0.005 0 0 +2.1167 0.005 0 0 0 0.005 0 0 +2.12158 0.005 0 0 0 0.005 0 0 +2.12646 0.005 0 0 0 0.005 0 0 +2.13135 0.005 0 0 0 0.005 0 0 +2.13623 0.005 0 0 0 0.005 0 0 +2.14111 0.005 0 0 0 0.005 0 0 +2.146 0.005 0 0 0 0.005 0 0 +2.15088 0.005 0 0 0 0.005 0 0 +2.15576 0.005 0 0 0 0.005 0 0 +2.16064 0.005 0 0 0 0.005 0 0 +2.16553 0.005 0 0 0 0.005 0 0 +2.17041 0.005 0 0 0 0.005 0 0 +2.17529 0.005 0 0 0 0.005 0 0 +2.18018 0.005 0 0 0 0.005 0 0 +2.18506 0.005 0 0 0 0.005 0 0 +2.18994 0.005 0 0 0 0.005 0 0 +2.19482 0.005 0 0 0 0.005 0 0 +2.19971 0.005 0 0 0 0.005 0 0 +2.20459 0.005 0 0 0 0.005 0 0 +2.20947 0.005 0 0 0 0.005 0 0 +2.21436 0.005 0 0 0 0.005 0 0 +2.21924 0.005 0 0 0 0.005 0 0 +2.22412 0.005 0 0 0 0.005 0 0 +2.229 0.005 0 0 0 0.005 0 0 +2.23389 0.005 0 0 0 0.005 0 0 +2.23877 0.005 0 0 0 0.005 0 0 +2.24365 0.005 0 0 0 0.005 0 0 +2.24854 0.005 0 0 0 0.005 0 0 +2.25342 0.005 0 0 0 0.005 0 0 +2.2583 0.005 0 0 0 0.005 0 0 +2.26318 0.005 0 0 0 0.005 0 0 +2.26807 0.005 0 0 0 0.005 0 0 +2.27295 0.005 0 0 0 0.005 0 0 +2.27783 0.005 0 0 0 0.005 0 0 +2.28271 0.005 0 0 0 0.005 0 0 +2.2876 0.005 0 0 0 0.005 0 0 +2.29248 0.005 0 0 0 0.005 0 0 +2.29736 0.005 0 0 0 0.005 0 0 +2.30225 0.005 0 0 0 0.005 0 0 +2.30713 0.005 0 0 0 0.005 0 0 +2.31201 0.005 0 0 0 0.005 0 0 +2.31689 0.005 0 0 0 0.005 0 0 +2.32178 0.005 0 0 0 0.005 0 0 +2.32666 0.005 0 0 0 0.005 0 0 +2.33154 0.005 0 0 0 0.005 0 0 +2.33643 0.005 0 0 0 0.005 0 0 +2.34131 0.005 0 0 0 0.005 0 0 +2.34619 0.005 0 0 0 0.005 0 0 +2.35107 0.005 0 0 0 0.005 0 0 +2.35596 0.005 0 0 0 0.005 0 0 +2.36084 0.005 0 0 0 0.005 0 0 +2.36572 0.005 0 0 0 0.005 0 0 +2.37061 0.005 0 0 0 0.005 0 0 +2.37549 0.005 0 0 0 0.005 0 0 +2.38037 0.005 0 0 0 0.005 0 0 +2.38525 0.005 0 0 0 0.005 0 0 +2.39014 0.005 0 0 0 0.005 0 0 +2.39502 0.005 0 0 0 0.005 0 0 +2.3999 0.005 0 0 0 0.005 0 0 +2.40479 0.005 0 0 0 0.005 0 0 +2.40967 0.005 0 0 0 0.005 0 0 +2.41455 0.005 0 0 0 0.005 0 0 +2.41943 0.005 0 0 0 0.005 0 0 +2.42432 0.005 0 0 0 0.005 0 0 +2.4292 0.005 0 0 0 0.005 0 0 +2.43408 0.005 0 0 0 0.005 0 0 +2.43896 0.005 0 0 0 0.005 0 0 +2.44385 0.005 0 0 0 0.005 0 0 +2.44873 0.005 0 0 0 0.005 0 0 +2.45361 0.005 0 0 0 0.005 0 0 +2.4585 0.005 0 0 0 0.005 0 0 +2.46338 0.005 0 0 0 0.005 0 0 +2.46826 0.005 0 0 0 0.005 0 0 +2.47314 0.005 0 0 0 0.005 0 0 +2.47803 0.005 0 0 0 0.005 0 0 +2.48291 0.005 0 0 0 0.005 0 0 +2.48779 0.005 0 0 0 0.005 0 0 +2.49268 0.005 0 0 0 0.005 0 0 +2.49756 0.005 0 0 0 0.005 0 0 +2.50244 0.005 0 0 0 0.005 0 0 +2.50732 0.005 0 0 0 0.005 0 0 +2.51221 0.005 0 0 0 0.005 0 0 +2.51709 0.005 0 0 0 0.005 0 0 +2.52197 0.005 0 0 0 0.005 0 0 +2.52686 0.005 0 0 0 0.005 0 0 +2.53174 0.005 0 0 0 0.005 0 0 +2.53662 0.005 0 0 0 0.005 0 0 +2.5415 0.005 0 0 0 0.005 0 0 +2.54639 0.005 0 0 0 0.005 0 0 +2.55127 0.005 0 0 0 0.005 0 0 +2.55615 0.005 0 0 0 0.005 0 0 +2.56104 0.005 0 0 0 0.005 0 0 +2.56592 0.005 0 0 0 0.005 0 0 +2.5708 0.005 0 0 0 0.005 0 0 +2.57568 0.005 0 0 0 0.005 0 0 +2.58057 0.005 0 0 0 0.005 0 0 +2.58545 0.005 0 0 0 0.005 0 0 +2.59033 0.005 0 0 0 0.005 0 0 +2.59521 0.005 0 0 0 0.005 0 0 +2.6001 0.005 0 0 0 0.005 0 0 +2.60498 0.005 0 0 0 0.005 0 0 +2.60986 0.005 0 0 0 0.005 0 0 +2.61475 0.005 0 0 0 0.005 0 0 +2.61963 0.005 0 0 0 0.005 0 0 +2.62451 0.005 0 0 0 0.005 0 0 +2.62939 0.005 0 0 0 0.005 0 0 +2.63428 0.005 0 0 0 0.005 0 0 +2.63916 0.005 0 0 0 0.005 0 0 +2.64404 0.005 0 0 0 0.005 0 0 +2.64893 0.005 0 0 0 0.005 0 0 +2.65381 0.005 0 0 0 0.005 0 0 +2.65869 0.005 0 0 0 0.005 0 0 +2.66357 0.005 0 0 0 0.005 0 0 +2.66846 0.005 0 0 0 0.005 0 0 +2.67334 0.005 0 0 0 0.005 0 0 +2.67822 0.005 0 0 0 0.005 0 0 +2.68311 0.005 0 0 0 0.005 0 0 +2.68799 0.005 0 0 0 0.005 0 0 +2.69287 0.005 0 0 0 0.005 0 0 +2.69775 0.005 0 0 0 0.005 0 0 +2.70264 0.005 0 0 0 0.005 0 0 +2.70752 0.005 0 0 0 0.005 0 0 +2.7124 0.005 0 0 0 0.005 0 0 +2.71729 0.005 0 0 0 0.005 0 0 +2.72217 0.005 0 0 0 0.005 0 0 +2.72705 0.005 0 0 0 0.005 0 0 +2.73193 0.005 0 0 0 0.005 0 0 +2.73682 0.005 0 0 0 0.005 0 0 +2.7417 0.005 0 0 0 0.005 0 0 +2.74658 0.005 0 0 0 0.005 0 0 +2.75146 0.005 0 0 0 0.005 0 0 +2.75635 0.005 0 0 0 0.005 0 0 +2.76123 0.005 0 0 0 0.005 0 0 +2.76611 0.005 0 0 0 0.005 0 0 +2.771 0.005 0 0 0 0.005 0 0 +2.77588 0.005 0 0 0 0.005 0 0 +2.78076 0.005 0 0 0 0.005 0 0 +2.78564 0.005 0 0 0 0.005 0 0 +2.79053 0.005 0 0 0 0.005 0 0 +2.79541 0.005 0 0 0 0.005 0 0 +2.80029 0.005 0 0 0 0.005 0 0 +2.80518 0.005 0 0 0 0.005 0 0 +2.81006 0.005 0 0 0 0.005 0 0 +2.81494 0.005 0 0 0 0.005 0 0 +2.81982 0.005 0 0 0 0.005 0 0 +2.82471 0.005 0 0 0 0.005 0 0 +2.82959 0.005 0 0 0 0.005 0 0 +2.83447 0.005 0 0 0 0.005 0 0 +2.83936 0.005 0 0 0 0.005 0 0 +2.84424 0.005 0 0 0 0.005 0 0 +2.84912 0.005 0 0 0 0.005 0 0 +2.854 0.005 0 0 0 0.005 0 0 +2.85889 0.005 0 0 0 0.005 0 0 +2.86377 0.005 0 0 0 0.005 0 0 +2.86865 0.005 0 0 0 0.005 0 0 +2.87354 0.005 0 0 0 0.005 0 0 +2.87842 0.005 0 0 0 0.005 0 0 +2.8833 0.005 0 0 0 0.005 0 0 +2.88818 0.005 0 0 0 0.005 0 0 +2.89307 0.005 0 0 0 0.005 0 0 +2.89795 0.005 0 0 0 0.005 0 0 +2.90283 0.005 0 0 0 0.005 0 0 +2.90771 0.005 0 0 0 0.005 0 0 +2.9126 0.005 0 0 0 0.005 0 0 +2.91748 0.005 0 0 0 0.005 0 0 +2.92236 0.005 0 0 0 0.005 0 0 +2.92725 0.005 0 0 0 0.005 0 0 +2.93213 0.005 0 0 0 0.005 0 0 +2.93701 0.005 0 0 0 0.005 0 0 +2.94189 0.005 0 0 0 0.005 0 0 +2.94678 0.005 0 0 0 0.005 0 0 +2.95166 0.005 0 0 0 0.005 0 0 +2.95654 0.005 0 0 0 0.005 0 0 +2.96143 0.005 0 0 0 0.005 0 0 +2.96631 0.005 0 0 0 0.005 0 0 +2.97119 0.005 0 0 0 0.005 0 0 +2.97607 0.005 0 0 0 0.005 0 0 +2.98096 0.005 0 0 0 0.005 0 0 +2.98584 0.005 0 0 0 0.005 0 0 +2.99072 0.005 0 0 0 0.005 0 0 +2.99561 0.005 0 0 0 0.005 0 0 +3.00049 0.005 0 0 0 0.005 0 0 +3.00537 0.005 0 0 0 0.005 0 0 +3.01025 0.005 0 0 0 0.005 0 0 +3.01514 0.005 0 0 0 0.005 0 0 +3.02002 0.005 0 0 0 0.005 0 0 +3.0249 0.005 0 0 0 0.005 0 0 +3.02979 0.005 0 0 0 0.005 0 0 +3.03467 0.005 0 0 0 0.005 0 0 +3.03955 0.005 0 0 0 0.005 0 0 +3.04443 0.005 0 0 0 0.005 0 0 +3.04932 0.005 0 0 0 0.005 0 0 +3.0542 0.005 0 0 0 0.005 0 0 +3.05908 0.005 0 0 0 0.005 0 0 +3.06396 0.005 0 0 0 0.005 0 0 +3.06885 0.005 0 0 0 0.005 0 0 +3.07373 0.005 0 0 0 0.005 0 0 +3.07861 0.005 0 0 0 0.005 0 0 +3.0835 0.005 0 0 0 0.005 0 0 +3.08838 0.005 0 0 0 0.005 0 0 +3.09326 0.005 0 0 0 0.005 0 0 +3.09814 0.005 0 0 0 0.005 0 0 +3.10303 0.005 0 0 0 0.005 0 0 +3.10791 0.005 0 0 0 0.005 0 0 +3.11279 0.005 0 0 0 0.005 0 0 +3.11768 0.005 0 0 0 0.005 0 0 +3.12256 0.005 0 0 0 0.005 0 0 +3.12744 0.005 0 0 0 0.005 0 0 +3.13232 0.005 0 0 0 0.005 0 0 +3.13721 0.005 0 0 0 0.005 0 0 +3.14209 0.005 0 0 0 0.005 0 0 +3.14697 0.005 0 0 0 0.005 0 0 +3.15186 0.005 0 0 0 0.005 0 0 +3.15674 0.005 0 0 0 0.005 0 0 +3.16162 0.005 0 0 0 0.005 0 0 +3.1665 0.005 0 0 0 0.005 0 0 +3.17139 0.005 0 0 0 0.005 0 0 +3.17627 0.005 0 0 0 0.005 0 0 +3.18115 0.005 0 0 0 0.005 0 0 +3.18604 0.005 0 0 0 0.005 0 0 +3.19092 0.005 0 0 0 0.005 0 0 +3.1958 0.005 0 0 0 0.005 0 0 +3.20068 0.005 0 0 0 0.005 0 0 +3.20557 0.005 0 0 0 0.005 0 0 +3.21045 0.005 0 0 0 0.005 0 0 +3.21533 0.005 0 0 0 0.005 0 0 +3.22021 0.005 0 0 0 0.005 0 0 +3.2251 0.005 0 0 0 0.005 0 0 +3.22998 0.005 0 0 0 0.005 0 0 +3.23486 0.005 0 0 0 0.005 0 0 +3.23975 0.005 0 0 0 0.005 0 0 +3.24463 0.005 0 0 0 0.005 0 0 +3.24951 0.005 0 0 0 0.005 0 0 +3.25439 0.005 0 0 0 0.005 0 0 +3.25928 0.005 0 0 0 0.005 0 0 +3.26416 0.005 0 0 0 0.005 0 0 +3.26904 0.005 0 0 0 0.005 0 0 +3.27393 0.005 0 0 0 0.005 0 0 +3.27881 0.005 0 0 0 0.005 0 0 +3.28369 0.005 0 0 0 0.005 0 0 +3.28857 0.005 0 0 0 0.005 0 0 +3.29346 0.005 0 0 0 0.005 0 0 +3.29834 0.005 0 0 0 0.005 0 0 +3.30322 0.005 0 0 0 0.005 0 0 +3.30811 0.005 0 0 0 0.005 0 0 +3.31299 0.005 0 0 0 0.005 0 0 +3.31787 0.005 0 0 0 0.005 0 0 +3.32275 0.005 0 0 0 0.005 0 0 +3.32764 0.005 0 0 0 0.005 0 0 +3.33252 0.005 0 0 0 0.005 0 0 +3.3374 0.005 0 0 0 0.005 0 0 +3.34229 0.005 0 0 0 0.005 0 0 +3.34717 0.005 0 0 0 0.005 0 0 +3.35205 0.005 0 0 0 0.005 0 0 +3.35693 0.005 0 0 0 0.005 0 0 +3.36182 0.005 0 0 0 0.005 0 0 +3.3667 0.005 0 0 0 0.005 0 0 +3.37158 0.005 0 0 0 0.005 0 0 +3.37646 0.005 0 0 0 0.005 0 0 +3.38135 0.005 0 0 0 0.005 0 0 +3.38623 0.005 0 0 0 0.005 0 0 +3.39111 0.005 0 0 0 0.005 0 0 +3.396 0.005 0 0 0 0.005 0 0 +3.40088 0.005 0 0 0 0.005 0 0 +3.40576 0.005 0 0 0 0.005 0 0 +3.41064 0.005 0 0 0 0.005 0 0 +3.41553 0.005 0 0 0 0.005 0 0 +3.42041 0.005 0 0 0 0.005 0 0 +3.42529 0.005 0 0 0 0.005 0 0 +3.43018 0.005 0 0 0 0.005 0 0 +3.43506 0.005 0 0 0 0.005 0 0 +3.43994 0.005 0 0 0 0.005 0 0 +3.44482 0.005 0 0 0 0.005 0 0 +3.44971 0.005 0 0 0 0.005 0 0 +3.45459 0.005 0 0 0 0.005 0 0 +3.45947 0.005 0 0 0 0.005 0 0 +3.46436 0.005 0 0 0 0.005 0 0 +3.46924 0.005 0 0 0 0.005 0 0 +3.47412 0.005 0 0 0 0.005 0 0 +3.479 0.005 0 0 0 0.005 0 0 +3.48389 0.005 0 0 0 0.005 0 0 +3.48877 0.005 0 0 0 0.005 0 0 +3.49365 0.005 0 0 0 0.005 0 0 +3.49854 0.005 0 0 0 0.005 0 0 +3.50342 0.005 0 0 0 0.005 0 0 +3.5083 0.005 0 0 0 0.005 0 0 +3.51318 0.005 0 0 0 0.005 0 0 +3.51807 0.005 0 0 0 0.005 0 0 +3.52295 0.005 0 0 0 0.005 0 0 +3.52783 0.005 0 0 0 0.005 0 0 +3.53271 0.005 0 0 0 0.005 0 0 +3.5376 0.005 0 0 0 0.005 0 0 +3.54248 0.005 0 0 0 0.005 0 0 +3.54736 0.005 0 0 0 0.005 0 0 +3.55225 0.005 0 0 0 0.005 0 0 +3.55713 0.005 0 0 0 0.005 0 0 +3.56201 0.005 0 0 0 0.005 0 0 +3.56689 0.005 0 0 0 0.005 0 0 +3.57178 0.005 0 0 0 0.005 0 0 +3.57666 0.005 0 0 0 0.005 0 0 +3.58154 0.005 0 0 0 0.005 0 0 +3.58643 0.005 0 0 0 0.005 0 0 +3.59131 0.005 0 0 0 0.005 0 0 +3.59619 0.005 0 0 0 0.005 0 0 +3.60107 0.005 0 0 0 0.005 0 0 +3.60596 0.005 0 0 0 0.005 0 0 +3.61084 0.005 0 0 0 0.005 0 0 +3.61572 0.005 0 0 0 0.005 0 0 +3.62061 0.005 0 0 0 0.005 0 0 +3.62549 0.005 0 0 0 0.005 0 0 +3.63037 0.005 0 0 0 0.005 0 0 +3.63525 0.005 0 0 0 0.005 0 0 +3.64014 0.005 0 0 0 0.005 0 0 +3.64502 0.005 0 0 0 0.005 0 0 +3.6499 0.005 0 0 0 0.005 0 0 +3.65479 0.005 0 0 0 0.005 0 0 +3.65967 0.005 0 0 0 0.005 0 0 +3.66455 0.005 0 0 0 0.005 0 0 +3.66943 0.005 0 0 0 0.005 0 0 +3.67432 0.0049921 0.000350054 0 1.7475e-006 0.0049921 0.00158182 6.77728e-005 +3.6792 0.00497987 0.000892588 0 4.44497e-006 0.00497987 0.00403838 0.000126286 +3.68408 0.00496765 0.00143512 0 7.12919e-006 0.00496765 0.00650098 0.000173036 +3.68896 0.00495545 0.00197766 0 9.80019e-006 0.00495545 0.00896964 0.000213927 +3.69385 0.00494327 0.00252019 0 1.2458e-005 0.00494327 0.0114444 0.00025104 +3.69873 0.00493109 0.00306273 0 1.51026e-005 0.00493109 0.0139252 0.000285416 +3.70361 0.00491894 0.00360526 0 1.77341e-005 0.00491894 0.0164122 0.000317675 +3.7085 0.0049068 0.0041478 0 2.03524e-005 0.0049068 0.0189053 0.00034822 +3.71338 0.00489467 0.00469033 0 2.29576e-005 0.00489467 0.0214046 0.000377336 +3.71826 0.00488256 0.00523287 0 2.55498e-005 0.00488256 0.0239101 0.00040523 +3.72314 0.00487046 0.0057754 0 2.81289e-005 0.00487046 0.0264218 0.000432061 +3.72803 0.00485838 0.00631794 0 3.06949e-005 0.00485838 0.0289398 0.000457954 +3.73291 0.00484632 0.00686047 0 3.3248e-005 0.00484632 0.031464 0.000483008 +3.73779 0.00483427 0.007403 0 3.57881e-005 0.00483427 0.0339945 0.000507305 +3.74268 0.00482223 0.00794554 0 3.83152e-005 0.00482223 0.0365313 0.000530914 +3.74756 0.00481021 0.00848807 0 4.08294e-005 0.00481021 0.0390744 0.000553893 +3.75244 0.0047982 0.00903061 0 4.33307e-005 0.0047982 0.041624 0.000576289 +3.75732 0.00478621 0.00957314 0 4.58191e-005 0.00478621 0.0441798 0.000598147 +3.76221 0.00477423 0.0101157 0 4.82946e-005 0.00477423 0.0467421 0.000619502 +3.76709 0.00476227 0.0106582 0 5.07573e-005 0.00476227 0.0493109 0.000640387 +3.77197 0.00475033 0.0112007 0 5.32072e-005 0.00475033 0.0518861 0.000660831 +3.77686 0.0047384 0.0117433 0 5.56443e-005 0.0047384 0.0544677 0.00068086 +3.78174 0.00472648 0.0122858 0 5.80687e-005 0.00472648 0.0570559 0.000700495 +3.78662 0.00471458 0.0128284 0 6.04803e-005 0.00471458 0.0596506 0.000719757 +3.7915 0.00470269 0.0133709 0 6.28792e-005 0.00470269 0.0622519 0.000738666 +3.79639 0.00469082 0.0139134 0 6.52654e-005 0.00469082 0.0648597 0.000757238 +3.80127 0.00467897 0.014456 0 6.76389e-005 0.00467897 0.0674741 0.000775487 +3.80615 0.00466712 0.0149985 0 6.99998e-005 0.00466712 0.0700952 0.000793429 +3.81104 0.0046553 0.015541 0 7.23481e-005 0.0046553 0.0727229 0.000811076 +3.81592 0.00464349 0.0160836 0 7.46838e-005 0.00464349 0.0753573 0.00082844 +3.8208 0.00463169 0.0166261 0 7.70069e-005 0.00463169 0.0779985 0.000845532 +3.82568 0.00461991 0.0171686 0 7.93175e-005 0.00461991 0.0806463 0.000862362 +3.83057 0.00460814 0.0177112 0 8.16156e-005 0.00460814 0.0833009 0.00087894 +3.83545 0.00459639 0.0182537 0 8.39012e-005 0.00459639 0.0859623 0.000895273 +3.84033 0.00458466 0.0187962 0 8.61743e-005 0.00458466 0.0886304 0.000911371 +3.84521 0.00457294 0.0193388 0 8.84349e-005 0.00457294 0.0913055 0.000927242 +3.8501 0.00456123 0.0198813 0 9.06832e-005 0.00456123 0.0939874 0.000942891 +3.85498 0.00454954 0.0204238 0 9.2919e-005 0.00454954 0.0966761 0.000958326 +3.85986 0.00453786 0.0209664 0 9.51425e-005 0.00453786 0.0993718 0.000973554 +3.86475 0.0045262 0.0215089 0 9.73536e-005 0.0045262 0.102074 0.00098858 +3.86963 0.00451455 0.0220514 0 9.95524e-005 0.00451455 0.104784 0.00100341 +3.87451 0.00450292 0.022594 0 0.000101739 0.00450292 0.107501 0.00101805 +3.87939 0.00449131 0.0231365 0 0.000103913 0.00449131 0.110224 0.0010325 +3.88428 0.00447971 0.023679 0 0.000106075 0.00447971 0.112955 0.00104677 +3.88916 0.00446812 0.0242216 0 0.000108225 0.00446812 0.115693 0.00106087 +3.89404 0.00445655 0.0247641 0 0.000110363 0.00445655 0.118437 0.00107479 +3.89893 0.00444499 0.0253067 0 0.000112488 0.00444499 0.121189 0.00108855 +3.90381 0.00443345 0.0258492 0 0.000114601 0.00443345 0.123949 0.00110214 +3.90869 0.00442193 0.0263917 0 0.000116702 0.00442193 0.126715 0.00111557 +3.91357 0.00441041 0.0269343 0 0.000118791 0.00441041 0.129488 0.00112884 +3.91846 0.00439892 0.0274768 0 0.000120868 0.00439892 0.132269 0.00114196 +3.92334 0.00438744 0.0280193 0 0.000122933 0.00438744 0.135057 0.00115493 +3.92822 0.00437597 0.0285619 0 0.000124986 0.00437597 0.137853 0.00116775 +3.93311 0.00436452 0.0291044 0 0.000127027 0.00436452 0.140655 0.00118043 +3.93799 0.00435308 0.0296469 0 0.000129056 0.00435308 0.143465 0.00119297 +3.94287 0.00434166 0.0301895 0 0.000131072 0.00434166 0.146283 0.00120536 +3.94775 0.00433026 0.030732 0 0.000133077 0.00433026 0.149107 0.00121762 +3.95264 0.00431887 0.0312745 0 0.000135071 0.00431887 0.15194 0.00122975 +3.95752 0.00430749 0.0318171 0 0.000137052 0.00430749 0.15478 0.00124175 +3.9624 0.00429613 0.0323596 0 0.000139021 0.00429613 0.157627 0.00125361 +3.96729 0.00428478 0.0329021 0 0.000140979 0.00428478 0.160482 0.00126535 +3.97217 0.00427345 0.0334447 0 0.000142924 0.00427345 0.163344 0.00127697 +3.97705 0.00426214 0.0339872 0 0.000144858 0.00426214 0.166214 0.00128846 +3.98193 0.00425084 0.0345297 0 0.00014678 0.00425084 0.169091 0.00129984 +3.98682 0.00423955 0.0350723 0 0.000148691 0.00423955 0.171977 0.00131109 +3.9917 0.00422828 0.0356148 0 0.000150589 0.00422828 0.17487 0.00132223 +3.99658 0.00421702 0.0361573 0 0.000152476 0.00421702 0.17777 0.00133325 +4.00146 0.00420578 0.0366999 0 0.000154352 0.00420578 0.180679 0.00134416 +4.00635 0.00419455 0.0372424 0 0.000156215 0.00419455 0.183595 0.00135496 +4.01123 0.00418334 0.0377849 0 0.000158067 0.00418334 0.186519 0.00136565 +4.01611 0.00417215 0.0383275 0 0.000159908 0.00417215 0.189451 0.00137623 +4.021 0.00416097 0.03887 0 0.000161737 0.00416097 0.19239 0.0013867 +4.02588 0.0041498 0.0394126 0 0.000163554 0.0041498 0.195338 0.00139707 +4.03076 0.00413865 0.0399551 0 0.00016536 0.00413865 0.198293 0.00140734 +4.03564 0.00412751 0.0404976 0 0.000167154 0.00412751 0.201257 0.0014175 +4.04053 0.00411639 0.0410402 0 0.000168937 0.00411639 0.204228 0.00142756 +4.04541 0.00410529 0.0415827 0 0.000170709 0.00410529 0.207208 0.00143752 +4.05029 0.0040942 0.0421252 0 0.000172469 0.0040942 0.210196 0.00144739 +4.05518 0.00408312 0.0426678 0 0.000174218 0.00408312 0.213191 0.00145715 +4.06006 0.00407206 0.0432103 0 0.000175955 0.00407206 0.216195 0.00146682 +4.06494 0.00406101 0.0437528 0 0.000177681 0.00406101 0.219207 0.0014764 +4.06982 0.00404998 0.0442954 0 0.000179395 0.00404998 0.222227 0.00148588 +4.07471 0.00403897 0.0448379 0 0.000181099 0.00403897 0.225256 0.00149527 +4.07959 0.00402796 0.0453804 0 0.000182791 0.00402796 0.228292 0.00150457 +4.08447 0.00401698 0.045923 0 0.000184472 0.00401698 0.231337 0.00151378 +4.08936 0.00400601 0.0464655 0 0.000186141 0.00400601 0.234391 0.0015229 +4.09424 0.00399505 0.047008 0 0.0001878 0.00399505 0.237452 0.00153193 +4.09912 0.00398411 0.0475506 0 0.000189447 0.00398411 0.240522 0.00154088 +4.104 0.00397318 0.0480931 0 0.000191083 0.00397318 0.243601 0.00154974 +4.10889 0.00396227 0.0486356 0 0.000192708 0.00396227 0.246688 0.00155851 +4.11377 0.00395138 0.0491782 0 0.000194322 0.00395138 0.249783 0.0015672 +4.11865 0.0039405 0.0497207 0 0.000195924 0.0039405 0.252887 0.00157581 +4.12354 0.00392963 0.0502632 0 0.000197516 0.00392963 0.256 0.00158433 +4.12842 0.00391878 0.0508058 0 0.000199097 0.00391878 0.259121 0.00159277 +4.1333 0.00390794 0.0513483 0 0.000200666 0.00390794 0.262251 0.00160113 +4.13818 0.00389712 0.0518909 0 0.000202225 0.00389712 0.26539 0.00160941 +4.14307 0.00388632 0.0524334 0 0.000203773 0.00388632 0.268537 0.00161761 +4.14795 0.00387553 0.0529759 0 0.00020531 0.00387553 0.271693 0.00162574 +4.15283 0.00386475 0.0535185 0 0.000206835 0.00386475 0.274858 0.00163378 +4.15771 0.00385399 0.054061 0 0.00020835 0.00385399 0.278032 0.00164175 +4.1626 0.00384324 0.0546035 0 0.000209855 0.00384324 0.281214 0.00164964 +4.16748 0.00383251 0.0551461 0 0.000211348 0.00383251 0.284406 0.00165746 +4.17236 0.0038218 0.0556886 0 0.00021283 0.0038218 0.287606 0.0016652 +4.17725 0.0038111 0.0562311 0 0.000214302 0.0038111 0.290815 0.00167287 +4.18213 0.00380041 0.0567737 0 0.000215763 0.00380041 0.294034 0.00168046 +4.18701 0.00378974 0.0573162 0 0.000217213 0.00378974 0.297261 0.00168798 +4.19189 0.00377908 0.0578587 0 0.000218653 0.00377908 0.300498 0.00169543 +4.19678 0.00376844 0.0584013 0 0.000220082 0.00376844 0.303743 0.00170281 +4.20166 0.00375782 0.0589438 0 0.0002215 0.00375782 0.306998 0.00171012 +4.20654 0.0037472 0.0594863 0 0.000222907 0.0037472 0.310262 0.00171736 +4.21143 0.00373661 0.0600289 0 0.000224304 0.00373661 0.313536 0.00172452 +4.21631 0.00372603 0.0605714 0 0.000225691 0.00372603 0.316818 0.00173162 +4.22119 0.00371546 0.0611139 0 0.000227067 0.00371546 0.32011 0.00173865 +4.22607 0.00370491 0.0616565 0 0.000228432 0.00370491 0.323411 0.00174561 +4.23096 0.00369437 0.062199 0 0.000229786 0.00369437 0.326722 0.00175251 +4.23584 0.00368385 0.0627415 0 0.000231131 0.00368385 0.330042 0.00175934 +4.24072 0.00367335 0.0632841 0 0.000232464 0.00367335 0.333372 0.0017661 +4.24561 0.00366286 0.0638266 0 0.000233788 0.00366286 0.336711 0.00177279 +4.25049 0.00365238 0.0643692 0 0.000235101 0.00365238 0.34006 0.00177942 +4.25537 0.00364192 0.0649117 0 0.000236403 0.00364192 0.343418 0.00178599 +4.26025 0.00363147 0.0654542 0 0.000237695 0.00363147 0.346786 0.00179249 +4.26514 0.00362104 0.0659968 0 0.000238977 0.00362104 0.350164 0.00179893 +4.27002 0.00361063 0.0665393 0 0.000240249 0.00361063 0.353551 0.00180531 +4.2749 0.00360023 0.0670818 0 0.00024151 0.00360023 0.356948 0.00181162 +4.27979 0.00358984 0.0676244 0 0.000242761 0.00358984 0.360355 0.00181787 +4.28467 0.00357947 0.0681669 0 0.000244001 0.00357947 0.363772 0.00182406 +4.28955 0.00356911 0.0687094 0 0.000245232 0.00356911 0.367199 0.00183018 +4.29443 0.00355877 0.069252 0 0.000246452 0.00355877 0.370636 0.00183625 +4.29932 0.00354845 0.0697945 0 0.000247662 0.00354845 0.374083 0.00184226 +4.3042 0.00353814 0.070337 0 0.000248862 0.00353814 0.377539 0.0018482 +4.30908 0.00352784 0.0708796 0 0.000250052 0.00352784 0.381006 0.00185409 +4.31396 0.00351756 0.0714221 0 0.000251231 0.00351756 0.384483 0.00185991 +4.31885 0.00350729 0.0719646 0 0.000252401 0.00350729 0.38797 0.00186568 +4.32373 0.00349704 0.0725072 0 0.000253561 0.00349704 0.391468 0.00187139 +4.32861 0.00348681 0.0730497 0 0.00025471 0.00348681 0.394975 0.00187704 +4.3335 0.00347659 0.0735922 0 0.00025585 0.00347659 0.398493 0.00188264 +4.33838 0.00346638 0.0741348 0 0.000256979 0.00346638 0.402022 0.00188818 +4.34326 0.00345619 0.0746773 0 0.000258099 0.00345619 0.40556 0.00189366 +4.34814 0.00344601 0.0752198 0 0.000259209 0.00344601 0.409109 0.00189908 +4.35303 0.00343585 0.0757624 0 0.000260308 0.00343585 0.412669 0.00190445 +4.35791 0.00342571 0.0763049 0 0.000261398 0.00342571 0.416239 0.00190976 +4.36279 0.00341558 0.0768474 0 0.000262478 0.00341558 0.41982 0.00191502 +4.36768 0.00340546 0.07739 0 0.000263548 0.00340546 0.423411 0.00192022 +4.37256 0.00339536 0.0779325 0 0.000264609 0.00339536 0.427013 0.00192537 +4.37744 0.00338527 0.0784751 0 0.000265659 0.00338527 0.430626 0.00193046 +4.38232 0.0033752 0.0790176 0 0.0002667 0.0033752 0.43425 0.0019355 +4.38721 0.00336515 0.0795601 0 0.000267731 0.00336515 0.437884 0.00194048 +4.39209 0.00335511 0.0801027 0 0.000268753 0.00335511 0.441529 0.00194542 +4.39697 0.00334508 0.0806452 0 0.000269765 0.00334508 0.445185 0.00195029 +4.40186 0.00333507 0.0811877 0 0.000270767 0.00333507 0.448852 0.00195512 +4.40674 0.00332507 0.0817303 0 0.000271759 0.00332507 0.45253 0.0019599 +4.41162 0.00331509 0.0822728 0 0.000272742 0.00331509 0.45622 0.00196462 +4.4165 0.00330513 0.0828153 0 0.000273715 0.00330513 0.45992 0.00196929 +4.42139 0.00329518 0.0833579 0 0.000274679 0.00329518 0.463631 0.00197391 +4.42627 0.00328524 0.0839004 0 0.000275633 0.00328524 0.467354 0.00197848 +4.43115 0.00327532 0.0844429 0 0.000276578 0.00327532 0.471088 0.00198299 +4.43604 0.00326541 0.0849855 0 0.000277513 0.00326541 0.474833 0.00198746 +4.44092 0.00325552 0.085528 0 0.000278438 0.00325552 0.47859 0.00199188 +4.4458 0.00324565 0.0860705 0 0.000279355 0.00324565 0.482358 0.00199625 +4.45068 0.00323579 0.0866131 0 0.000280261 0.00323579 0.486137 0.00200056 +4.45557 0.00322594 0.0871556 0 0.000281159 0.00322594 0.489929 0.00200483 +4.46045 0.00321611 0.0876981 0 0.000282047 0.00321611 0.493731 0.00200905 +4.46533 0.00320629 0.0882407 0 0.000282925 0.00320629 0.497545 0.00201322 +4.47021 0.00319649 0.0887832 0 0.000283795 0.00319649 0.501371 0.00201734 +4.4751 0.00318671 0.0893257 0 0.000284655 0.00318671 0.505209 0.00202142 +4.47998 0.00317694 0.0898683 0 0.000285506 0.00317694 0.509059 0.00202544 +4.48486 0.00316718 0.0904108 0 0.000286347 0.00316718 0.51292 0.00202942 +4.48975 0.00315744 0.0909534 0 0.00028718 0.00315744 0.516793 0.00203335 +4.49463 0.00314771 0.0914959 0 0.000288003 0.00314771 0.520678 0.00203724 +4.49951 0.003138 0.0920384 0 0.000288817 0.003138 0.524576 0.00204107 +4.50439 0.00312831 0.092581 0 0.000289622 0.00312831 0.528485 0.00204486 +4.50928 0.00311863 0.0931235 0 0.000290417 0.00311863 0.532406 0.00204861 +4.51416 0.00310896 0.093666 0 0.000291204 0.00310896 0.53634 0.0020523 +4.51904 0.00309931 0.0942086 0 0.000291981 0.00309931 0.540286 0.00205596 +4.52393 0.00308967 0.0947511 0 0.00029275 0.00308967 0.544244 0.00205956 +4.52881 0.00308005 0.0952936 0 0.000293509 0.00308005 0.548214 0.00206312 +4.53369 0.00307045 0.0958362 0 0.00029426 0.00307045 0.552197 0.00206664 +4.53857 0.00306086 0.0963787 0 0.000295001 0.00306086 0.556192 0.00207011 +4.54346 0.00305128 0.0969212 0 0.000295734 0.00305128 0.5602 0.00207353 +4.54834 0.00304172 0.0974638 0 0.000296457 0.00304172 0.564221 0.00207691 +4.55322 0.00303217 0.0980063 0 0.000297172 0.00303217 0.568254 0.00208025 +4.55811 0.00302264 0.0985488 0 0.000297878 0.00302264 0.5723 0.00208354 +4.56299 0.00301313 0.0990914 0 0.000298575 0.00301313 0.576358 0.00208679 +4.56787 0.00300363 0.0996339 0 0.000299263 0.00300363 0.58043 0.00209 +4.57275 0.00299414 0.100176 0 0.000299942 0.00299414 0.584514 0.00209316 +4.57764 0.00298467 0.100719 0 0.000300613 0.00298467 0.588611 0.00209628 +4.58252 0.00297521 0.101262 0 0.000301275 0.00297521 0.592722 0.00209935 +4.5874 0.00296577 0.101804 0 0.000301928 0.00296577 0.596845 0.00210239 +4.59229 0.00295635 0.102347 0 0.000302572 0.00295635 0.600981 0.00210538 +4.59717 0.00294694 0.102889 0 0.000303208 0.00294694 0.605131 0.00210832 +4.60205 0.00293754 0.103432 0 0.000303835 0.00293754 0.609294 0.00211123 +4.60693 0.00292816 0.103974 0 0.000304453 0.00292816 0.61347 0.00211409 +4.61182 0.00291879 0.104517 0 0.000305063 0.00291879 0.61766 0.00211691 +4.6167 0.00290944 0.105059 0 0.000305664 0.00290944 0.621863 0.00211969 +4.62158 0.00290011 0.105602 0 0.000306257 0.00290011 0.62608 0.00212243 +4.62646 0.00289079 0.106144 0 0.000306841 0.00289079 0.63031 0.00212513 +4.63135 0.00288148 0.106687 0 0.000307416 0.00288148 0.634554 0.00212779 +4.63623 0.00287219 0.107229 0 0.000307983 0.00287219 0.638811 0.0021304 +4.64111 0.00286291 0.107772 0 0.000308542 0.00286291 0.643083 0.00213298 +4.646 0.00285365 0.108314 0 0.000309092 0.00285365 0.647368 0.00213551 +4.65088 0.00284441 0.108857 0 0.000309634 0.00284441 0.651667 0.00213801 +4.65576 0.00283518 0.1094 0 0.000310167 0.00283518 0.65598 0.00214046 +4.66064 0.00282596 0.109942 0 0.000310692 0.00282596 0.660308 0.00214288 +4.66553 0.00281676 0.110485 0 0.000311209 0.00281676 0.664649 0.00214525 +4.67041 0.00280758 0.111027 0 0.000311717 0.00280758 0.669004 0.00214759 +4.67529 0.0027984 0.11157 0 0.000312217 0.0027984 0.673374 0.00214988 +4.68018 0.00278925 0.112112 0 0.000312709 0.00278925 0.677758 0.00215214 +4.68506 0.00278011 0.112655 0 0.000313192 0.00278011 0.682157 0.00215436 +4.68994 0.00277098 0.113197 0 0.000313668 0.00277098 0.68657 0.00215654 +4.69482 0.00276187 0.11374 0 0.000314135 0.00276187 0.690997 0.00215868 +4.69971 0.00275278 0.114282 0 0.000314594 0.00275278 0.695439 0.00216078 +4.70459 0.0027437 0.114825 0 0.000315045 0.0027437 0.699896 0.00216284 +4.70947 0.00273463 0.115367 0 0.000315487 0.00273463 0.704368 0.00216487 +4.71436 0.00272558 0.11591 0 0.000315922 0.00272558 0.708854 0.00216686 +4.71924 0.00271654 0.116452 0 0.000316348 0.00271654 0.713355 0.00216881 +4.72412 0.00270752 0.116995 0 0.000316767 0.00270752 0.717872 0.00217072 +4.729 0.00269852 0.117538 0 0.000317177 0.00269852 0.722403 0.00217259 +4.73389 0.00268953 0.11808 0 0.000317579 0.00268953 0.72695 0.00217443 +4.73877 0.00268055 0.118623 0 0.000317974 0.00268055 0.731511 0.00217623 +4.74365 0.00267159 0.119165 0 0.00031836 0.00267159 0.736088 0.00217799 +4.74854 0.00266264 0.119708 0 0.000318739 0.00266264 0.740681 0.00217972 +4.75342 0.00265371 0.12025 0 0.00031911 0.00265371 0.745288 0.00218141 +4.7583 0.0026448 0.120793 0 0.000319472 0.0026448 0.749912 0.00218306 +4.76318 0.0026359 0.121335 0 0.000319827 0.0026359 0.754551 0.00218468 +4.76807 0.00262701 0.121878 0 0.000320174 0.00262701 0.759205 0.00218626 +4.77295 0.00261814 0.12242 0 0.000320514 0.00261814 0.763876 0.0021878 +4.77783 0.00260929 0.122963 0 0.000320845 0.00260929 0.768562 0.00218931 +4.78271 0.00260044 0.123505 0 0.000321169 0.00260044 0.773264 0.00219078 +4.7876 0.00259162 0.124048 0 0.000321485 0.00259162 0.777982 0.00219222 +4.79248 0.00258281 0.124591 0 0.000321793 0.00258281 0.782716 0.00219362 +4.79736 0.00257401 0.125133 0 0.000322094 0.00257401 0.787467 0.00219499 +4.80225 0.00256523 0.125676 0 0.000322387 0.00256523 0.792233 0.00219632 +4.80713 0.00255647 0.126218 0 0.000322672 0.00255647 0.797016 0.00219761 +4.81201 0.00254772 0.126761 0 0.00032295 0.00254772 0.801816 0.00219888 +4.81689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01221 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01709 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02197 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02686 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03174 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03662 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04639 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05127 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05615 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06592 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07568 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08057 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08545 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09033 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10498 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10986 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11475 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11963 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12451 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13428 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13916 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14404 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14893 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15381 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15869 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16357 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16846 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17334 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17822 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18799 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19287 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19775 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20264 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20752 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22217 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22705 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23193 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23682 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24658 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25635 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26123 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26611 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27588 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28076 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28564 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29053 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29541 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30029 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30518 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31006 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31494 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31982 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32471 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32959 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33447 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33936 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34424 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34912 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35889 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36377 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36865 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37842 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38818 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39307 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39795 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40283 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41748 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42236 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42725 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43213 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43701 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44678 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45166 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45654 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46143 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46631 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47119 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47607 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48096 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48584 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49072 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50049 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50537 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51025 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51514 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52002 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53467 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53955 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54443 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54932 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55908 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56885 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57373 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57861 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58838 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59326 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59814 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60303 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60791 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61279 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61768 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62256 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62744 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63232 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63721 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64209 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64697 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65674 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66162 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67139 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67627 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68115 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69092 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70068 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70557 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71045 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71533 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72998 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73486 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73975 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74463 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74951 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75928 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76416 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76904 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77393 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77881 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78369 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78857 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79346 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79834 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80322 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80811 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81299 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81787 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82275 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82764 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83252 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84717 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85205 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85693 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86182 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87158 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88135 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88623 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89111 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90088 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90576 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91553 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92041 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92529 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93018 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93506 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93994 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94482 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94971 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95459 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95947 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96924 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97412 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98389 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98877 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99365 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00342 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01318 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01807 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02295 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02783 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04248 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04736 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05225 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05713 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06201 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06689 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07178 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07666 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08154 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08643 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09131 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09619 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10107 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10596 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11084 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11572 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12549 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13037 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13525 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14014 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14502 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15967 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16455 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16943 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17432 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18408 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19385 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19873 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20361 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21338 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21826 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22803 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23291 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23779 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24268 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24756 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25244 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25732 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26221 0.001 0 0 0 0.001 0 0 +6.26709 0.001 0 0 0 0.001 0 0 +6.27197 0.001 0 0 0 0.001 0 0 +6.27686 0.001 0 0 0 0.001 0 0 +6.28174 0.001 0 0 0 0.001 0 0 +6.28662 0.001 0 0 0 0.001 0 0 +6.2915 0.001 0 0 0 0.001 0 0 +6.29639 0.001 0 0 0 0.001 0 0 +6.30127 0.001 0 0 0 0.001 0 0 +6.30615 0.001 0 0 0 0.001 0 0 +6.31104 0.001 0 0 0 0.001 0 0 +6.31592 0.001 0 0 0 0.001 0 0 +6.3208 0.001 0 0 0 0.001 0 0 +6.32568 0.001 0 0 0 0.001 0 0 +6.33057 0.001 0 0 0 0.001 0 0 +6.33545 0.001 0 0 0 0.001 0 0 +6.34033 0.001 0 0 0 0.001 0 0 +6.34521 0.001 0 0 0 0.001 0 0 +6.3501 0.001 0 0 0 0.001 0 0 +6.35498 0.001 0 0 0 0.001 0 0 +6.35986 0.001 0 0 0 0.001 0 0 +6.36475 0.001 0 0 0 0.001 0 0 +6.36963 0.001 0 0 0 0.001 0 0 +6.37451 0.001 0 0 0 0.001 0 0 +6.37939 0.001 0 0 0 0.001 0 0 +6.38428 0.001 0 0 0 0.001 0 0 +6.38916 0.001 0 0 0 0.001 0 0 +6.39404 0.001 0 0 0 0.001 0 0 +6.39893 0.001 0 0 0 0.001 0 0 +6.40381 0.001 0 0 0 0.001 0 0 +6.40869 0.001 0 0 0 0.001 0 0 +6.41357 0.001 0 0 0 0.001 0 0 +6.41846 0.001 0 0 0 0.001 0 0 +6.42334 0.001 0 0 0 0.001 0 0 +6.42822 0.001 0 0 0 0.001 0 0 +6.43311 0.001 0 0 0 0.001 0 0 +6.43799 0.001 0 0 0 0.001 0 0 +6.44287 0.001 0 0 0 0.001 0 0 +6.44775 0.001 0 0 0 0.001 0 0 +6.45264 0.001 0 0 0 0.001 0 0 +6.45752 0.001 0 0 0 0.001 0 0 +6.4624 0.001 0 0 0 0.001 0 0 +6.46729 0.001 0 0 0 0.001 0 0 +6.47217 0.001 0 0 0 0.001 0 0 +6.47705 0.001 0 0 0 0.001 0 0 +6.48193 0.001 0 0 0 0.001 0 0 +6.48682 0.001 0 0 0 0.001 0 0 +6.4917 0.001 0 0 0 0.001 0 0 +6.49658 0.001 0 0 0 0.001 0 0 +6.50146 0.001 0 0 0 0.001 0 0 +6.50635 0.001 0 0 0 0.001 0 0 +6.51123 0.001 0 0 0 0.001 0 0 +6.51611 0.001 0 0 0 0.001 0 0 +6.521 0.001 0 0 0 0.001 0 0 +6.52588 0.001 0 0 0 0.001 0 0 +6.53076 0.001 0 0 0 0.001 0 0 +6.53564 0.001 0 0 0 0.001 0 0 +6.54053 0.001 0 0 0 0.001 0 0 +6.54541 0.001 0 0 0 0.001 0 0 +6.55029 0.001 0 0 0 0.001 0 0 +6.55518 0.001 0 0 0 0.001 0 0 +6.56006 0.001 0 0 0 0.001 0 0 +6.56494 0.001 0 0 0 0.001 0 0 +6.56982 0.001 0 0 0 0.001 0 0 +6.57471 0.001 0 0 0 0.001 0 0 +6.57959 0.001 0 0 0 0.001 0 0 +6.58447 0.001 0 0 0 0.001 0 0 +6.58936 0.001 0 0 0 0.001 0 0 +6.59424 0.001 0 0 0 0.001 0 0 +6.59912 0.001 0 0 0 0.001 0 0 +6.604 0.001 0 0 0 0.001 0 0 +6.60889 0.001 0 0 0 0.001 0 0 +6.61377 0.001 0 0 0 0.001 0 0 +6.61865 0.001 0 0 0 0.001 0 0 +6.62354 0.001 0 0 0 0.001 0 0 +6.62842 0.001 0 0 0 0.001 0 0 +6.6333 0.001 0 0 0 0.001 0 0 +6.63818 0.001 0 0 0 0.001 0 0 +6.64307 0.001 0 0 0 0.001 0 0 +6.64795 0.001 0 0 0 0.001 0 0 +6.65283 0.001 0 0 0 0.001 0 0 +6.65771 0.001 0 0 0 0.001 0 0 +6.6626 0.001 0 0 0 0.001 0 0 +6.66748 0.001 0 0 0 0.001 0 0 +6.67236 0.001 0 0 0 0.001 0 0 +6.67725 0.001 0 0 0 0.001 0 0 +6.68213 0.001 0 0 0 0.001 0 0 +6.68701 0.001 0 0 0 0.001 0 0 +6.69189 0.001 0 0 0 0.001 0 0 +6.69678 0.001 0 0 0 0.001 0 0 +6.70166 0.001 0 0 0 0.001 0 0 +6.70654 0.001 0 0 0 0.001 0 0 +6.71143 0.001 0 0 0 0.001 0 0 +6.71631 0.001 0 0 0 0.001 0 0 +6.72119 0.001 0 0 0 0.001 0 0 +6.72607 0.001 0 0 0 0.001 0 0 +6.73096 0.001 0 0 0 0.001 0 0 +6.73584 0.001 0 0 0 0.001 0 0 +6.74072 0.001 0 0 0 0.001 0 0 +6.74561 0.001 0 0 0 0.001 0 0 +6.75049 0.001 0 0 0 0.001 0 0 +6.75537 0.001 0 0 0 0.001 0 0 +6.76025 0.001 0 0 0 0.001 0 0 +6.76514 0.001 0 0 0 0.001 0 0 +6.77002 0.001 0 0 0 0.001 0 0 +6.7749 0.001 0 0 0 0.001 0 0 +6.77979 0.001 0 0 0 0.001 0 0 +6.78467 0.001 0 0 0 0.001 0 0 +6.78955 0.001 0 0 0 0.001 0 0 +6.79443 0.001 0 0 0 0.001 0 0 +6.79932 0.001 0 0 0 0.001 0 0 +6.8042 0.001 0 0 0 0.001 0 0 +6.80908 0.001 0 0 0 0.001 0 0 +6.81396 0.001 0 0 0 0.001 0 0 +6.81885 0.001 0 0 0 0.001 0 0 +6.82373 0.001 0 0 0 0.001 0 0 +6.82861 0.001 0 0 0 0.001 0 0 +6.8335 0.001 0 0 0 0.001 0 0 +6.83838 0.001 0 0 0 0.001 0 0 +6.84326 0.001 0 0 0 0.001 0 0 +6.84814 0.001 0 0 0 0.001 0 0 +6.85303 0.001 0 0 0 0.001 0 0 +6.85791 0.001 0 0 0 0.001 0 0 +6.86279 0.001 0 0 0 0.001 0 0 +6.86768 0.001 0 0 0 0.001 0 0 +6.87256 0.001 0 0 0 0.001 0 0 +6.87744 0.001 0 0 0 0.001 0 0 +6.88232 0.001 0 0 0 0.001 0 0 +6.88721 0.001 0 0 0 0.001 0 0 +6.89209 0.001 0 0 0 0.001 0 0 +6.89697 0.001 0 0 0 0.001 0 0 +6.90186 0.001 0 0 0 0.001 0 0 +6.90674 0.001 0 0 0 0.001 0 0 +6.91162 0.001 0 0 0 0.001 0 0 +6.9165 0.001 0 0 0 0.001 0 0 +6.92139 0.001 0 0 0 0.001 0 0 +6.92627 0.001 0 0 0 0.001 0 0 +6.93115 0.001 0 0 0 0.001 0 0 +6.93604 0.001 0 0 0 0.001 0 0 +6.94092 0.001 0 0 0 0.001 0 0 +6.9458 0.001 0 0 0 0.001 0 0 +6.95068 0.001 0 0 0 0.001 0 0 +6.95557 0.001 0 0 0 0.001 0 0 +6.96045 0.001 0 0 0 0.001 0 0 +6.96533 0.001 0 0 0 0.001 0 0 +6.97021 0.001 0 0 0 0.001 0 0 +6.9751 0.001 0 0 0 0.001 0 0 +6.97998 0.001 0 0 0 0.001 0 0 +6.98486 0.001 0 0 0 0.001 0 0 +6.98975 0.001 0 0 0 0.001 0 0 +6.99463 0.001 0 0 0 0.001 0 0 +6.99951 0.001 0 0 0 0.001 0 0 +7.00439 0.001 0 0 0 0.001 0 0 +7.00928 0.001 0 0 0 0.001 0 0 +7.01416 0.001 0 0 0 0.001 0 0 +7.01904 0.001 0 0 0 0.001 0 0 +7.02393 0.001 0 0 0 0.001 0 0 +7.02881 0.001 0 0 0 0.001 0 0 +7.03369 0.001 0 0 0 0.001 0 0 +7.03857 0.001 0 0 0 0.001 0 0 +7.04346 0.001 0 0 0 0.001 0 0 +7.04834 0.001 0 0 0 0.001 0 0 +7.05322 0.001 0 0 0 0.001 0 0 +7.05811 0.001 0 0 0 0.001 0 0 +7.06299 0.001 0 0 0 0.001 0 0 +7.06787 0.001 0 0 0 0.001 0 0 +7.07275 0.001 0 0 0 0.001 0 0 +7.07764 0.001 0 0 0 0.001 0 0 +7.08252 0.001 0 0 0 0.001 0 0 +7.0874 0.001 0 0 0 0.001 0 0 +7.09229 0.001 0 0 0 0.001 0 0 +7.09717 0.001 0 0 0 0.001 0 0 +7.10205 0.001 0 0 0 0.001 0 0 +7.10693 0.001 0 0 0 0.001 0 0 +7.11182 0.001 0 0 0 0.001 0 0 +7.1167 0.001 0 0 0 0.001 0 0 +7.12158 0.001 0 0 0 0.001 0 0 +7.12646 0.001 0 0 0 0.001 0 0 +7.13135 0.001 0 0 0 0.001 0 0 +7.13623 0.001 0 0 0 0.001 0 0 +7.14111 0.001 0 0 0 0.001 0 0 +7.146 0.001 0 0 0 0.001 0 0 +7.15088 0.001 0 0 0 0.001 0 0 +7.15576 0.001 0 0 0 0.001 0 0 +7.16064 0.001 0 0 0 0.001 0 0 +7.16553 0.001 0 0 0 0.001 0 0 +7.17041 0.001 0 0 0 0.001 0 0 +7.17529 0.001 0 0 0 0.001 0 0 +7.18018 0.001 0 0 0 0.001 0 0 +7.18506 0.001 0 0 0 0.001 0 0 +7.18994 0.001 0 0 0 0.001 0 0 +7.19482 0.001 0 0 0 0.001 0 0 +7.19971 0.001 0 0 0 0.001 0 0 +7.20459 0.001 0 0 0 0.001 0 0 +7.20947 0.001 0 0 0 0.001 0 0 +7.21436 0.001 0 0 0 0.001 0 0 +7.21924 0.001 0 0 0 0.001 0 0 +7.22412 0.001 0 0 0 0.001 0 0 +7.229 0.001 0 0 0 0.001 0 0 +7.23389 0.001 0 0 0 0.001 0 0 +7.23877 0.001 0 0 0 0.001 0 0 +7.24365 0.001 0 0 0 0.001 0 0 +7.24854 0.001 0 0 0 0.001 0 0 +7.25342 0.001 0 0 0 0.001 0 0 +7.2583 0.001 0 0 0 0.001 0 0 +7.26318 0.001 0 0 0 0.001 0 0 +7.26807 0.001 0 0 0 0.001 0 0 +7.27295 0.001 0 0 0 0.001 0 0 +7.27783 0.001 0 0 0 0.001 0 0 +7.28271 0.001 0 0 0 0.001 0 0 +7.2876 0.001 0 0 0 0.001 0 0 +7.29248 0.001 0 0 0 0.001 0 0 +7.29736 0.001 0 0 0 0.001 0 0 +7.30225 0.001 0 0 0 0.001 0 0 +7.30713 0.001 0 0 0 0.001 0 0 +7.31201 0.001 0 0 0 0.001 0 0 +7.31689 0.001 0 0 0 0.001 0 0 +7.32178 0.001 0 0 0 0.001 0 0 +7.32666 0.001 0 0 0 0.001 0 0 +7.33154 0.001 0 0 0 0.001 0 0 +7.33643 0.001 0 0 0 0.001 0 0 +7.34131 0.001 0 0 0 0.001 0 0 +7.34619 0.001 0 0 0 0.001 0 0 +7.35107 0.001 0 0 0 0.001 0 0 +7.35596 0.001 0 0 0 0.001 0 0 +7.36084 0.001 0 0 0 0.001 0 0 +7.36572 0.001 0 0 0 0.001 0 0 +7.37061 0.001 0 0 0 0.001 0 0 +7.37549 0.001 0 0 0 0.001 0 0 +7.38037 0.001 0 0 0 0.001 0 0 +7.38525 0.001 0 0 0 0.001 0 0 +7.39014 0.001 0 0 0 0.001 0 0 +7.39502 0.001 0 0 0 0.001 0 0 +7.3999 0.001 0 0 0 0.001 0 0 +7.40479 0.001 0 0 0 0.001 0 0 +7.40967 0.001 0 0 0 0.001 0 0 +7.41455 0.001 0 0 0 0.001 0 0 +7.41943 0.001 0 0 0 0.001 0 0 +7.42432 0.001 0 0 0 0.001 0 0 +7.4292 0.001 0 0 0 0.001 0 0 +7.43408 0.001 0 0 0 0.001 0 0 +7.43896 0.001 0 0 0 0.001 0 0 +7.44385 0.001 0 0 0 0.001 0 0 +7.44873 0.001 0 0 0 0.001 0 0 +7.45361 0.001 0 0 0 0.001 0 0 +7.4585 0.001 0 0 0 0.001 0 0 +7.46338 0.001 0 0 0 0.001 0 0 +7.46826 0.001 0 0 0 0.001 0 0 +7.47314 0.001 0 0 0 0.001 0 0 +7.47803 0.001 0 0 0 0.001 0 0 +7.48291 0.001 0 0 0 0.001 0 0 +7.48779 0.001 0 0 0 0.001 0 0 +7.49268 0.001 0 0 0 0.001 0 0 +7.49756 0.001 0 0 0 0.001 0 0 +7.50244 0.001 0 0 0 0.001 0 0 +7.50732 0.001 0 0 0 0.001 0 0 +7.51221 0.001 0 0 0 0.001 0 0 +7.51709 0.001 0 0 0 0.001 0 0 +7.52197 0.001 0 0 0 0.001 0 0 +7.52686 0.001 0 0 0 0.001 0 0 +7.53174 0.001 0 0 0 0.001 0 0 +7.53662 0.001 0 0 0 0.001 0 0 +7.5415 0.001 0 0 0 0.001 0 0 +7.54639 0.001 0 0 0 0.001 0 0 +7.55127 0.001 0 0 0 0.001 0 0 +7.55615 0.001 0 0 0 0.001 0 0 +7.56104 0.001 0 0 0 0.001 0 0 +7.56592 0.001 0 0 0 0.001 0 0 +7.5708 0.001 0 0 0 0.001 0 0 +7.57568 0.001 0 0 0 0.001 0 0 +7.58057 0.001 0 0 0 0.001 0 0 +7.58545 0.001 0 0 0 0.001 0 0 +7.59033 0.001 0 0 0 0.001 0 0 +7.59521 0.001 0 0 0 0.001 0 0 +7.6001 0.001 0 0 0 0.001 0 0 +7.60498 0.001 0 0 0 0.001 0 0 +7.60986 0.001 0 0 0 0.001 0 0 +7.61475 0.001 0 0 0 0.001 0 0 +7.61963 0.001 0 0 0 0.001 0 0 +7.62451 0.001 0 0 0 0.001 0 0 +7.62939 0.001 0 0 0 0.001 0 0 +7.63428 0.001 0 0 0 0.001 0 0 +7.63916 0.001 0 0 0 0.001 0 0 +7.64404 0.001 0 0 0 0.001 0 0 +7.64893 0.001 0 0 0 0.001 0 0 +7.65381 0.001 0 0 0 0.001 0 0 +7.65869 0.001 0 0 0 0.001 0 0 +7.66357 0.001 0 0 0 0.001 0 0 +7.66846 0.001 0 0 0 0.001 0 0 +7.67334 0.001 0 0 0 0.001 0 0 +7.67822 0.001 0 0 0 0.001 0 0 +7.68311 0.001 0 0 0 0.001 0 0 +7.68799 0.001 0 0 0 0.001 0 0 +7.69287 0.001 0 0 0 0.001 0 0 +7.69775 0.001 0 0 0 0.001 0 0 +7.70264 0.001 0 0 0 0.001 0 0 +7.70752 0.001 0 0 0 0.001 0 0 +7.7124 0.001 0 0 0 0.001 0 0 +7.71729 0.001 0 0 0 0.001 0 0 +7.72217 0.001 0 0 0 0.001 0 0 +7.72705 0.001 0 0 0 0.001 0 0 +7.73193 0.001 0 0 0 0.001 0 0 +7.73682 0.001 0 0 0 0.001 0 0 +7.7417 0.001 0 0 0 0.001 0 0 +7.74658 0.001 0 0 0 0.001 0 0 +7.75146 0.001 0 0 0 0.001 0 0 +7.75635 0.001 0 0 0 0.001 0 0 +7.76123 0.001 0 0 0 0.001 0 0 +7.76611 0.001 0 0 0 0.001 0 0 +7.771 0.001 0 0 0 0.001 0 0 +7.77588 0.001 0 0 0 0.001 0 0 +7.78076 0.001 0 0 0 0.001 0 0 +7.78564 0.001 0 0 0 0.001 0 0 +7.79053 0.001 0 0 0 0.001 0 0 +7.79541 0.001 0 0 0 0.001 0 0 +7.80029 0.001 0 0 0 0.001 0 0 +7.80518 0.001 0 0 0 0.001 0 0 +7.81006 0.001 0 0 0 0.001 0 0 +7.81494 0.001 0 0 0 0.001 0 0 +7.81982 0.001 0 0 0 0.001 0 0 +7.82471 0.001 0 0 0 0.001 0 0 +7.82959 0.001 0 0 0 0.001 0 0 +7.83447 0.001 0 0 0 0.001 0 0 +7.83936 0.001 0 0 0 0.001 0 0 +7.84424 0.001 0 0 0 0.001 0 0 +7.84912 0.001 0 0 0 0.001 0 0 +7.854 0.001 0 0 0 0.001 0 0 +7.85889 0.001 0 0 0 0.001 0 0 +7.86377 0.001 0 0 0 0.001 0 0 +7.86865 0.001 0 0 0 0.001 0 0 +7.87354 0.001 0 0 0 0.001 0 0 +7.87842 0.001 0 0 0 0.001 0 0 +7.8833 0.001 0 0 0 0.001 0 0 +7.88818 0.001 0 0 0 0.001 0 0 +7.89307 0.001 0 0 0 0.001 0 0 +7.89795 0.001 0 0 0 0.001 0 0 +7.90283 0.001 0 0 0 0.001 0 0 +7.90771 0.001 0 0 0 0.001 0 0 +7.9126 0.001 0 0 0 0.001 0 0 +7.91748 0.001 0 0 0 0.001 0 0 +7.92236 0.001 0 0 0 0.001 0 0 +7.92725 0.001 0 0 0 0.001 0 0 +7.93213 0.001 0 0 0 0.001 0 0 +7.93701 0.001 0 0 0 0.001 0 0 +7.94189 0.001 0 0 0 0.001 0 0 +7.94678 0.001 0 0 0 0.001 0 0 +7.95166 0.001 0 0 0 0.001 0 0 +7.95654 0.001 0 0 0 0.001 0 0 +7.96143 0.001 0 0 0 0.001 0 0 +7.96631 0.001 0 0 0 0.001 0 0 +7.97119 0.001 0 0 0 0.001 0 0 +7.97607 0.001 0 0 0 0.001 0 0 +7.98096 0.001 0 0 0 0.001 0 0 +7.98584 0.001 0 0 0 0.001 0 0 +7.99072 0.001 0 0 0 0.001 0 0 +7.99561 0.001 0 0 0 0.001 0 0 +8.00049 0.001 0 0 0 0.001 0 0 +8.00537 0.001 0 0 0 0.001 0 0 +8.01025 0.001 0 0 0 0.001 0 0 +8.01514 0.001 0 0 0 0.001 0 0 +8.02002 0.001 0 0 0 0.001 0 0 +8.0249 0.001 0 0 0 0.001 0 0 +8.02979 0.001 0 0 0 0.001 0 0 +8.03467 0.001 0 0 0 0.001 0 0 +8.03955 0.001 0 0 0 0.001 0 0 +8.04443 0.001 0 0 0 0.001 0 0 +8.04932 0.001 0 0 0 0.001 0 0 +8.0542 0.001 0 0 0 0.001 0 0 +8.05908 0.001 0 0 0 0.001 0 0 +8.06396 0.001 0 0 0 0.001 0 0 +8.06885 0.001 0 0 0 0.001 0 0 +8.07373 0.001 0 0 0 0.001 0 0 +8.07861 0.001 0 0 0 0.001 0 0 +8.0835 0.001 0 0 0 0.001 0 0 +8.08838 0.001 0 0 0 0.001 0 0 +8.09326 0.001 0 0 0 0.001 0 0 +8.09814 0.001 0 0 0 0.001 0 0 +8.10303 0.001 0 0 0 0.001 0 0 +8.10791 0.001 0 0 0 0.001 0 0 +8.11279 0.001 0 0 0 0.001 0 0 +8.11768 0.001 0 0 0 0.001 0 0 +8.12256 0.001 0 0 0 0.001 0 0 +8.12744 0.001 0 0 0 0.001 0 0 +8.13232 0.001 0 0 0 0.001 0 0 +8.13721 0.001 0 0 0 0.001 0 0 +8.14209 0.001 0 0 0 0.001 0 0 +8.14697 0.001 0 0 0 0.001 0 0 +8.15186 0.001 0 0 0 0.001 0 0 +8.15674 0.001 0 0 0 0.001 0 0 +8.16162 0.001 0 0 0 0.001 0 0 +8.1665 0.001 0 0 0 0.001 0 0 +8.17139 0.001 0 0 0 0.001 0 0 +8.17627 0.001 0 0 0 0.001 0 0 +8.18115 0.001 0 0 0 0.001 0 0 +8.18604 0.001 0 0 0 0.001 0 0 +8.19092 0.001 0 0 0 0.001 0 0 +8.1958 0.001 0 0 0 0.001 0 0 +8.20068 0.001 0 0 0 0.001 0 0 +8.20557 0.001 0 0 0 0.001 0 0 +8.21045 0.001 0 0 0 0.001 0 0 +8.21533 0.001 0 0 0 0.001 0 0 +8.22021 0.001 0 0 0 0.001 0 0 +8.2251 0.001 0 0 0 0.001 0 0 +8.22998 0.001 0 0 0 0.001 0 0 +8.23486 0.001 0 0 0 0.001 0 0 +8.23975 0.001 0 0 0 0.001 0 0 +8.24463 0.001 0 0 0 0.001 0 0 +8.24951 0.001 0 0 0 0.001 0 0 +8.25439 0.001 0 0 0 0.001 0 0 +8.25928 0.001 0 0 0 0.001 0 0 +8.26416 0.001 0 0 0 0.001 0 0 +8.26904 0.001 0 0 0 0.001 0 0 +8.27393 0.001 0 0 0 0.001 0 0 +8.27881 0.001 0 0 0 0.001 0 0 +8.28369 0.001 0 0 0 0.001 0 0 +8.28857 0.001 0 0 0 0.001 0 0 +8.29346 0.001 0 0 0 0.001 0 0 +8.29834 0.001 0 0 0 0.001 0 0 +8.30322 0.001 0 0 0 0.001 0 0 +8.30811 0.001 0 0 0 0.001 0 0 +8.31299 0.001 0 0 0 0.001 0 0 +8.31787 0.001 0 0 0 0.001 0 0 +8.32275 0.001 0 0 0 0.001 0 0 +8.32764 0.001 0 0 0 0.001 0 0 +8.33252 0.001 0 0 0 0.001 0 0 +8.3374 0.001 0 0 0 0.001 0 0 +8.34229 0.001 0 0 0 0.001 0 0 +8.34717 0.001 0 0 0 0.001 0 0 +8.35205 0.001 0 0 0 0.001 0 0 +8.35693 0.001 0 0 0 0.001 0 0 +8.36182 0.001 0 0 0 0.001 0 0 +8.3667 0.001 0 0 0 0.001 0 0 +8.37158 0.001 0 0 0 0.001 0 0 +8.37646 0.001 0 0 0 0.001 0 0 +8.38135 0.001 0 0 0 0.001 0 0 +8.38623 0.001 0 0 0 0.001 0 0 +8.39111 0.001 0 0 0 0.001 0 0 +8.396 0.001 0 0 0 0.001 0 0 +8.40088 0.001 0 0 0 0.001 0 0 +8.40576 0.001 0 0 0 0.001 0 0 +8.41064 0.001 0 0 0 0.001 0 0 +8.41553 0.001 0 0 0 0.001 0 0 +8.42041 0.001 0 0 0 0.001 0 0 +8.42529 0.001 0 0 0 0.001 0 0 +8.43018 0.001 0 0 0 0.001 0 0 +8.43506 0.001 0 0 0 0.001 0 0 +8.43994 0.001 0 0 0 0.001 0 0 +8.44482 0.001 0 0 0 0.001 0 0 +8.44971 0.001 0 0 0 0.001 0 0 +8.45459 0.001 0 0 0 0.001 0 0 +8.45947 0.001 0 0 0 0.001 0 0 +8.46436 0.001 0 0 0 0.001 0 0 +8.46924 0.001 0 0 0 0.001 0 0 +8.47412 0.001 0 0 0 0.001 0 0 +8.479 0.001 0 0 0 0.001 0 0 +8.48389 0.001 0 0 0 0.001 0 0 +8.48877 0.001 0 0 0 0.001 0 0 +8.49365 0.001 0 0 0 0.001 0 0 +8.49854 0.001 0 0 0 0.001 0 0 +8.50342 0.001 0 0 0 0.001 0 0 +8.5083 0.001 0 0 0 0.001 0 0 +8.51318 0.001 0 0 0 0.001 0 0 +8.51807 0.001 0 0 0 0.001 0 0 +8.52295 0.001 0 0 0 0.001 0 0 +8.52783 0.001 0 0 0 0.001 0 0 +8.53271 0.001 0 0 0 0.001 0 0 +8.5376 0.001 0 0 0 0.001 0 0 +8.54248 0.001 0 0 0 0.001 0 0 +8.54736 0.001 0 0 0 0.001 0 0 +8.55225 0.001 0 0 0 0.001 0 0 +8.55713 0.001 0 0 0 0.001 0 0 +8.56201 0.001 0 0 0 0.001 0 0 +8.56689 0.001 0 0 0 0.001 0 0 +8.57178 0.001 0 0 0 0.001 0 0 +8.57666 0.001 0 0 0 0.001 0 0 +8.58154 0.001 0 0 0 0.001 0 0 +8.58643 0.001 0 0 0 0.001 0 0 +8.59131 0.001 0 0 0 0.001 0 0 +8.59619 0.001 0 0 0 0.001 0 0 +8.60107 0.001 0 0 0 0.001 0 0 +8.60596 0.001 0 0 0 0.001 0 0 +8.61084 0.001 0 0 0 0.001 0 0 +8.61572 0.001 0 0 0 0.001 0 0 +8.62061 0.001 0 0 0 0.001 0 0 +8.62549 0.001 0 0 0 0.001 0 0 +8.63037 0.001 0 0 0 0.001 0 0 +8.63525 0.001 0 0 0 0.001 0 0 +8.64014 0.001 0 0 0 0.001 0 0 +8.64502 0.001 0 0 0 0.001 0 0 +8.6499 0.001 0 0 0 0.001 0 0 +8.65479 0.001 0 0 0 0.001 0 0 +8.65967 0.001 0 0 0 0.001 0 0 +8.66455 0.001 0 0 0 0.001 0 0 +8.66943 0.001 0 0 0 0.001 0 0 +8.67432 0.001 0 0 0 0.001 0 0 +8.6792 0.001 0 0 0 0.001 0 0 +8.68408 0.001 0 0 0 0.001 0 0 +8.68896 0.001 0 0 0 0.001 0 0 +8.69385 0.001 0 0 0 0.001 0 0 +8.69873 0.001 0 0 0 0.001 0 0 +8.70361 0.001 0 0 0 0.001 0 0 +8.7085 0.001 0 0 0 0.001 0 0 +8.71338 0.001 0 0 0 0.001 0 0 +8.71826 0.001 0 0 0 0.001 0 0 +8.72314 0.001 0 0 0 0.001 0 0 +8.72803 0.001 0 0 0 0.001 0 0 +8.73291 0.001 0 0 0 0.001 0 0 +8.73779 0.001 0 0 0 0.001 0 0 +8.74268 0.001 0 0 0 0.001 0 0 +8.74756 0.001 0 0 0 0.001 0 0 +8.75244 0.001 0 0 0 0.001 0 0 +8.75732 0.001 0 0 0 0.001 0 0 +8.76221 0.001 0 0 0 0.001 0 0 +8.76709 0.001 0 0 0 0.001 0 0 +8.77197 0.001 0 0 0 0.001 0 0 +8.77686 0.001 0 0 0 0.001 0 0 +8.78174 0.001 0 0 0 0.001 0 0 +8.78662 0.001 0 0 0 0.001 0 0 +8.7915 0.001 0 0 0 0.001 0 0 +8.79639 0.001 0 0 0 0.001 0 0 +8.80127 0.001 0 0 0 0.001 0 0 +8.80615 0.001 0 0 0 0.001 0 0 +8.81104 0.001 0 0 0 0.001 0 0 +8.81592 0.001 0 0 0 0.001 0 0 +8.8208 0.001 0 0 0 0.001 0 0 +8.82568 0.001 0 0 0 0.001 0 0 +8.83057 0.001 0 0 0 0.001 0 0 +8.83545 0.001 0 0 0 0.001 0 0 +8.84033 0.001 0 0 0 0.001 0 0 +8.84521 0.001 0 0 0 0.001 0 0 +8.8501 0.001 0 0 0 0.001 0 0 +8.85498 0.001 0 0 0 0.001 0 0 +8.85986 0.001 0 0 0 0.001 0 0 +8.86475 0.001 0 0 0 0.001 0 0 +8.86963 0.001 0 0 0 0.001 0 0 +8.87451 0.001 0 0 0 0.001 0 0 +8.87939 0.001 0 0 0 0.001 0 0 +8.88428 0.001 0 0 0 0.001 0 0 +8.88916 0.001 0 0 0 0.001 0 0 +8.89404 0.001 0 0 0 0.001 0 0 +8.89893 0.001 0 0 0 0.001 0 0 +8.90381 0.001 0 0 0 0.001 0 0 +8.90869 0.001 0 0 0 0.001 0 0 +8.91357 0.001 0 0 0 0.001 0 0 +8.91846 0.001 0 0 0 0.001 0 0 +8.92334 0.001 0 0 0 0.001 0 0 +8.92822 0.001 0 0 0 0.001 0 0 +8.93311 0.001 0 0 0 0.001 0 0 +8.93799 0.001 0 0 0 0.001 0 0 +8.94287 0.001 0 0 0 0.001 0 0 +8.94775 0.001 0 0 0 0.001 0 0 +8.95264 0.001 0 0 0 0.001 0 0 +8.95752 0.001 0 0 0 0.001 0 0 +8.9624 0.001 0 0 0 0.001 0 0 +8.96729 0.001 0 0 0 0.001 0 0 +8.97217 0.001 0 0 0 0.001 0 0 +8.97705 0.001 0 0 0 0.001 0 0 +8.98193 0.001 0 0 0 0.001 0 0 +8.98682 0.001 0 0 0 0.001 0 0 +8.9917 0.001 0 0 0 0.001 0 0 +8.99658 0.001 0 0 0 0.001 0 0 +9.00146 0.001 0 0 0 0.001 0 0 +9.00635 0.001 0 0 0 0.001 0 0 +9.01123 0.001 0 0 0 0.001 0 0 +9.01611 0.001 0 0 0 0.001 0 0 +9.021 0.001 0 0 0 0.001 0 0 +9.02588 0.001 0 0 0 0.001 0 0 +9.03076 0.001 0 0 0 0.001 0 0 +9.03564 0.001 0 0 0 0.001 0 0 +9.04053 0.001 0 0 0 0.001 0 0 +9.04541 0.001 0 0 0 0.001 0 0 +9.05029 0.001 0 0 0 0.001 0 0 +9.05518 0.001 0 0 0 0.001 0 0 +9.06006 0.001 0 0 0 0.001 0 0 +9.06494 0.001 0 0 0 0.001 0 0 +9.06982 0.001 0 0 0 0.001 0 0 +9.07471 0.001 0 0 0 0.001 0 0 +9.07959 0.001 0 0 0 0.001 0 0 +9.08447 0.001 0 0 0 0.001 0 0 +9.08936 0.001 0 0 0 0.001 0 0 +9.09424 0.001 0 0 0 0.001 0 0 +9.09912 0.001 0 0 0 0.001 0 0 +9.104 0.001 0 0 0 0.001 0 0 +9.10889 0.001 0 0 0 0.001 0 0 +9.11377 0.001 0 0 0 0.001 0 0 +9.11865 0.001 0 0 0 0.001 0 0 +9.12354 0.001 0 0 0 0.001 0 0 +9.12842 0.001 0 0 0 0.001 0 0 +9.1333 0.001 0 0 0 0.001 0 0 +9.13818 0.001 0 0 0 0.001 0 0 +9.14307 0.001 0 0 0 0.001 0 0 +9.14795 0.001 0 0 0 0.001 0 0 +9.15283 0.001 0 0 0 0.001 0 0 +9.15771 0.001 0 0 0 0.001 0 0 +9.1626 0.001 0 0 0 0.001 0 0 +9.16748 0.001 0 0 0 0.001 0 0 +9.17236 0.001 0 0 0 0.001 0 0 +9.17725 0.001 0 0 0 0.001 0 0 +9.18213 0.001 0 0 0 0.001 0 0 +9.18701 0.001 0 0 0 0.001 0 0 +9.19189 0.001 0 0 0 0.001 0 0 +9.19678 0.001 0 0 0 0.001 0 0 +9.20166 0.001 0 0 0 0.001 0 0 +9.20654 0.001 0 0 0 0.001 0 0 +9.21143 0.001 0 0 0 0.001 0 0 +9.21631 0.001 0 0 0 0.001 0 0 +9.22119 0.001 0 0 0 0.001 0 0 +9.22607 0.001 0 0 0 0.001 0 0 +9.23096 0.001 0 0 0 0.001 0 0 +9.23584 0.001 0 0 0 0.001 0 0 +9.24072 0.001 0 0 0 0.001 0 0 +9.24561 0.001 0 0 0 0.001 0 0 +9.25049 0.001 0 0 0 0.001 0 0 +9.25537 0.001 0 0 0 0.001 0 0 +9.26025 0.001 0 0 0 0.001 0 0 +9.26514 0.001 0 0 0 0.001 0 0 +9.27002 0.001 0 0 0 0.001 0 0 +9.2749 0.001 0 0 0 0.001 0 0 +9.27979 0.001 0 0 0 0.001 0 0 +9.28467 0.001 0 0 0 0.001 0 0 +9.28955 0.001 0 0 0 0.001 0 0 +9.29443 0.001 0 0 0 0.001 0 0 +9.29932 0.001 0 0 0 0.001 0 0 +9.3042 0.001 0 0 0 0.001 0 0 +9.30908 0.001 0 0 0 0.001 0 0 +9.31396 0.001 0 0 0 0.001 0 0 +9.31885 0.001 0 0 0 0.001 0 0 +9.32373 0.001 0 0 0 0.001 0 0 +9.32861 0.001 0 0 0 0.001 0 0 +9.3335 0.001 0 0 0 0.001 0 0 +9.33838 0.001 0 0 0 0.001 0 0 +9.34326 0.001 0 0 0 0.001 0 0 +9.34814 0.001 0 0 0 0.001 0 0 +9.35303 0.001 0 0 0 0.001 0 0 +9.35791 0.001 0 0 0 0.001 0 0 +9.36279 0.001 0 0 0 0.001 0 0 +9.36768 0.001 0 0 0 0.001 0 0 +9.37256 0.001 0 0 0 0.001 0 0 +9.37744 0.001 0 0 0 0.001 0 0 +9.38232 0.001 0 0 0 0.001 0 0 +9.38721 0.001 0 0 0 0.001 0 0 +9.39209 0.001 0 0 0 0.001 0 0 +9.39697 0.001 0 0 0 0.001 0 0 +9.40186 0.001 0 0 0 0.001 0 0 +9.40674 0.001 0 0 0 0.001 0 0 +9.41162 0.001 0 0 0 0.001 0 0 +9.4165 0.001 0 0 0 0.001 0 0 +9.42139 0.001 0 0 0 0.001 0 0 +9.42627 0.001 0 0 0 0.001 0 0 +9.43115 0.001 0 0 0 0.001 0 0 +9.43604 0.001 0 0 0 0.001 0 0 +9.44092 0.001 0 0 0 0.001 0 0 +9.4458 0.001 0 0 0 0.001 0 0 +9.45068 0.001 0 0 0 0.001 0 0 +9.45557 0.001 0 0 0 0.001 0 0 +9.46045 0.001 0 0 0 0.001 0 0 +9.46533 0.001 0 0 0 0.001 0 0 +9.47021 0.001 0 0 0 0.001 0 0 +9.4751 0.001 0 0 0 0.001 0 0 +9.47998 0.001 0 0 0 0.001 0 0 +9.48486 0.001 0 0 0 0.001 0 0 +9.48975 0.001 0 0 0 0.001 0 0 +9.49463 0.001 0 0 0 0.001 0 0 +9.49951 0.001 0 0 0 0.001 0 0 +9.50439 0.001 0 0 0 0.001 0 0 +9.50928 0.001 0 0 0 0.001 0 0 +9.51416 0.001 0 0 0 0.001 0 0 +9.51904 0.001 0 0 0 0.001 0 0 +9.52393 0.001 0 0 0 0.001 0 0 +9.52881 0.001 0 0 0 0.001 0 0 +9.53369 0.001 0 0 0 0.001 0 0 +9.53857 0.001 0 0 0 0.001 0 0 +9.54346 0.001 0 0 0 0.001 0 0 +9.54834 0.001 0 0 0 0.001 0 0 +9.55322 0.001 0 0 0 0.001 0 0 +9.55811 0.001 0 0 0 0.001 0 0 +9.56299 0.001 0 0 0 0.001 0 0 +9.56787 0.001 0 0 0 0.001 0 0 +9.57275 0.001 0 0 0 0.001 0 0 +9.57764 0.001 0 0 0 0.001 0 0 +9.58252 0.001 0 0 0 0.001 0 0 +9.5874 0.001 0 0 0 0.001 0 0 +9.59229 0.001 0 0 0 0.001 0 0 +9.59717 0.001 0 0 0 0.001 0 0 +9.60205 0.001 0 0 0 0.001 0 0 +9.60693 0.001 0 0 0 0.001 0 0 +9.61182 0.001 0 0 0 0.001 0 0 +9.6167 0.001 0 0 0 0.001 0 0 +9.62158 0.001 0 0 0 0.001 0 0 +9.62646 0.001 0 0 0 0.001 0 0 +9.63135 0.001 0 0 0 0.001 0 0 +9.63623 0.001 0 0 0 0.001 0 0 +9.64111 0.001 0 0 0 0.001 0 0 +9.646 0.001 0 0 0 0.001 0 0 +9.65088 0.001 0 0 0 0.001 0 0 +9.65576 0.001 0 0 0 0.001 0 0 +9.66064 0.001 0 0 0 0.001 0 0 +9.66553 0.001 0 0 0 0.001 0 0 +9.67041 0.001 0 0 0 0.001 0 0 +9.67529 0.001 0 0 0 0.001 0 0 +9.68018 0.001 0 0 0 0.001 0 0 +9.68506 0.001 0 0 0 0.001 0 0 +9.68994 0.001 0 0 0 0.001 0 0 +9.69482 0.001 0 0 0 0.001 0 0 +9.69971 0.001 0 0 0 0.001 0 0 +9.70459 0.001 0 0 0 0.001 0 0 +9.70947 0.001 0 0 0 0.001 0 0 +9.71436 0.001 0 0 0 0.001 0 0 +9.71924 0.001 0 0 0 0.001 0 0 +9.72412 0.001 0 0 0 0.001 0 0 +9.729 0.001 0 0 0 0.001 0 0 +9.73389 0.001 0 0 0 0.001 0 0 +9.73877 0.001 0 0 0 0.001 0 0 +9.74365 0.001 0 0 0 0.001 0 0 +9.74854 0.001 0 0 0 0.001 0 0 +9.75342 0.001 0 0 0 0.001 0 0 +9.7583 0.001 0 0 0 0.001 0 0 +9.76318 0.001 0 0 0 0.001 0 0 +9.76807 0.001 0 0 0 0.001 0 0 +9.77295 0.001 0 0 0 0.001 0 0 +9.77783 0.001 0 0 0 0.001 0 0 +9.78271 0.001 0 0 0 0.001 0 0 +9.7876 0.001 0 0 0 0.001 0 0 +9.79248 0.001 0 0 0 0.001 0 0 +9.79736 0.001 0 0 0 0.001 0 0 +9.80225 0.001 0 0 0 0.001 0 0 +9.80713 0.001 0 0 0 0.001 0 0 +9.81201 0.001 0 0 0 0.001 0 0 +9.81689 0.001 0 0 0 0.001 0 0 +9.82178 0.001 0 0 0 0.001 0 0 +9.82666 0.001 0 0 0 0.001 0 0 +9.83154 0.001 0 0 0 0.001 0 0 +9.83643 0.001 0 0 0 0.001 0 0 +9.84131 0.001 0 0 0 0.001 0 0 +9.84619 0.001 0 0 0 0.001 0 0 +9.85107 0.001 0 0 0 0.001 0 0 +9.85596 0.001 0 0 0 0.001 0 0 +9.86084 0.001 0 0 0 0.001 0 0 +9.86572 0.001 0 0 0 0.001 0 0 +9.87061 0.001 0 0 0 0.001 0 0 +9.87549 0.001 0 0 0 0.001 0 0 +9.88037 0.001 0 0 0 0.001 0 0 +9.88525 0.001 0 0 0 0.001 0 0 +9.89014 0.001 0 0 0 0.001 0 0 +9.89502 0.001 0 0 0 0.001 0 0 +9.8999 0.001 0 0 0 0.001 0 0 +9.90479 0.001 0 0 0 0.001 0 0 +9.90967 0.001 0 0 0 0.001 0 0 +9.91455 0.001 0 0 0 0.001 0 0 +9.91943 0.001 0 0 0 0.001 0 0 +9.92432 0.001 0 0 0 0.001 0 0 +9.9292 0.001 0 0 0 0.001 0 0 +9.93408 0.001 0 0 0 0.001 0 0 +9.93896 0.001 0 0 0 0.001 0 0 +9.94385 0.001 0 0 0 0.001 0 0 +9.94873 0.001 0 0 0 0.001 0 0 +9.95361 0.001 0 0 0 0.001 0 0 +9.9585 0.001 0 0 0 0.001 0 0 +9.96338 0.001 0 0 0 0.001 0 0 +9.96826 0.001 0 0 0 0.001 0 0 +9.97314 0.001 0 0 0 0.001 0 0 +9.97803 0.001 0 0 0 0.001 0 0 +9.98291 0.001 0 0 0 0.001 0 0 +9.98779 0.001 0 0 0 0.001 0 0 +9.99268 0.001 0 0 0 0.001 0 0 +9.99756 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=256.csv b/reference/swashes_1_nx=256.csv new file mode 100644 index 0000000..65a55ba --- /dev/null +++ b/reference/swashes_1_nx=256.csv @@ -0,0 +1,274 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.0390625 meters +# Number of cells: 256 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0195312 0.005 0 0 0 0.005 0 0 +0.0585938 0.005 0 0 0 0.005 0 0 +0.0976562 0.005 0 0 0 0.005 0 0 +0.136719 0.005 0 0 0 0.005 0 0 +0.175781 0.005 0 0 0 0.005 0 0 +0.214844 0.005 0 0 0 0.005 0 0 +0.253906 0.005 0 0 0 0.005 0 0 +0.292969 0.005 0 0 0 0.005 0 0 +0.332031 0.005 0 0 0 0.005 0 0 +0.371094 0.005 0 0 0 0.005 0 0 +0.410156 0.005 0 0 0 0.005 0 0 +0.449219 0.005 0 0 0 0.005 0 0 +0.488281 0.005 0 0 0 0.005 0 0 +0.527344 0.005 0 0 0 0.005 0 0 +0.566406 0.005 0 0 0 0.005 0 0 +0.605469 0.005 0 0 0 0.005 0 0 +0.644531 0.005 0 0 0 0.005 0 0 +0.683594 0.005 0 0 0 0.005 0 0 +0.722656 0.005 0 0 0 0.005 0 0 +0.761719 0.005 0 0 0 0.005 0 0 +0.800781 0.005 0 0 0 0.005 0 0 +0.839844 0.005 0 0 0 0.005 0 0 +0.878906 0.005 0 0 0 0.005 0 0 +0.917969 0.005 0 0 0 0.005 0 0 +0.957031 0.005 0 0 0 0.005 0 0 +0.996094 0.005 0 0 0 0.005 0 0 +1.03516 0.005 0 0 0 0.005 0 0 +1.07422 0.005 0 0 0 0.005 0 0 +1.11328 0.005 0 0 0 0.005 0 0 +1.15234 0.005 0 0 0 0.005 0 0 +1.19141 0.005 0 0 0 0.005 0 0 +1.23047 0.005 0 0 0 0.005 0 0 +1.26953 0.005 0 0 0 0.005 0 0 +1.30859 0.005 0 0 0 0.005 0 0 +1.34766 0.005 0 0 0 0.005 0 0 +1.38672 0.005 0 0 0 0.005 0 0 +1.42578 0.005 0 0 0 0.005 0 0 +1.46484 0.005 0 0 0 0.005 0 0 +1.50391 0.005 0 0 0 0.005 0 0 +1.54297 0.005 0 0 0 0.005 0 0 +1.58203 0.005 0 0 0 0.005 0 0 +1.62109 0.005 0 0 0 0.005 0 0 +1.66016 0.005 0 0 0 0.005 0 0 +1.69922 0.005 0 0 0 0.005 0 0 +1.73828 0.005 0 0 0 0.005 0 0 +1.77734 0.005 0 0 0 0.005 0 0 +1.81641 0.005 0 0 0 0.005 0 0 +1.85547 0.005 0 0 0 0.005 0 0 +1.89453 0.005 0 0 0 0.005 0 0 +1.93359 0.005 0 0 0 0.005 0 0 +1.97266 0.005 0 0 0 0.005 0 0 +2.01172 0.005 0 0 0 0.005 0 0 +2.05078 0.005 0 0 0 0.005 0 0 +2.08984 0.005 0 0 0 0.005 0 0 +2.12891 0.005 0 0 0 0.005 0 0 +2.16797 0.005 0 0 0 0.005 0 0 +2.20703 0.005 0 0 0 0.005 0 0 +2.24609 0.005 0 0 0 0.005 0 0 +2.28516 0.005 0 0 0 0.005 0 0 +2.32422 0.005 0 0 0 0.005 0 0 +2.36328 0.005 0 0 0 0.005 0 0 +2.40234 0.005 0 0 0 0.005 0 0 +2.44141 0.005 0 0 0 0.005 0 0 +2.48047 0.005 0 0 0 0.005 0 0 +2.51953 0.005 0 0 0 0.005 0 0 +2.55859 0.005 0 0 0 0.005 0 0 +2.59766 0.005 0 0 0 0.005 0 0 +2.63672 0.005 0 0 0 0.005 0 0 +2.67578 0.005 0 0 0 0.005 0 0 +2.71484 0.005 0 0 0 0.005 0 0 +2.75391 0.005 0 0 0 0.005 0 0 +2.79297 0.005 0 0 0 0.005 0 0 +2.83203 0.005 0 0 0 0.005 0 0 +2.87109 0.005 0 0 0 0.005 0 0 +2.91016 0.005 0 0 0 0.005 0 0 +2.94922 0.005 0 0 0 0.005 0 0 +2.98828 0.005 0 0 0 0.005 0 0 +3.02734 0.005 0 0 0 0.005 0 0 +3.06641 0.005 0 0 0 0.005 0 0 +3.10547 0.005 0 0 0 0.005 0 0 +3.14453 0.005 0 0 0 0.005 0 0 +3.18359 0.005 0 0 0 0.005 0 0 +3.22266 0.005 0 0 0 0.005 0 0 +3.26172 0.005 0 0 0 0.005 0 0 +3.30078 0.005 0 0 0 0.005 0 0 +3.33984 0.005 0 0 0 0.005 0 0 +3.37891 0.005 0 0 0 0.005 0 0 +3.41797 0.005 0 0 0 0.005 0 0 +3.45703 0.005 0 0 0 0.005 0 0 +3.49609 0.005 0 0 0 0.005 0 0 +3.53516 0.005 0 0 0 0.005 0 0 +3.57422 0.005 0 0 0 0.005 0 0 +3.61328 0.005 0 0 0 0.005 0 0 +3.65234 0.005 0 0 0 0.005 0 0 +3.69141 0.00494936 0.00224893 0 1.11307e-005 0.00494936 0.0102062 0.000232877 +3.73047 0.00485235 0.0065892 0 3.19731e-005 0.00485235 0.0302011 0.00047058 +3.76953 0.0047563 0.0109295 0 5.19839e-005 0.0047563 0.0505977 0.000650663 +3.80859 0.00466121 0.0152698 0 7.11755e-005 0.00466121 0.0714082 0.000802289 +3.84766 0.00456708 0.01961 0 8.95606e-005 0.00456708 0.0926456 0.000935093 +3.88672 0.00447391 0.0239503 0 0.000107152 0.00447391 0.114323 0.00105384 +3.92578 0.0043817 0.0282906 0 0.000123961 0.0043817 0.136454 0.00116136 +3.96484 0.00429045 0.0326309 0 0.000140001 0.00429045 0.159053 0.0012595 +4.00391 0.00420017 0.0369711 0 0.000155285 0.00420017 0.182136 0.00134957 +4.04297 0.00411084 0.0413114 0 0.000169825 0.00411084 0.205717 0.00143255 +4.08203 0.00402247 0.0456517 0 0.000183633 0.00402247 0.229814 0.00150919 +4.12109 0.00393506 0.049992 0 0.000196722 0.00393506 0.254443 0.00158008 +4.16016 0.00384861 0.0543323 0 0.000209104 0.00384861 0.279622 0.0016457 +4.19922 0.00376313 0.0586725 0 0.000220792 0.00376313 0.30537 0.00170647 +4.23828 0.0036786 0.0630128 0 0.000231799 0.0036786 0.331706 0.00176272 +4.27734 0.00359503 0.0673531 0 0.000242137 0.00359503 0.358651 0.00181475 +4.31641 0.00351242 0.0716934 0 0.000251818 0.00351242 0.386226 0.00186281 +4.35547 0.00343078 0.0760336 0 0.000260855 0.00343078 0.414453 0.00190711 +4.39453 0.00335009 0.0803739 0 0.00026926 0.00335009 0.443356 0.00194786 +4.43359 0.00327036 0.0847142 0 0.000277046 0.00327036 0.472959 0.00198523 +4.47266 0.0031916 0.0890545 0 0.000284226 0.0031916 0.503289 0.00201939 +4.51172 0.00311379 0.0933948 0 0.000290812 0.00311379 0.534371 0.00205046 +4.55078 0.00303694 0.097735 0 0.000296816 0.00303694 0.566236 0.00207859 +4.58984 0.00296106 0.102075 0 0.000302251 0.00296106 0.598912 0.00210389 +4.62891 0.00288613 0.106416 0 0.000307129 0.00288613 0.63243 0.00212646 +4.66797 0.00281217 0.110756 0 0.000311464 0.00281217 0.666825 0.00214642 +4.70703 0.00273916 0.115096 0 0.000315267 0.00273916 0.70213 0.00216386 +4.74609 0.00266712 0.119436 0 0.000318551 0.00266712 0.738383 0.00217886 +4.78516 0.00259603 0.123777 0 0.000321328 0.00259603 0.775621 0.00219151 +4.82422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01953 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05859 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09766 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13672 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17578 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21484 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25391 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29297 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33203 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37109 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41016 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44922 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48828 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52734 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56641 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60547 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64453 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68359 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72266 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76172 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80078 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83984 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87891 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91797 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95703 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99609 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03516 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07422 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11328 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15234 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19141 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23047 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26953 0.001 0 0 0 0.001 0 0 +6.30859 0.001 0 0 0 0.001 0 0 +6.34766 0.001 0 0 0 0.001 0 0 +6.38672 0.001 0 0 0 0.001 0 0 +6.42578 0.001 0 0 0 0.001 0 0 +6.46484 0.001 0 0 0 0.001 0 0 +6.50391 0.001 0 0 0 0.001 0 0 +6.54297 0.001 0 0 0 0.001 0 0 +6.58203 0.001 0 0 0 0.001 0 0 +6.62109 0.001 0 0 0 0.001 0 0 +6.66016 0.001 0 0 0 0.001 0 0 +6.69922 0.001 0 0 0 0.001 0 0 +6.73828 0.001 0 0 0 0.001 0 0 +6.77734 0.001 0 0 0 0.001 0 0 +6.81641 0.001 0 0 0 0.001 0 0 +6.85547 0.001 0 0 0 0.001 0 0 +6.89453 0.001 0 0 0 0.001 0 0 +6.93359 0.001 0 0 0 0.001 0 0 +6.97266 0.001 0 0 0 0.001 0 0 +7.01172 0.001 0 0 0 0.001 0 0 +7.05078 0.001 0 0 0 0.001 0 0 +7.08984 0.001 0 0 0 0.001 0 0 +7.12891 0.001 0 0 0 0.001 0 0 +7.16797 0.001 0 0 0 0.001 0 0 +7.20703 0.001 0 0 0 0.001 0 0 +7.24609 0.001 0 0 0 0.001 0 0 +7.28516 0.001 0 0 0 0.001 0 0 +7.32422 0.001 0 0 0 0.001 0 0 +7.36328 0.001 0 0 0 0.001 0 0 +7.40234 0.001 0 0 0 0.001 0 0 +7.44141 0.001 0 0 0 0.001 0 0 +7.48047 0.001 0 0 0 0.001 0 0 +7.51953 0.001 0 0 0 0.001 0 0 +7.55859 0.001 0 0 0 0.001 0 0 +7.59766 0.001 0 0 0 0.001 0 0 +7.63672 0.001 0 0 0 0.001 0 0 +7.67578 0.001 0 0 0 0.001 0 0 +7.71484 0.001 0 0 0 0.001 0 0 +7.75391 0.001 0 0 0 0.001 0 0 +7.79297 0.001 0 0 0 0.001 0 0 +7.83203 0.001 0 0 0 0.001 0 0 +7.87109 0.001 0 0 0 0.001 0 0 +7.91016 0.001 0 0 0 0.001 0 0 +7.94922 0.001 0 0 0 0.001 0 0 +7.98828 0.001 0 0 0 0.001 0 0 +8.02734 0.001 0 0 0 0.001 0 0 +8.06641 0.001 0 0 0 0.001 0 0 +8.10547 0.001 0 0 0 0.001 0 0 +8.14453 0.001 0 0 0 0.001 0 0 +8.18359 0.001 0 0 0 0.001 0 0 +8.22266 0.001 0 0 0 0.001 0 0 +8.26172 0.001 0 0 0 0.001 0 0 +8.30078 0.001 0 0 0 0.001 0 0 +8.33984 0.001 0 0 0 0.001 0 0 +8.37891 0.001 0 0 0 0.001 0 0 +8.41797 0.001 0 0 0 0.001 0 0 +8.45703 0.001 0 0 0 0.001 0 0 +8.49609 0.001 0 0 0 0.001 0 0 +8.53516 0.001 0 0 0 0.001 0 0 +8.57422 0.001 0 0 0 0.001 0 0 +8.61328 0.001 0 0 0 0.001 0 0 +8.65234 0.001 0 0 0 0.001 0 0 +8.69141 0.001 0 0 0 0.001 0 0 +8.73047 0.001 0 0 0 0.001 0 0 +8.76953 0.001 0 0 0 0.001 0 0 +8.80859 0.001 0 0 0 0.001 0 0 +8.84766 0.001 0 0 0 0.001 0 0 +8.88672 0.001 0 0 0 0.001 0 0 +8.92578 0.001 0 0 0 0.001 0 0 +8.96484 0.001 0 0 0 0.001 0 0 +9.00391 0.001 0 0 0 0.001 0 0 +9.04297 0.001 0 0 0 0.001 0 0 +9.08203 0.001 0 0 0 0.001 0 0 +9.12109 0.001 0 0 0 0.001 0 0 +9.16016 0.001 0 0 0 0.001 0 0 +9.19922 0.001 0 0 0 0.001 0 0 +9.23828 0.001 0 0 0 0.001 0 0 +9.27734 0.001 0 0 0 0.001 0 0 +9.31641 0.001 0 0 0 0.001 0 0 +9.35547 0.001 0 0 0 0.001 0 0 +9.39453 0.001 0 0 0 0.001 0 0 +9.43359 0.001 0 0 0 0.001 0 0 +9.47266 0.001 0 0 0 0.001 0 0 +9.51172 0.001 0 0 0 0.001 0 0 +9.55078 0.001 0 0 0 0.001 0 0 +9.58984 0.001 0 0 0 0.001 0 0 +9.62891 0.001 0 0 0 0.001 0 0 +9.66797 0.001 0 0 0 0.001 0 0 +9.70703 0.001 0 0 0 0.001 0 0 +9.74609 0.001 0 0 0 0.001 0 0 +9.78516 0.001 0 0 0 0.001 0 0 +9.82422 0.001 0 0 0 0.001 0 0 +9.86328 0.001 0 0 0 0.001 0 0 +9.90234 0.001 0 0 0 0.001 0 0 +9.94141 0.001 0 0 0 0.001 0 0 +9.98047 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=4096.csv b/reference/swashes_1_nx=4096.csv new file mode 100644 index 0000000..be77bf7 --- /dev/null +++ b/reference/swashes_1_nx=4096.csv @@ -0,0 +1,4114 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.00244141 meters +# Number of cells: 4096 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.0012207 0.005 0 0 0 0.005 0 0 +0.00366211 0.005 0 0 0 0.005 0 0 +0.00610352 0.005 0 0 0 0.005 0 0 +0.00854492 0.005 0 0 0 0.005 0 0 +0.0109863 0.005 0 0 0 0.005 0 0 +0.0134277 0.005 0 0 0 0.005 0 0 +0.0158691 0.005 0 0 0 0.005 0 0 +0.0183105 0.005 0 0 0 0.005 0 0 +0.020752 0.005 0 0 0 0.005 0 0 +0.0231934 0.005 0 0 0 0.005 0 0 +0.0256348 0.005 0 0 0 0.005 0 0 +0.0280762 0.005 0 0 0 0.005 0 0 +0.0305176 0.005 0 0 0 0.005 0 0 +0.032959 0.005 0 0 0 0.005 0 0 +0.0354004 0.005 0 0 0 0.005 0 0 +0.0378418 0.005 0 0 0 0.005 0 0 +0.0402832 0.005 0 0 0 0.005 0 0 +0.0427246 0.005 0 0 0 0.005 0 0 +0.045166 0.005 0 0 0 0.005 0 0 +0.0476074 0.005 0 0 0 0.005 0 0 +0.0500488 0.005 0 0 0 0.005 0 0 +0.0524902 0.005 0 0 0 0.005 0 0 +0.0549316 0.005 0 0 0 0.005 0 0 +0.057373 0.005 0 0 0 0.005 0 0 +0.0598145 0.005 0 0 0 0.005 0 0 +0.0622559 0.005 0 0 0 0.005 0 0 +0.0646973 0.005 0 0 0 0.005 0 0 +0.0671387 0.005 0 0 0 0.005 0 0 +0.0695801 0.005 0 0 0 0.005 0 0 +0.0720215 0.005 0 0 0 0.005 0 0 +0.0744629 0.005 0 0 0 0.005 0 0 +0.0769043 0.005 0 0 0 0.005 0 0 +0.0793457 0.005 0 0 0 0.005 0 0 +0.0817871 0.005 0 0 0 0.005 0 0 +0.0842285 0.005 0 0 0 0.005 0 0 +0.0866699 0.005 0 0 0 0.005 0 0 +0.0891113 0.005 0 0 0 0.005 0 0 +0.0915527 0.005 0 0 0 0.005 0 0 +0.0939941 0.005 0 0 0 0.005 0 0 +0.0964355 0.005 0 0 0 0.005 0 0 +0.098877 0.005 0 0 0 0.005 0 0 +0.101318 0.005 0 0 0 0.005 0 0 +0.10376 0.005 0 0 0 0.005 0 0 +0.106201 0.005 0 0 0 0.005 0 0 +0.108643 0.005 0 0 0 0.005 0 0 +0.111084 0.005 0 0 0 0.005 0 0 +0.113525 0.005 0 0 0 0.005 0 0 +0.115967 0.005 0 0 0 0.005 0 0 +0.118408 0.005 0 0 0 0.005 0 0 +0.12085 0.005 0 0 0 0.005 0 0 +0.123291 0.005 0 0 0 0.005 0 0 +0.125732 0.005 0 0 0 0.005 0 0 +0.128174 0.005 0 0 0 0.005 0 0 +0.130615 0.005 0 0 0 0.005 0 0 +0.133057 0.005 0 0 0 0.005 0 0 +0.135498 0.005 0 0 0 0.005 0 0 +0.137939 0.005 0 0 0 0.005 0 0 +0.140381 0.005 0 0 0 0.005 0 0 +0.142822 0.005 0 0 0 0.005 0 0 +0.145264 0.005 0 0 0 0.005 0 0 +0.147705 0.005 0 0 0 0.005 0 0 +0.150146 0.005 0 0 0 0.005 0 0 +0.152588 0.005 0 0 0 0.005 0 0 +0.155029 0.005 0 0 0 0.005 0 0 +0.157471 0.005 0 0 0 0.005 0 0 +0.159912 0.005 0 0 0 0.005 0 0 +0.162354 0.005 0 0 0 0.005 0 0 +0.164795 0.005 0 0 0 0.005 0 0 +0.167236 0.005 0 0 0 0.005 0 0 +0.169678 0.005 0 0 0 0.005 0 0 +0.172119 0.005 0 0 0 0.005 0 0 +0.174561 0.005 0 0 0 0.005 0 0 +0.177002 0.005 0 0 0 0.005 0 0 +0.179443 0.005 0 0 0 0.005 0 0 +0.181885 0.005 0 0 0 0.005 0 0 +0.184326 0.005 0 0 0 0.005 0 0 +0.186768 0.005 0 0 0 0.005 0 0 +0.189209 0.005 0 0 0 0.005 0 0 +0.19165 0.005 0 0 0 0.005 0 0 +0.194092 0.005 0 0 0 0.005 0 0 +0.196533 0.005 0 0 0 0.005 0 0 +0.198975 0.005 0 0 0 0.005 0 0 +0.201416 0.005 0 0 0 0.005 0 0 +0.203857 0.005 0 0 0 0.005 0 0 +0.206299 0.005 0 0 0 0.005 0 0 +0.20874 0.005 0 0 0 0.005 0 0 +0.211182 0.005 0 0 0 0.005 0 0 +0.213623 0.005 0 0 0 0.005 0 0 +0.216064 0.005 0 0 0 0.005 0 0 +0.218506 0.005 0 0 0 0.005 0 0 +0.220947 0.005 0 0 0 0.005 0 0 +0.223389 0.005 0 0 0 0.005 0 0 +0.22583 0.005 0 0 0 0.005 0 0 +0.228271 0.005 0 0 0 0.005 0 0 +0.230713 0.005 0 0 0 0.005 0 0 +0.233154 0.005 0 0 0 0.005 0 0 +0.235596 0.005 0 0 0 0.005 0 0 +0.238037 0.005 0 0 0 0.005 0 0 +0.240479 0.005 0 0 0 0.005 0 0 +0.24292 0.005 0 0 0 0.005 0 0 +0.245361 0.005 0 0 0 0.005 0 0 +0.247803 0.005 0 0 0 0.005 0 0 +0.250244 0.005 0 0 0 0.005 0 0 +0.252686 0.005 0 0 0 0.005 0 0 +0.255127 0.005 0 0 0 0.005 0 0 +0.257568 0.005 0 0 0 0.005 0 0 +0.26001 0.005 0 0 0 0.005 0 0 +0.262451 0.005 0 0 0 0.005 0 0 +0.264893 0.005 0 0 0 0.005 0 0 +0.267334 0.005 0 0 0 0.005 0 0 +0.269775 0.005 0 0 0 0.005 0 0 +0.272217 0.005 0 0 0 0.005 0 0 +0.274658 0.005 0 0 0 0.005 0 0 +0.2771 0.005 0 0 0 0.005 0 0 +0.279541 0.005 0 0 0 0.005 0 0 +0.281982 0.005 0 0 0 0.005 0 0 +0.284424 0.005 0 0 0 0.005 0 0 +0.286865 0.005 0 0 0 0.005 0 0 +0.289307 0.005 0 0 0 0.005 0 0 +0.291748 0.005 0 0 0 0.005 0 0 +0.294189 0.005 0 0 0 0.005 0 0 +0.296631 0.005 0 0 0 0.005 0 0 +0.299072 0.005 0 0 0 0.005 0 0 +0.301514 0.005 0 0 0 0.005 0 0 +0.303955 0.005 0 0 0 0.005 0 0 +0.306396 0.005 0 0 0 0.005 0 0 +0.308838 0.005 0 0 0 0.005 0 0 +0.311279 0.005 0 0 0 0.005 0 0 +0.313721 0.005 0 0 0 0.005 0 0 +0.316162 0.005 0 0 0 0.005 0 0 +0.318604 0.005 0 0 0 0.005 0 0 +0.321045 0.005 0 0 0 0.005 0 0 +0.323486 0.005 0 0 0 0.005 0 0 +0.325928 0.005 0 0 0 0.005 0 0 +0.328369 0.005 0 0 0 0.005 0 0 +0.330811 0.005 0 0 0 0.005 0 0 +0.333252 0.005 0 0 0 0.005 0 0 +0.335693 0.005 0 0 0 0.005 0 0 +0.338135 0.005 0 0 0 0.005 0 0 +0.340576 0.005 0 0 0 0.005 0 0 +0.343018 0.005 0 0 0 0.005 0 0 +0.345459 0.005 0 0 0 0.005 0 0 +0.3479 0.005 0 0 0 0.005 0 0 +0.350342 0.005 0 0 0 0.005 0 0 +0.352783 0.005 0 0 0 0.005 0 0 +0.355225 0.005 0 0 0 0.005 0 0 +0.357666 0.005 0 0 0 0.005 0 0 +0.360107 0.005 0 0 0 0.005 0 0 +0.362549 0.005 0 0 0 0.005 0 0 +0.36499 0.005 0 0 0 0.005 0 0 +0.367432 0.005 0 0 0 0.005 0 0 +0.369873 0.005 0 0 0 0.005 0 0 +0.372314 0.005 0 0 0 0.005 0 0 +0.374756 0.005 0 0 0 0.005 0 0 +0.377197 0.005 0 0 0 0.005 0 0 +0.379639 0.005 0 0 0 0.005 0 0 +0.38208 0.005 0 0 0 0.005 0 0 +0.384521 0.005 0 0 0 0.005 0 0 +0.386963 0.005 0 0 0 0.005 0 0 +0.389404 0.005 0 0 0 0.005 0 0 +0.391846 0.005 0 0 0 0.005 0 0 +0.394287 0.005 0 0 0 0.005 0 0 +0.396729 0.005 0 0 0 0.005 0 0 +0.39917 0.005 0 0 0 0.005 0 0 +0.401611 0.005 0 0 0 0.005 0 0 +0.404053 0.005 0 0 0 0.005 0 0 +0.406494 0.005 0 0 0 0.005 0 0 +0.408936 0.005 0 0 0 0.005 0 0 +0.411377 0.005 0 0 0 0.005 0 0 +0.413818 0.005 0 0 0 0.005 0 0 +0.41626 0.005 0 0 0 0.005 0 0 +0.418701 0.005 0 0 0 0.005 0 0 +0.421143 0.005 0 0 0 0.005 0 0 +0.423584 0.005 0 0 0 0.005 0 0 +0.426025 0.005 0 0 0 0.005 0 0 +0.428467 0.005 0 0 0 0.005 0 0 +0.430908 0.005 0 0 0 0.005 0 0 +0.43335 0.005 0 0 0 0.005 0 0 +0.435791 0.005 0 0 0 0.005 0 0 +0.438232 0.005 0 0 0 0.005 0 0 +0.440674 0.005 0 0 0 0.005 0 0 +0.443115 0.005 0 0 0 0.005 0 0 +0.445557 0.005 0 0 0 0.005 0 0 +0.447998 0.005 0 0 0 0.005 0 0 +0.450439 0.005 0 0 0 0.005 0 0 +0.452881 0.005 0 0 0 0.005 0 0 +0.455322 0.005 0 0 0 0.005 0 0 +0.457764 0.005 0 0 0 0.005 0 0 +0.460205 0.005 0 0 0 0.005 0 0 +0.462646 0.005 0 0 0 0.005 0 0 +0.465088 0.005 0 0 0 0.005 0 0 +0.467529 0.005 0 0 0 0.005 0 0 +0.469971 0.005 0 0 0 0.005 0 0 +0.472412 0.005 0 0 0 0.005 0 0 +0.474854 0.005 0 0 0 0.005 0 0 +0.477295 0.005 0 0 0 0.005 0 0 +0.479736 0.005 0 0 0 0.005 0 0 +0.482178 0.005 0 0 0 0.005 0 0 +0.484619 0.005 0 0 0 0.005 0 0 +0.487061 0.005 0 0 0 0.005 0 0 +0.489502 0.005 0 0 0 0.005 0 0 +0.491943 0.005 0 0 0 0.005 0 0 +0.494385 0.005 0 0 0 0.005 0 0 +0.496826 0.005 0 0 0 0.005 0 0 +0.499268 0.005 0 0 0 0.005 0 0 +0.501709 0.005 0 0 0 0.005 0 0 +0.50415 0.005 0 0 0 0.005 0 0 +0.506592 0.005 0 0 0 0.005 0 0 +0.509033 0.005 0 0 0 0.005 0 0 +0.511475 0.005 0 0 0 0.005 0 0 +0.513916 0.005 0 0 0 0.005 0 0 +0.516357 0.005 0 0 0 0.005 0 0 +0.518799 0.005 0 0 0 0.005 0 0 +0.52124 0.005 0 0 0 0.005 0 0 +0.523682 0.005 0 0 0 0.005 0 0 +0.526123 0.005 0 0 0 0.005 0 0 +0.528564 0.005 0 0 0 0.005 0 0 +0.531006 0.005 0 0 0 0.005 0 0 +0.533447 0.005 0 0 0 0.005 0 0 +0.535889 0.005 0 0 0 0.005 0 0 +0.53833 0.005 0 0 0 0.005 0 0 +0.540771 0.005 0 0 0 0.005 0 0 +0.543213 0.005 0 0 0 0.005 0 0 +0.545654 0.005 0 0 0 0.005 0 0 +0.548096 0.005 0 0 0 0.005 0 0 +0.550537 0.005 0 0 0 0.005 0 0 +0.552979 0.005 0 0 0 0.005 0 0 +0.55542 0.005 0 0 0 0.005 0 0 +0.557861 0.005 0 0 0 0.005 0 0 +0.560303 0.005 0 0 0 0.005 0 0 +0.562744 0.005 0 0 0 0.005 0 0 +0.565186 0.005 0 0 0 0.005 0 0 +0.567627 0.005 0 0 0 0.005 0 0 +0.570068 0.005 0 0 0 0.005 0 0 +0.57251 0.005 0 0 0 0.005 0 0 +0.574951 0.005 0 0 0 0.005 0 0 +0.577393 0.005 0 0 0 0.005 0 0 +0.579834 0.005 0 0 0 0.005 0 0 +0.582275 0.005 0 0 0 0.005 0 0 +0.584717 0.005 0 0 0 0.005 0 0 +0.587158 0.005 0 0 0 0.005 0 0 +0.5896 0.005 0 0 0 0.005 0 0 +0.592041 0.005 0 0 0 0.005 0 0 +0.594482 0.005 0 0 0 0.005 0 0 +0.596924 0.005 0 0 0 0.005 0 0 +0.599365 0.005 0 0 0 0.005 0 0 +0.601807 0.005 0 0 0 0.005 0 0 +0.604248 0.005 0 0 0 0.005 0 0 +0.606689 0.005 0 0 0 0.005 0 0 +0.609131 0.005 0 0 0 0.005 0 0 +0.611572 0.005 0 0 0 0.005 0 0 +0.614014 0.005 0 0 0 0.005 0 0 +0.616455 0.005 0 0 0 0.005 0 0 +0.618896 0.005 0 0 0 0.005 0 0 +0.621338 0.005 0 0 0 0.005 0 0 +0.623779 0.005 0 0 0 0.005 0 0 +0.626221 0.005 0 0 0 0.005 0 0 +0.628662 0.005 0 0 0 0.005 0 0 +0.631104 0.005 0 0 0 0.005 0 0 +0.633545 0.005 0 0 0 0.005 0 0 +0.635986 0.005 0 0 0 0.005 0 0 +0.638428 0.005 0 0 0 0.005 0 0 +0.640869 0.005 0 0 0 0.005 0 0 +0.643311 0.005 0 0 0 0.005 0 0 +0.645752 0.005 0 0 0 0.005 0 0 +0.648193 0.005 0 0 0 0.005 0 0 +0.650635 0.005 0 0 0 0.005 0 0 +0.653076 0.005 0 0 0 0.005 0 0 +0.655518 0.005 0 0 0 0.005 0 0 +0.657959 0.005 0 0 0 0.005 0 0 +0.6604 0.005 0 0 0 0.005 0 0 +0.662842 0.005 0 0 0 0.005 0 0 +0.665283 0.005 0 0 0 0.005 0 0 +0.667725 0.005 0 0 0 0.005 0 0 +0.670166 0.005 0 0 0 0.005 0 0 +0.672607 0.005 0 0 0 0.005 0 0 +0.675049 0.005 0 0 0 0.005 0 0 +0.67749 0.005 0 0 0 0.005 0 0 +0.679932 0.005 0 0 0 0.005 0 0 +0.682373 0.005 0 0 0 0.005 0 0 +0.684814 0.005 0 0 0 0.005 0 0 +0.687256 0.005 0 0 0 0.005 0 0 +0.689697 0.005 0 0 0 0.005 0 0 +0.692139 0.005 0 0 0 0.005 0 0 +0.69458 0.005 0 0 0 0.005 0 0 +0.697021 0.005 0 0 0 0.005 0 0 +0.699463 0.005 0 0 0 0.005 0 0 +0.701904 0.005 0 0 0 0.005 0 0 +0.704346 0.005 0 0 0 0.005 0 0 +0.706787 0.005 0 0 0 0.005 0 0 +0.709229 0.005 0 0 0 0.005 0 0 +0.71167 0.005 0 0 0 0.005 0 0 +0.714111 0.005 0 0 0 0.005 0 0 +0.716553 0.005 0 0 0 0.005 0 0 +0.718994 0.005 0 0 0 0.005 0 0 +0.721436 0.005 0 0 0 0.005 0 0 +0.723877 0.005 0 0 0 0.005 0 0 +0.726318 0.005 0 0 0 0.005 0 0 +0.72876 0.005 0 0 0 0.005 0 0 +0.731201 0.005 0 0 0 0.005 0 0 +0.733643 0.005 0 0 0 0.005 0 0 +0.736084 0.005 0 0 0 0.005 0 0 +0.738525 0.005 0 0 0 0.005 0 0 +0.740967 0.005 0 0 0 0.005 0 0 +0.743408 0.005 0 0 0 0.005 0 0 +0.74585 0.005 0 0 0 0.005 0 0 +0.748291 0.005 0 0 0 0.005 0 0 +0.750732 0.005 0 0 0 0.005 0 0 +0.753174 0.005 0 0 0 0.005 0 0 +0.755615 0.005 0 0 0 0.005 0 0 +0.758057 0.005 0 0 0 0.005 0 0 +0.760498 0.005 0 0 0 0.005 0 0 +0.762939 0.005 0 0 0 0.005 0 0 +0.765381 0.005 0 0 0 0.005 0 0 +0.767822 0.005 0 0 0 0.005 0 0 +0.770264 0.005 0 0 0 0.005 0 0 +0.772705 0.005 0 0 0 0.005 0 0 +0.775146 0.005 0 0 0 0.005 0 0 +0.777588 0.005 0 0 0 0.005 0 0 +0.780029 0.005 0 0 0 0.005 0 0 +0.782471 0.005 0 0 0 0.005 0 0 +0.784912 0.005 0 0 0 0.005 0 0 +0.787354 0.005 0 0 0 0.005 0 0 +0.789795 0.005 0 0 0 0.005 0 0 +0.792236 0.005 0 0 0 0.005 0 0 +0.794678 0.005 0 0 0 0.005 0 0 +0.797119 0.005 0 0 0 0.005 0 0 +0.799561 0.005 0 0 0 0.005 0 0 +0.802002 0.005 0 0 0 0.005 0 0 +0.804443 0.005 0 0 0 0.005 0 0 +0.806885 0.005 0 0 0 0.005 0 0 +0.809326 0.005 0 0 0 0.005 0 0 +0.811768 0.005 0 0 0 0.005 0 0 +0.814209 0.005 0 0 0 0.005 0 0 +0.81665 0.005 0 0 0 0.005 0 0 +0.819092 0.005 0 0 0 0.005 0 0 +0.821533 0.005 0 0 0 0.005 0 0 +0.823975 0.005 0 0 0 0.005 0 0 +0.826416 0.005 0 0 0 0.005 0 0 +0.828857 0.005 0 0 0 0.005 0 0 +0.831299 0.005 0 0 0 0.005 0 0 +0.83374 0.005 0 0 0 0.005 0 0 +0.836182 0.005 0 0 0 0.005 0 0 +0.838623 0.005 0 0 0 0.005 0 0 +0.841064 0.005 0 0 0 0.005 0 0 +0.843506 0.005 0 0 0 0.005 0 0 +0.845947 0.005 0 0 0 0.005 0 0 +0.848389 0.005 0 0 0 0.005 0 0 +0.85083 0.005 0 0 0 0.005 0 0 +0.853271 0.005 0 0 0 0.005 0 0 +0.855713 0.005 0 0 0 0.005 0 0 +0.858154 0.005 0 0 0 0.005 0 0 +0.860596 0.005 0 0 0 0.005 0 0 +0.863037 0.005 0 0 0 0.005 0 0 +0.865479 0.005 0 0 0 0.005 0 0 +0.86792 0.005 0 0 0 0.005 0 0 +0.870361 0.005 0 0 0 0.005 0 0 +0.872803 0.005 0 0 0 0.005 0 0 +0.875244 0.005 0 0 0 0.005 0 0 +0.877686 0.005 0 0 0 0.005 0 0 +0.880127 0.005 0 0 0 0.005 0 0 +0.882568 0.005 0 0 0 0.005 0 0 +0.88501 0.005 0 0 0 0.005 0 0 +0.887451 0.005 0 0 0 0.005 0 0 +0.889893 0.005 0 0 0 0.005 0 0 +0.892334 0.005 0 0 0 0.005 0 0 +0.894775 0.005 0 0 0 0.005 0 0 +0.897217 0.005 0 0 0 0.005 0 0 +0.899658 0.005 0 0 0 0.005 0 0 +0.9021 0.005 0 0 0 0.005 0 0 +0.904541 0.005 0 0 0 0.005 0 0 +0.906982 0.005 0 0 0 0.005 0 0 +0.909424 0.005 0 0 0 0.005 0 0 +0.911865 0.005 0 0 0 0.005 0 0 +0.914307 0.005 0 0 0 0.005 0 0 +0.916748 0.005 0 0 0 0.005 0 0 +0.919189 0.005 0 0 0 0.005 0 0 +0.921631 0.005 0 0 0 0.005 0 0 +0.924072 0.005 0 0 0 0.005 0 0 +0.926514 0.005 0 0 0 0.005 0 0 +0.928955 0.005 0 0 0 0.005 0 0 +0.931396 0.005 0 0 0 0.005 0 0 +0.933838 0.005 0 0 0 0.005 0 0 +0.936279 0.005 0 0 0 0.005 0 0 +0.938721 0.005 0 0 0 0.005 0 0 +0.941162 0.005 0 0 0 0.005 0 0 +0.943604 0.005 0 0 0 0.005 0 0 +0.946045 0.005 0 0 0 0.005 0 0 +0.948486 0.005 0 0 0 0.005 0 0 +0.950928 0.005 0 0 0 0.005 0 0 +0.953369 0.005 0 0 0 0.005 0 0 +0.955811 0.005 0 0 0 0.005 0 0 +0.958252 0.005 0 0 0 0.005 0 0 +0.960693 0.005 0 0 0 0.005 0 0 +0.963135 0.005 0 0 0 0.005 0 0 +0.965576 0.005 0 0 0 0.005 0 0 +0.968018 0.005 0 0 0 0.005 0 0 +0.970459 0.005 0 0 0 0.005 0 0 +0.9729 0.005 0 0 0 0.005 0 0 +0.975342 0.005 0 0 0 0.005 0 0 +0.977783 0.005 0 0 0 0.005 0 0 +0.980225 0.005 0 0 0 0.005 0 0 +0.982666 0.005 0 0 0 0.005 0 0 +0.985107 0.005 0 0 0 0.005 0 0 +0.987549 0.005 0 0 0 0.005 0 0 +0.98999 0.005 0 0 0 0.005 0 0 +0.992432 0.005 0 0 0 0.005 0 0 +0.994873 0.005 0 0 0 0.005 0 0 +0.997314 0.005 0 0 0 0.005 0 0 +0.999756 0.005 0 0 0 0.005 0 0 +1.0022 0.005 0 0 0 0.005 0 0 +1.00464 0.005 0 0 0 0.005 0 0 +1.00708 0.005 0 0 0 0.005 0 0 +1.00952 0.005 0 0 0 0.005 0 0 +1.01196 0.005 0 0 0 0.005 0 0 +1.0144 0.005 0 0 0 0.005 0 0 +1.01685 0.005 0 0 0 0.005 0 0 +1.01929 0.005 0 0 0 0.005 0 0 +1.02173 0.005 0 0 0 0.005 0 0 +1.02417 0.005 0 0 0 0.005 0 0 +1.02661 0.005 0 0 0 0.005 0 0 +1.02905 0.005 0 0 0 0.005 0 0 +1.03149 0.005 0 0 0 0.005 0 0 +1.03394 0.005 0 0 0 0.005 0 0 +1.03638 0.005 0 0 0 0.005 0 0 +1.03882 0.005 0 0 0 0.005 0 0 +1.04126 0.005 0 0 0 0.005 0 0 +1.0437 0.005 0 0 0 0.005 0 0 +1.04614 0.005 0 0 0 0.005 0 0 +1.04858 0.005 0 0 0 0.005 0 0 +1.05103 0.005 0 0 0 0.005 0 0 +1.05347 0.005 0 0 0 0.005 0 0 +1.05591 0.005 0 0 0 0.005 0 0 +1.05835 0.005 0 0 0 0.005 0 0 +1.06079 0.005 0 0 0 0.005 0 0 +1.06323 0.005 0 0 0 0.005 0 0 +1.06567 0.005 0 0 0 0.005 0 0 +1.06812 0.005 0 0 0 0.005 0 0 +1.07056 0.005 0 0 0 0.005 0 0 +1.073 0.005 0 0 0 0.005 0 0 +1.07544 0.005 0 0 0 0.005 0 0 +1.07788 0.005 0 0 0 0.005 0 0 +1.08032 0.005 0 0 0 0.005 0 0 +1.08276 0.005 0 0 0 0.005 0 0 +1.08521 0.005 0 0 0 0.005 0 0 +1.08765 0.005 0 0 0 0.005 0 0 +1.09009 0.005 0 0 0 0.005 0 0 +1.09253 0.005 0 0 0 0.005 0 0 +1.09497 0.005 0 0 0 0.005 0 0 +1.09741 0.005 0 0 0 0.005 0 0 +1.09985 0.005 0 0 0 0.005 0 0 +1.10229 0.005 0 0 0 0.005 0 0 +1.10474 0.005 0 0 0 0.005 0 0 +1.10718 0.005 0 0 0 0.005 0 0 +1.10962 0.005 0 0 0 0.005 0 0 +1.11206 0.005 0 0 0 0.005 0 0 +1.1145 0.005 0 0 0 0.005 0 0 +1.11694 0.005 0 0 0 0.005 0 0 +1.11938 0.005 0 0 0 0.005 0 0 +1.12183 0.005 0 0 0 0.005 0 0 +1.12427 0.005 0 0 0 0.005 0 0 +1.12671 0.005 0 0 0 0.005 0 0 +1.12915 0.005 0 0 0 0.005 0 0 +1.13159 0.005 0 0 0 0.005 0 0 +1.13403 0.005 0 0 0 0.005 0 0 +1.13647 0.005 0 0 0 0.005 0 0 +1.13892 0.005 0 0 0 0.005 0 0 +1.14136 0.005 0 0 0 0.005 0 0 +1.1438 0.005 0 0 0 0.005 0 0 +1.14624 0.005 0 0 0 0.005 0 0 +1.14868 0.005 0 0 0 0.005 0 0 +1.15112 0.005 0 0 0 0.005 0 0 +1.15356 0.005 0 0 0 0.005 0 0 +1.15601 0.005 0 0 0 0.005 0 0 +1.15845 0.005 0 0 0 0.005 0 0 +1.16089 0.005 0 0 0 0.005 0 0 +1.16333 0.005 0 0 0 0.005 0 0 +1.16577 0.005 0 0 0 0.005 0 0 +1.16821 0.005 0 0 0 0.005 0 0 +1.17065 0.005 0 0 0 0.005 0 0 +1.1731 0.005 0 0 0 0.005 0 0 +1.17554 0.005 0 0 0 0.005 0 0 +1.17798 0.005 0 0 0 0.005 0 0 +1.18042 0.005 0 0 0 0.005 0 0 +1.18286 0.005 0 0 0 0.005 0 0 +1.1853 0.005 0 0 0 0.005 0 0 +1.18774 0.005 0 0 0 0.005 0 0 +1.19019 0.005 0 0 0 0.005 0 0 +1.19263 0.005 0 0 0 0.005 0 0 +1.19507 0.005 0 0 0 0.005 0 0 +1.19751 0.005 0 0 0 0.005 0 0 +1.19995 0.005 0 0 0 0.005 0 0 +1.20239 0.005 0 0 0 0.005 0 0 +1.20483 0.005 0 0 0 0.005 0 0 +1.20728 0.005 0 0 0 0.005 0 0 +1.20972 0.005 0 0 0 0.005 0 0 +1.21216 0.005 0 0 0 0.005 0 0 +1.2146 0.005 0 0 0 0.005 0 0 +1.21704 0.005 0 0 0 0.005 0 0 +1.21948 0.005 0 0 0 0.005 0 0 +1.22192 0.005 0 0 0 0.005 0 0 +1.22437 0.005 0 0 0 0.005 0 0 +1.22681 0.005 0 0 0 0.005 0 0 +1.22925 0.005 0 0 0 0.005 0 0 +1.23169 0.005 0 0 0 0.005 0 0 +1.23413 0.005 0 0 0 0.005 0 0 +1.23657 0.005 0 0 0 0.005 0 0 +1.23901 0.005 0 0 0 0.005 0 0 +1.24146 0.005 0 0 0 0.005 0 0 +1.2439 0.005 0 0 0 0.005 0 0 +1.24634 0.005 0 0 0 0.005 0 0 +1.24878 0.005 0 0 0 0.005 0 0 +1.25122 0.005 0 0 0 0.005 0 0 +1.25366 0.005 0 0 0 0.005 0 0 +1.2561 0.005 0 0 0 0.005 0 0 +1.25854 0.005 0 0 0 0.005 0 0 +1.26099 0.005 0 0 0 0.005 0 0 +1.26343 0.005 0 0 0 0.005 0 0 +1.26587 0.005 0 0 0 0.005 0 0 +1.26831 0.005 0 0 0 0.005 0 0 +1.27075 0.005 0 0 0 0.005 0 0 +1.27319 0.005 0 0 0 0.005 0 0 +1.27563 0.005 0 0 0 0.005 0 0 +1.27808 0.005 0 0 0 0.005 0 0 +1.28052 0.005 0 0 0 0.005 0 0 +1.28296 0.005 0 0 0 0.005 0 0 +1.2854 0.005 0 0 0 0.005 0 0 +1.28784 0.005 0 0 0 0.005 0 0 +1.29028 0.005 0 0 0 0.005 0 0 +1.29272 0.005 0 0 0 0.005 0 0 +1.29517 0.005 0 0 0 0.005 0 0 +1.29761 0.005 0 0 0 0.005 0 0 +1.30005 0.005 0 0 0 0.005 0 0 +1.30249 0.005 0 0 0 0.005 0 0 +1.30493 0.005 0 0 0 0.005 0 0 +1.30737 0.005 0 0 0 0.005 0 0 +1.30981 0.005 0 0 0 0.005 0 0 +1.31226 0.005 0 0 0 0.005 0 0 +1.3147 0.005 0 0 0 0.005 0 0 +1.31714 0.005 0 0 0 0.005 0 0 +1.31958 0.005 0 0 0 0.005 0 0 +1.32202 0.005 0 0 0 0.005 0 0 +1.32446 0.005 0 0 0 0.005 0 0 +1.3269 0.005 0 0 0 0.005 0 0 +1.32935 0.005 0 0 0 0.005 0 0 +1.33179 0.005 0 0 0 0.005 0 0 +1.33423 0.005 0 0 0 0.005 0 0 +1.33667 0.005 0 0 0 0.005 0 0 +1.33911 0.005 0 0 0 0.005 0 0 +1.34155 0.005 0 0 0 0.005 0 0 +1.34399 0.005 0 0 0 0.005 0 0 +1.34644 0.005 0 0 0 0.005 0 0 +1.34888 0.005 0 0 0 0.005 0 0 +1.35132 0.005 0 0 0 0.005 0 0 +1.35376 0.005 0 0 0 0.005 0 0 +1.3562 0.005 0 0 0 0.005 0 0 +1.35864 0.005 0 0 0 0.005 0 0 +1.36108 0.005 0 0 0 0.005 0 0 +1.36353 0.005 0 0 0 0.005 0 0 +1.36597 0.005 0 0 0 0.005 0 0 +1.36841 0.005 0 0 0 0.005 0 0 +1.37085 0.005 0 0 0 0.005 0 0 +1.37329 0.005 0 0 0 0.005 0 0 +1.37573 0.005 0 0 0 0.005 0 0 +1.37817 0.005 0 0 0 0.005 0 0 +1.38062 0.005 0 0 0 0.005 0 0 +1.38306 0.005 0 0 0 0.005 0 0 +1.3855 0.005 0 0 0 0.005 0 0 +1.38794 0.005 0 0 0 0.005 0 0 +1.39038 0.005 0 0 0 0.005 0 0 +1.39282 0.005 0 0 0 0.005 0 0 +1.39526 0.005 0 0 0 0.005 0 0 +1.39771 0.005 0 0 0 0.005 0 0 +1.40015 0.005 0 0 0 0.005 0 0 +1.40259 0.005 0 0 0 0.005 0 0 +1.40503 0.005 0 0 0 0.005 0 0 +1.40747 0.005 0 0 0 0.005 0 0 +1.40991 0.005 0 0 0 0.005 0 0 +1.41235 0.005 0 0 0 0.005 0 0 +1.41479 0.005 0 0 0 0.005 0 0 +1.41724 0.005 0 0 0 0.005 0 0 +1.41968 0.005 0 0 0 0.005 0 0 +1.42212 0.005 0 0 0 0.005 0 0 +1.42456 0.005 0 0 0 0.005 0 0 +1.427 0.005 0 0 0 0.005 0 0 +1.42944 0.005 0 0 0 0.005 0 0 +1.43188 0.005 0 0 0 0.005 0 0 +1.43433 0.005 0 0 0 0.005 0 0 +1.43677 0.005 0 0 0 0.005 0 0 +1.43921 0.005 0 0 0 0.005 0 0 +1.44165 0.005 0 0 0 0.005 0 0 +1.44409 0.005 0 0 0 0.005 0 0 +1.44653 0.005 0 0 0 0.005 0 0 +1.44897 0.005 0 0 0 0.005 0 0 +1.45142 0.005 0 0 0 0.005 0 0 +1.45386 0.005 0 0 0 0.005 0 0 +1.4563 0.005 0 0 0 0.005 0 0 +1.45874 0.005 0 0 0 0.005 0 0 +1.46118 0.005 0 0 0 0.005 0 0 +1.46362 0.005 0 0 0 0.005 0 0 +1.46606 0.005 0 0 0 0.005 0 0 +1.46851 0.005 0 0 0 0.005 0 0 +1.47095 0.005 0 0 0 0.005 0 0 +1.47339 0.005 0 0 0 0.005 0 0 +1.47583 0.005 0 0 0 0.005 0 0 +1.47827 0.005 0 0 0 0.005 0 0 +1.48071 0.005 0 0 0 0.005 0 0 +1.48315 0.005 0 0 0 0.005 0 0 +1.4856 0.005 0 0 0 0.005 0 0 +1.48804 0.005 0 0 0 0.005 0 0 +1.49048 0.005 0 0 0 0.005 0 0 +1.49292 0.005 0 0 0 0.005 0 0 +1.49536 0.005 0 0 0 0.005 0 0 +1.4978 0.005 0 0 0 0.005 0 0 +1.50024 0.005 0 0 0 0.005 0 0 +1.50269 0.005 0 0 0 0.005 0 0 +1.50513 0.005 0 0 0 0.005 0 0 +1.50757 0.005 0 0 0 0.005 0 0 +1.51001 0.005 0 0 0 0.005 0 0 +1.51245 0.005 0 0 0 0.005 0 0 +1.51489 0.005 0 0 0 0.005 0 0 +1.51733 0.005 0 0 0 0.005 0 0 +1.51978 0.005 0 0 0 0.005 0 0 +1.52222 0.005 0 0 0 0.005 0 0 +1.52466 0.005 0 0 0 0.005 0 0 +1.5271 0.005 0 0 0 0.005 0 0 +1.52954 0.005 0 0 0 0.005 0 0 +1.53198 0.005 0 0 0 0.005 0 0 +1.53442 0.005 0 0 0 0.005 0 0 +1.53687 0.005 0 0 0 0.005 0 0 +1.53931 0.005 0 0 0 0.005 0 0 +1.54175 0.005 0 0 0 0.005 0 0 +1.54419 0.005 0 0 0 0.005 0 0 +1.54663 0.005 0 0 0 0.005 0 0 +1.54907 0.005 0 0 0 0.005 0 0 +1.55151 0.005 0 0 0 0.005 0 0 +1.55396 0.005 0 0 0 0.005 0 0 +1.5564 0.005 0 0 0 0.005 0 0 +1.55884 0.005 0 0 0 0.005 0 0 +1.56128 0.005 0 0 0 0.005 0 0 +1.56372 0.005 0 0 0 0.005 0 0 +1.56616 0.005 0 0 0 0.005 0 0 +1.5686 0.005 0 0 0 0.005 0 0 +1.57104 0.005 0 0 0 0.005 0 0 +1.57349 0.005 0 0 0 0.005 0 0 +1.57593 0.005 0 0 0 0.005 0 0 +1.57837 0.005 0 0 0 0.005 0 0 +1.58081 0.005 0 0 0 0.005 0 0 +1.58325 0.005 0 0 0 0.005 0 0 +1.58569 0.005 0 0 0 0.005 0 0 +1.58813 0.005 0 0 0 0.005 0 0 +1.59058 0.005 0 0 0 0.005 0 0 +1.59302 0.005 0 0 0 0.005 0 0 +1.59546 0.005 0 0 0 0.005 0 0 +1.5979 0.005 0 0 0 0.005 0 0 +1.60034 0.005 0 0 0 0.005 0 0 +1.60278 0.005 0 0 0 0.005 0 0 +1.60522 0.005 0 0 0 0.005 0 0 +1.60767 0.005 0 0 0 0.005 0 0 +1.61011 0.005 0 0 0 0.005 0 0 +1.61255 0.005 0 0 0 0.005 0 0 +1.61499 0.005 0 0 0 0.005 0 0 +1.61743 0.005 0 0 0 0.005 0 0 +1.61987 0.005 0 0 0 0.005 0 0 +1.62231 0.005 0 0 0 0.005 0 0 +1.62476 0.005 0 0 0 0.005 0 0 +1.6272 0.005 0 0 0 0.005 0 0 +1.62964 0.005 0 0 0 0.005 0 0 +1.63208 0.005 0 0 0 0.005 0 0 +1.63452 0.005 0 0 0 0.005 0 0 +1.63696 0.005 0 0 0 0.005 0 0 +1.6394 0.005 0 0 0 0.005 0 0 +1.64185 0.005 0 0 0 0.005 0 0 +1.64429 0.005 0 0 0 0.005 0 0 +1.64673 0.005 0 0 0 0.005 0 0 +1.64917 0.005 0 0 0 0.005 0 0 +1.65161 0.005 0 0 0 0.005 0 0 +1.65405 0.005 0 0 0 0.005 0 0 +1.65649 0.005 0 0 0 0.005 0 0 +1.65894 0.005 0 0 0 0.005 0 0 +1.66138 0.005 0 0 0 0.005 0 0 +1.66382 0.005 0 0 0 0.005 0 0 +1.66626 0.005 0 0 0 0.005 0 0 +1.6687 0.005 0 0 0 0.005 0 0 +1.67114 0.005 0 0 0 0.005 0 0 +1.67358 0.005 0 0 0 0.005 0 0 +1.67603 0.005 0 0 0 0.005 0 0 +1.67847 0.005 0 0 0 0.005 0 0 +1.68091 0.005 0 0 0 0.005 0 0 +1.68335 0.005 0 0 0 0.005 0 0 +1.68579 0.005 0 0 0 0.005 0 0 +1.68823 0.005 0 0 0 0.005 0 0 +1.69067 0.005 0 0 0 0.005 0 0 +1.69312 0.005 0 0 0 0.005 0 0 +1.69556 0.005 0 0 0 0.005 0 0 +1.698 0.005 0 0 0 0.005 0 0 +1.70044 0.005 0 0 0 0.005 0 0 +1.70288 0.005 0 0 0 0.005 0 0 +1.70532 0.005 0 0 0 0.005 0 0 +1.70776 0.005 0 0 0 0.005 0 0 +1.71021 0.005 0 0 0 0.005 0 0 +1.71265 0.005 0 0 0 0.005 0 0 +1.71509 0.005 0 0 0 0.005 0 0 +1.71753 0.005 0 0 0 0.005 0 0 +1.71997 0.005 0 0 0 0.005 0 0 +1.72241 0.005 0 0 0 0.005 0 0 +1.72485 0.005 0 0 0 0.005 0 0 +1.72729 0.005 0 0 0 0.005 0 0 +1.72974 0.005 0 0 0 0.005 0 0 +1.73218 0.005 0 0 0 0.005 0 0 +1.73462 0.005 0 0 0 0.005 0 0 +1.73706 0.005 0 0 0 0.005 0 0 +1.7395 0.005 0 0 0 0.005 0 0 +1.74194 0.005 0 0 0 0.005 0 0 +1.74438 0.005 0 0 0 0.005 0 0 +1.74683 0.005 0 0 0 0.005 0 0 +1.74927 0.005 0 0 0 0.005 0 0 +1.75171 0.005 0 0 0 0.005 0 0 +1.75415 0.005 0 0 0 0.005 0 0 +1.75659 0.005 0 0 0 0.005 0 0 +1.75903 0.005 0 0 0 0.005 0 0 +1.76147 0.005 0 0 0 0.005 0 0 +1.76392 0.005 0 0 0 0.005 0 0 +1.76636 0.005 0 0 0 0.005 0 0 +1.7688 0.005 0 0 0 0.005 0 0 +1.77124 0.005 0 0 0 0.005 0 0 +1.77368 0.005 0 0 0 0.005 0 0 +1.77612 0.005 0 0 0 0.005 0 0 +1.77856 0.005 0 0 0 0.005 0 0 +1.78101 0.005 0 0 0 0.005 0 0 +1.78345 0.005 0 0 0 0.005 0 0 +1.78589 0.005 0 0 0 0.005 0 0 +1.78833 0.005 0 0 0 0.005 0 0 +1.79077 0.005 0 0 0 0.005 0 0 +1.79321 0.005 0 0 0 0.005 0 0 +1.79565 0.005 0 0 0 0.005 0 0 +1.7981 0.005 0 0 0 0.005 0 0 +1.80054 0.005 0 0 0 0.005 0 0 +1.80298 0.005 0 0 0 0.005 0 0 +1.80542 0.005 0 0 0 0.005 0 0 +1.80786 0.005 0 0 0 0.005 0 0 +1.8103 0.005 0 0 0 0.005 0 0 +1.81274 0.005 0 0 0 0.005 0 0 +1.81519 0.005 0 0 0 0.005 0 0 +1.81763 0.005 0 0 0 0.005 0 0 +1.82007 0.005 0 0 0 0.005 0 0 +1.82251 0.005 0 0 0 0.005 0 0 +1.82495 0.005 0 0 0 0.005 0 0 +1.82739 0.005 0 0 0 0.005 0 0 +1.82983 0.005 0 0 0 0.005 0 0 +1.83228 0.005 0 0 0 0.005 0 0 +1.83472 0.005 0 0 0 0.005 0 0 +1.83716 0.005 0 0 0 0.005 0 0 +1.8396 0.005 0 0 0 0.005 0 0 +1.84204 0.005 0 0 0 0.005 0 0 +1.84448 0.005 0 0 0 0.005 0 0 +1.84692 0.005 0 0 0 0.005 0 0 +1.84937 0.005 0 0 0 0.005 0 0 +1.85181 0.005 0 0 0 0.005 0 0 +1.85425 0.005 0 0 0 0.005 0 0 +1.85669 0.005 0 0 0 0.005 0 0 +1.85913 0.005 0 0 0 0.005 0 0 +1.86157 0.005 0 0 0 0.005 0 0 +1.86401 0.005 0 0 0 0.005 0 0 +1.86646 0.005 0 0 0 0.005 0 0 +1.8689 0.005 0 0 0 0.005 0 0 +1.87134 0.005 0 0 0 0.005 0 0 +1.87378 0.005 0 0 0 0.005 0 0 +1.87622 0.005 0 0 0 0.005 0 0 +1.87866 0.005 0 0 0 0.005 0 0 +1.8811 0.005 0 0 0 0.005 0 0 +1.88354 0.005 0 0 0 0.005 0 0 +1.88599 0.005 0 0 0 0.005 0 0 +1.88843 0.005 0 0 0 0.005 0 0 +1.89087 0.005 0 0 0 0.005 0 0 +1.89331 0.005 0 0 0 0.005 0 0 +1.89575 0.005 0 0 0 0.005 0 0 +1.89819 0.005 0 0 0 0.005 0 0 +1.90063 0.005 0 0 0 0.005 0 0 +1.90308 0.005 0 0 0 0.005 0 0 +1.90552 0.005 0 0 0 0.005 0 0 +1.90796 0.005 0 0 0 0.005 0 0 +1.9104 0.005 0 0 0 0.005 0 0 +1.91284 0.005 0 0 0 0.005 0 0 +1.91528 0.005 0 0 0 0.005 0 0 +1.91772 0.005 0 0 0 0.005 0 0 +1.92017 0.005 0 0 0 0.005 0 0 +1.92261 0.005 0 0 0 0.005 0 0 +1.92505 0.005 0 0 0 0.005 0 0 +1.92749 0.005 0 0 0 0.005 0 0 +1.92993 0.005 0 0 0 0.005 0 0 +1.93237 0.005 0 0 0 0.005 0 0 +1.93481 0.005 0 0 0 0.005 0 0 +1.93726 0.005 0 0 0 0.005 0 0 +1.9397 0.005 0 0 0 0.005 0 0 +1.94214 0.005 0 0 0 0.005 0 0 +1.94458 0.005 0 0 0 0.005 0 0 +1.94702 0.005 0 0 0 0.005 0 0 +1.94946 0.005 0 0 0 0.005 0 0 +1.9519 0.005 0 0 0 0.005 0 0 +1.95435 0.005 0 0 0 0.005 0 0 +1.95679 0.005 0 0 0 0.005 0 0 +1.95923 0.005 0 0 0 0.005 0 0 +1.96167 0.005 0 0 0 0.005 0 0 +1.96411 0.005 0 0 0 0.005 0 0 +1.96655 0.005 0 0 0 0.005 0 0 +1.96899 0.005 0 0 0 0.005 0 0 +1.97144 0.005 0 0 0 0.005 0 0 +1.97388 0.005 0 0 0 0.005 0 0 +1.97632 0.005 0 0 0 0.005 0 0 +1.97876 0.005 0 0 0 0.005 0 0 +1.9812 0.005 0 0 0 0.005 0 0 +1.98364 0.005 0 0 0 0.005 0 0 +1.98608 0.005 0 0 0 0.005 0 0 +1.98853 0.005 0 0 0 0.005 0 0 +1.99097 0.005 0 0 0 0.005 0 0 +1.99341 0.005 0 0 0 0.005 0 0 +1.99585 0.005 0 0 0 0.005 0 0 +1.99829 0.005 0 0 0 0.005 0 0 +2.00073 0.005 0 0 0 0.005 0 0 +2.00317 0.005 0 0 0 0.005 0 0 +2.00562 0.005 0 0 0 0.005 0 0 +2.00806 0.005 0 0 0 0.005 0 0 +2.0105 0.005 0 0 0 0.005 0 0 +2.01294 0.005 0 0 0 0.005 0 0 +2.01538 0.005 0 0 0 0.005 0 0 +2.01782 0.005 0 0 0 0.005 0 0 +2.02026 0.005 0 0 0 0.005 0 0 +2.02271 0.005 0 0 0 0.005 0 0 +2.02515 0.005 0 0 0 0.005 0 0 +2.02759 0.005 0 0 0 0.005 0 0 +2.03003 0.005 0 0 0 0.005 0 0 +2.03247 0.005 0 0 0 0.005 0 0 +2.03491 0.005 0 0 0 0.005 0 0 +2.03735 0.005 0 0 0 0.005 0 0 +2.03979 0.005 0 0 0 0.005 0 0 +2.04224 0.005 0 0 0 0.005 0 0 +2.04468 0.005 0 0 0 0.005 0 0 +2.04712 0.005 0 0 0 0.005 0 0 +2.04956 0.005 0 0 0 0.005 0 0 +2.052 0.005 0 0 0 0.005 0 0 +2.05444 0.005 0 0 0 0.005 0 0 +2.05688 0.005 0 0 0 0.005 0 0 +2.05933 0.005 0 0 0 0.005 0 0 +2.06177 0.005 0 0 0 0.005 0 0 +2.06421 0.005 0 0 0 0.005 0 0 +2.06665 0.005 0 0 0 0.005 0 0 +2.06909 0.005 0 0 0 0.005 0 0 +2.07153 0.005 0 0 0 0.005 0 0 +2.07397 0.005 0 0 0 0.005 0 0 +2.07642 0.005 0 0 0 0.005 0 0 +2.07886 0.005 0 0 0 0.005 0 0 +2.0813 0.005 0 0 0 0.005 0 0 +2.08374 0.005 0 0 0 0.005 0 0 +2.08618 0.005 0 0 0 0.005 0 0 +2.08862 0.005 0 0 0 0.005 0 0 +2.09106 0.005 0 0 0 0.005 0 0 +2.09351 0.005 0 0 0 0.005 0 0 +2.09595 0.005 0 0 0 0.005 0 0 +2.09839 0.005 0 0 0 0.005 0 0 +2.10083 0.005 0 0 0 0.005 0 0 +2.10327 0.005 0 0 0 0.005 0 0 +2.10571 0.005 0 0 0 0.005 0 0 +2.10815 0.005 0 0 0 0.005 0 0 +2.1106 0.005 0 0 0 0.005 0 0 +2.11304 0.005 0 0 0 0.005 0 0 +2.11548 0.005 0 0 0 0.005 0 0 +2.11792 0.005 0 0 0 0.005 0 0 +2.12036 0.005 0 0 0 0.005 0 0 +2.1228 0.005 0 0 0 0.005 0 0 +2.12524 0.005 0 0 0 0.005 0 0 +2.12769 0.005 0 0 0 0.005 0 0 +2.13013 0.005 0 0 0 0.005 0 0 +2.13257 0.005 0 0 0 0.005 0 0 +2.13501 0.005 0 0 0 0.005 0 0 +2.13745 0.005 0 0 0 0.005 0 0 +2.13989 0.005 0 0 0 0.005 0 0 +2.14233 0.005 0 0 0 0.005 0 0 +2.14478 0.005 0 0 0 0.005 0 0 +2.14722 0.005 0 0 0 0.005 0 0 +2.14966 0.005 0 0 0 0.005 0 0 +2.1521 0.005 0 0 0 0.005 0 0 +2.15454 0.005 0 0 0 0.005 0 0 +2.15698 0.005 0 0 0 0.005 0 0 +2.15942 0.005 0 0 0 0.005 0 0 +2.16187 0.005 0 0 0 0.005 0 0 +2.16431 0.005 0 0 0 0.005 0 0 +2.16675 0.005 0 0 0 0.005 0 0 +2.16919 0.005 0 0 0 0.005 0 0 +2.17163 0.005 0 0 0 0.005 0 0 +2.17407 0.005 0 0 0 0.005 0 0 +2.17651 0.005 0 0 0 0.005 0 0 +2.17896 0.005 0 0 0 0.005 0 0 +2.1814 0.005 0 0 0 0.005 0 0 +2.18384 0.005 0 0 0 0.005 0 0 +2.18628 0.005 0 0 0 0.005 0 0 +2.18872 0.005 0 0 0 0.005 0 0 +2.19116 0.005 0 0 0 0.005 0 0 +2.1936 0.005 0 0 0 0.005 0 0 +2.19604 0.005 0 0 0 0.005 0 0 +2.19849 0.005 0 0 0 0.005 0 0 +2.20093 0.005 0 0 0 0.005 0 0 +2.20337 0.005 0 0 0 0.005 0 0 +2.20581 0.005 0 0 0 0.005 0 0 +2.20825 0.005 0 0 0 0.005 0 0 +2.21069 0.005 0 0 0 0.005 0 0 +2.21313 0.005 0 0 0 0.005 0 0 +2.21558 0.005 0 0 0 0.005 0 0 +2.21802 0.005 0 0 0 0.005 0 0 +2.22046 0.005 0 0 0 0.005 0 0 +2.2229 0.005 0 0 0 0.005 0 0 +2.22534 0.005 0 0 0 0.005 0 0 +2.22778 0.005 0 0 0 0.005 0 0 +2.23022 0.005 0 0 0 0.005 0 0 +2.23267 0.005 0 0 0 0.005 0 0 +2.23511 0.005 0 0 0 0.005 0 0 +2.23755 0.005 0 0 0 0.005 0 0 +2.23999 0.005 0 0 0 0.005 0 0 +2.24243 0.005 0 0 0 0.005 0 0 +2.24487 0.005 0 0 0 0.005 0 0 +2.24731 0.005 0 0 0 0.005 0 0 +2.24976 0.005 0 0 0 0.005 0 0 +2.2522 0.005 0 0 0 0.005 0 0 +2.25464 0.005 0 0 0 0.005 0 0 +2.25708 0.005 0 0 0 0.005 0 0 +2.25952 0.005 0 0 0 0.005 0 0 +2.26196 0.005 0 0 0 0.005 0 0 +2.2644 0.005 0 0 0 0.005 0 0 +2.26685 0.005 0 0 0 0.005 0 0 +2.26929 0.005 0 0 0 0.005 0 0 +2.27173 0.005 0 0 0 0.005 0 0 +2.27417 0.005 0 0 0 0.005 0 0 +2.27661 0.005 0 0 0 0.005 0 0 +2.27905 0.005 0 0 0 0.005 0 0 +2.28149 0.005 0 0 0 0.005 0 0 +2.28394 0.005 0 0 0 0.005 0 0 +2.28638 0.005 0 0 0 0.005 0 0 +2.28882 0.005 0 0 0 0.005 0 0 +2.29126 0.005 0 0 0 0.005 0 0 +2.2937 0.005 0 0 0 0.005 0 0 +2.29614 0.005 0 0 0 0.005 0 0 +2.29858 0.005 0 0 0 0.005 0 0 +2.30103 0.005 0 0 0 0.005 0 0 +2.30347 0.005 0 0 0 0.005 0 0 +2.30591 0.005 0 0 0 0.005 0 0 +2.30835 0.005 0 0 0 0.005 0 0 +2.31079 0.005 0 0 0 0.005 0 0 +2.31323 0.005 0 0 0 0.005 0 0 +2.31567 0.005 0 0 0 0.005 0 0 +2.31812 0.005 0 0 0 0.005 0 0 +2.32056 0.005 0 0 0 0.005 0 0 +2.323 0.005 0 0 0 0.005 0 0 +2.32544 0.005 0 0 0 0.005 0 0 +2.32788 0.005 0 0 0 0.005 0 0 +2.33032 0.005 0 0 0 0.005 0 0 +2.33276 0.005 0 0 0 0.005 0 0 +2.33521 0.005 0 0 0 0.005 0 0 +2.33765 0.005 0 0 0 0.005 0 0 +2.34009 0.005 0 0 0 0.005 0 0 +2.34253 0.005 0 0 0 0.005 0 0 +2.34497 0.005 0 0 0 0.005 0 0 +2.34741 0.005 0 0 0 0.005 0 0 +2.34985 0.005 0 0 0 0.005 0 0 +2.35229 0.005 0 0 0 0.005 0 0 +2.35474 0.005 0 0 0 0.005 0 0 +2.35718 0.005 0 0 0 0.005 0 0 +2.35962 0.005 0 0 0 0.005 0 0 +2.36206 0.005 0 0 0 0.005 0 0 +2.3645 0.005 0 0 0 0.005 0 0 +2.36694 0.005 0 0 0 0.005 0 0 +2.36938 0.005 0 0 0 0.005 0 0 +2.37183 0.005 0 0 0 0.005 0 0 +2.37427 0.005 0 0 0 0.005 0 0 +2.37671 0.005 0 0 0 0.005 0 0 +2.37915 0.005 0 0 0 0.005 0 0 +2.38159 0.005 0 0 0 0.005 0 0 +2.38403 0.005 0 0 0 0.005 0 0 +2.38647 0.005 0 0 0 0.005 0 0 +2.38892 0.005 0 0 0 0.005 0 0 +2.39136 0.005 0 0 0 0.005 0 0 +2.3938 0.005 0 0 0 0.005 0 0 +2.39624 0.005 0 0 0 0.005 0 0 +2.39868 0.005 0 0 0 0.005 0 0 +2.40112 0.005 0 0 0 0.005 0 0 +2.40356 0.005 0 0 0 0.005 0 0 +2.40601 0.005 0 0 0 0.005 0 0 +2.40845 0.005 0 0 0 0.005 0 0 +2.41089 0.005 0 0 0 0.005 0 0 +2.41333 0.005 0 0 0 0.005 0 0 +2.41577 0.005 0 0 0 0.005 0 0 +2.41821 0.005 0 0 0 0.005 0 0 +2.42065 0.005 0 0 0 0.005 0 0 +2.4231 0.005 0 0 0 0.005 0 0 +2.42554 0.005 0 0 0 0.005 0 0 +2.42798 0.005 0 0 0 0.005 0 0 +2.43042 0.005 0 0 0 0.005 0 0 +2.43286 0.005 0 0 0 0.005 0 0 +2.4353 0.005 0 0 0 0.005 0 0 +2.43774 0.005 0 0 0 0.005 0 0 +2.44019 0.005 0 0 0 0.005 0 0 +2.44263 0.005 0 0 0 0.005 0 0 +2.44507 0.005 0 0 0 0.005 0 0 +2.44751 0.005 0 0 0 0.005 0 0 +2.44995 0.005 0 0 0 0.005 0 0 +2.45239 0.005 0 0 0 0.005 0 0 +2.45483 0.005 0 0 0 0.005 0 0 +2.45728 0.005 0 0 0 0.005 0 0 +2.45972 0.005 0 0 0 0.005 0 0 +2.46216 0.005 0 0 0 0.005 0 0 +2.4646 0.005 0 0 0 0.005 0 0 +2.46704 0.005 0 0 0 0.005 0 0 +2.46948 0.005 0 0 0 0.005 0 0 +2.47192 0.005 0 0 0 0.005 0 0 +2.47437 0.005 0 0 0 0.005 0 0 +2.47681 0.005 0 0 0 0.005 0 0 +2.47925 0.005 0 0 0 0.005 0 0 +2.48169 0.005 0 0 0 0.005 0 0 +2.48413 0.005 0 0 0 0.005 0 0 +2.48657 0.005 0 0 0 0.005 0 0 +2.48901 0.005 0 0 0 0.005 0 0 +2.49146 0.005 0 0 0 0.005 0 0 +2.4939 0.005 0 0 0 0.005 0 0 +2.49634 0.005 0 0 0 0.005 0 0 +2.49878 0.005 0 0 0 0.005 0 0 +2.50122 0.005 0 0 0 0.005 0 0 +2.50366 0.005 0 0 0 0.005 0 0 +2.5061 0.005 0 0 0 0.005 0 0 +2.50854 0.005 0 0 0 0.005 0 0 +2.51099 0.005 0 0 0 0.005 0 0 +2.51343 0.005 0 0 0 0.005 0 0 +2.51587 0.005 0 0 0 0.005 0 0 +2.51831 0.005 0 0 0 0.005 0 0 +2.52075 0.005 0 0 0 0.005 0 0 +2.52319 0.005 0 0 0 0.005 0 0 +2.52563 0.005 0 0 0 0.005 0 0 +2.52808 0.005 0 0 0 0.005 0 0 +2.53052 0.005 0 0 0 0.005 0 0 +2.53296 0.005 0 0 0 0.005 0 0 +2.5354 0.005 0 0 0 0.005 0 0 +2.53784 0.005 0 0 0 0.005 0 0 +2.54028 0.005 0 0 0 0.005 0 0 +2.54272 0.005 0 0 0 0.005 0 0 +2.54517 0.005 0 0 0 0.005 0 0 +2.54761 0.005 0 0 0 0.005 0 0 +2.55005 0.005 0 0 0 0.005 0 0 +2.55249 0.005 0 0 0 0.005 0 0 +2.55493 0.005 0 0 0 0.005 0 0 +2.55737 0.005 0 0 0 0.005 0 0 +2.55981 0.005 0 0 0 0.005 0 0 +2.56226 0.005 0 0 0 0.005 0 0 +2.5647 0.005 0 0 0 0.005 0 0 +2.56714 0.005 0 0 0 0.005 0 0 +2.56958 0.005 0 0 0 0.005 0 0 +2.57202 0.005 0 0 0 0.005 0 0 +2.57446 0.005 0 0 0 0.005 0 0 +2.5769 0.005 0 0 0 0.005 0 0 +2.57935 0.005 0 0 0 0.005 0 0 +2.58179 0.005 0 0 0 0.005 0 0 +2.58423 0.005 0 0 0 0.005 0 0 +2.58667 0.005 0 0 0 0.005 0 0 +2.58911 0.005 0 0 0 0.005 0 0 +2.59155 0.005 0 0 0 0.005 0 0 +2.59399 0.005 0 0 0 0.005 0 0 +2.59644 0.005 0 0 0 0.005 0 0 +2.59888 0.005 0 0 0 0.005 0 0 +2.60132 0.005 0 0 0 0.005 0 0 +2.60376 0.005 0 0 0 0.005 0 0 +2.6062 0.005 0 0 0 0.005 0 0 +2.60864 0.005 0 0 0 0.005 0 0 +2.61108 0.005 0 0 0 0.005 0 0 +2.61353 0.005 0 0 0 0.005 0 0 +2.61597 0.005 0 0 0 0.005 0 0 +2.61841 0.005 0 0 0 0.005 0 0 +2.62085 0.005 0 0 0 0.005 0 0 +2.62329 0.005 0 0 0 0.005 0 0 +2.62573 0.005 0 0 0 0.005 0 0 +2.62817 0.005 0 0 0 0.005 0 0 +2.63062 0.005 0 0 0 0.005 0 0 +2.63306 0.005 0 0 0 0.005 0 0 +2.6355 0.005 0 0 0 0.005 0 0 +2.63794 0.005 0 0 0 0.005 0 0 +2.64038 0.005 0 0 0 0.005 0 0 +2.64282 0.005 0 0 0 0.005 0 0 +2.64526 0.005 0 0 0 0.005 0 0 +2.64771 0.005 0 0 0 0.005 0 0 +2.65015 0.005 0 0 0 0.005 0 0 +2.65259 0.005 0 0 0 0.005 0 0 +2.65503 0.005 0 0 0 0.005 0 0 +2.65747 0.005 0 0 0 0.005 0 0 +2.65991 0.005 0 0 0 0.005 0 0 +2.66235 0.005 0 0 0 0.005 0 0 +2.66479 0.005 0 0 0 0.005 0 0 +2.66724 0.005 0 0 0 0.005 0 0 +2.66968 0.005 0 0 0 0.005 0 0 +2.67212 0.005 0 0 0 0.005 0 0 +2.67456 0.005 0 0 0 0.005 0 0 +2.677 0.005 0 0 0 0.005 0 0 +2.67944 0.005 0 0 0 0.005 0 0 +2.68188 0.005 0 0 0 0.005 0 0 +2.68433 0.005 0 0 0 0.005 0 0 +2.68677 0.005 0 0 0 0.005 0 0 +2.68921 0.005 0 0 0 0.005 0 0 +2.69165 0.005 0 0 0 0.005 0 0 +2.69409 0.005 0 0 0 0.005 0 0 +2.69653 0.005 0 0 0 0.005 0 0 +2.69897 0.005 0 0 0 0.005 0 0 +2.70142 0.005 0 0 0 0.005 0 0 +2.70386 0.005 0 0 0 0.005 0 0 +2.7063 0.005 0 0 0 0.005 0 0 +2.70874 0.005 0 0 0 0.005 0 0 +2.71118 0.005 0 0 0 0.005 0 0 +2.71362 0.005 0 0 0 0.005 0 0 +2.71606 0.005 0 0 0 0.005 0 0 +2.71851 0.005 0 0 0 0.005 0 0 +2.72095 0.005 0 0 0 0.005 0 0 +2.72339 0.005 0 0 0 0.005 0 0 +2.72583 0.005 0 0 0 0.005 0 0 +2.72827 0.005 0 0 0 0.005 0 0 +2.73071 0.005 0 0 0 0.005 0 0 +2.73315 0.005 0 0 0 0.005 0 0 +2.7356 0.005 0 0 0 0.005 0 0 +2.73804 0.005 0 0 0 0.005 0 0 +2.74048 0.005 0 0 0 0.005 0 0 +2.74292 0.005 0 0 0 0.005 0 0 +2.74536 0.005 0 0 0 0.005 0 0 +2.7478 0.005 0 0 0 0.005 0 0 +2.75024 0.005 0 0 0 0.005 0 0 +2.75269 0.005 0 0 0 0.005 0 0 +2.75513 0.005 0 0 0 0.005 0 0 +2.75757 0.005 0 0 0 0.005 0 0 +2.76001 0.005 0 0 0 0.005 0 0 +2.76245 0.005 0 0 0 0.005 0 0 +2.76489 0.005 0 0 0 0.005 0 0 +2.76733 0.005 0 0 0 0.005 0 0 +2.76978 0.005 0 0 0 0.005 0 0 +2.77222 0.005 0 0 0 0.005 0 0 +2.77466 0.005 0 0 0 0.005 0 0 +2.7771 0.005 0 0 0 0.005 0 0 +2.77954 0.005 0 0 0 0.005 0 0 +2.78198 0.005 0 0 0 0.005 0 0 +2.78442 0.005 0 0 0 0.005 0 0 +2.78687 0.005 0 0 0 0.005 0 0 +2.78931 0.005 0 0 0 0.005 0 0 +2.79175 0.005 0 0 0 0.005 0 0 +2.79419 0.005 0 0 0 0.005 0 0 +2.79663 0.005 0 0 0 0.005 0 0 +2.79907 0.005 0 0 0 0.005 0 0 +2.80151 0.005 0 0 0 0.005 0 0 +2.80396 0.005 0 0 0 0.005 0 0 +2.8064 0.005 0 0 0 0.005 0 0 +2.80884 0.005 0 0 0 0.005 0 0 +2.81128 0.005 0 0 0 0.005 0 0 +2.81372 0.005 0 0 0 0.005 0 0 +2.81616 0.005 0 0 0 0.005 0 0 +2.8186 0.005 0 0 0 0.005 0 0 +2.82104 0.005 0 0 0 0.005 0 0 +2.82349 0.005 0 0 0 0.005 0 0 +2.82593 0.005 0 0 0 0.005 0 0 +2.82837 0.005 0 0 0 0.005 0 0 +2.83081 0.005 0 0 0 0.005 0 0 +2.83325 0.005 0 0 0 0.005 0 0 +2.83569 0.005 0 0 0 0.005 0 0 +2.83813 0.005 0 0 0 0.005 0 0 +2.84058 0.005 0 0 0 0.005 0 0 +2.84302 0.005 0 0 0 0.005 0 0 +2.84546 0.005 0 0 0 0.005 0 0 +2.8479 0.005 0 0 0 0.005 0 0 +2.85034 0.005 0 0 0 0.005 0 0 +2.85278 0.005 0 0 0 0.005 0 0 +2.85522 0.005 0 0 0 0.005 0 0 +2.85767 0.005 0 0 0 0.005 0 0 +2.86011 0.005 0 0 0 0.005 0 0 +2.86255 0.005 0 0 0 0.005 0 0 +2.86499 0.005 0 0 0 0.005 0 0 +2.86743 0.005 0 0 0 0.005 0 0 +2.86987 0.005 0 0 0 0.005 0 0 +2.87231 0.005 0 0 0 0.005 0 0 +2.87476 0.005 0 0 0 0.005 0 0 +2.8772 0.005 0 0 0 0.005 0 0 +2.87964 0.005 0 0 0 0.005 0 0 +2.88208 0.005 0 0 0 0.005 0 0 +2.88452 0.005 0 0 0 0.005 0 0 +2.88696 0.005 0 0 0 0.005 0 0 +2.8894 0.005 0 0 0 0.005 0 0 +2.89185 0.005 0 0 0 0.005 0 0 +2.89429 0.005 0 0 0 0.005 0 0 +2.89673 0.005 0 0 0 0.005 0 0 +2.89917 0.005 0 0 0 0.005 0 0 +2.90161 0.005 0 0 0 0.005 0 0 +2.90405 0.005 0 0 0 0.005 0 0 +2.90649 0.005 0 0 0 0.005 0 0 +2.90894 0.005 0 0 0 0.005 0 0 +2.91138 0.005 0 0 0 0.005 0 0 +2.91382 0.005 0 0 0 0.005 0 0 +2.91626 0.005 0 0 0 0.005 0 0 +2.9187 0.005 0 0 0 0.005 0 0 +2.92114 0.005 0 0 0 0.005 0 0 +2.92358 0.005 0 0 0 0.005 0 0 +2.92603 0.005 0 0 0 0.005 0 0 +2.92847 0.005 0 0 0 0.005 0 0 +2.93091 0.005 0 0 0 0.005 0 0 +2.93335 0.005 0 0 0 0.005 0 0 +2.93579 0.005 0 0 0 0.005 0 0 +2.93823 0.005 0 0 0 0.005 0 0 +2.94067 0.005 0 0 0 0.005 0 0 +2.94312 0.005 0 0 0 0.005 0 0 +2.94556 0.005 0 0 0 0.005 0 0 +2.948 0.005 0 0 0 0.005 0 0 +2.95044 0.005 0 0 0 0.005 0 0 +2.95288 0.005 0 0 0 0.005 0 0 +2.95532 0.005 0 0 0 0.005 0 0 +2.95776 0.005 0 0 0 0.005 0 0 +2.96021 0.005 0 0 0 0.005 0 0 +2.96265 0.005 0 0 0 0.005 0 0 +2.96509 0.005 0 0 0 0.005 0 0 +2.96753 0.005 0 0 0 0.005 0 0 +2.96997 0.005 0 0 0 0.005 0 0 +2.97241 0.005 0 0 0 0.005 0 0 +2.97485 0.005 0 0 0 0.005 0 0 +2.97729 0.005 0 0 0 0.005 0 0 +2.97974 0.005 0 0 0 0.005 0 0 +2.98218 0.005 0 0 0 0.005 0 0 +2.98462 0.005 0 0 0 0.005 0 0 +2.98706 0.005 0 0 0 0.005 0 0 +2.9895 0.005 0 0 0 0.005 0 0 +2.99194 0.005 0 0 0 0.005 0 0 +2.99438 0.005 0 0 0 0.005 0 0 +2.99683 0.005 0 0 0 0.005 0 0 +2.99927 0.005 0 0 0 0.005 0 0 +3.00171 0.005 0 0 0 0.005 0 0 +3.00415 0.005 0 0 0 0.005 0 0 +3.00659 0.005 0 0 0 0.005 0 0 +3.00903 0.005 0 0 0 0.005 0 0 +3.01147 0.005 0 0 0 0.005 0 0 +3.01392 0.005 0 0 0 0.005 0 0 +3.01636 0.005 0 0 0 0.005 0 0 +3.0188 0.005 0 0 0 0.005 0 0 +3.02124 0.005 0 0 0 0.005 0 0 +3.02368 0.005 0 0 0 0.005 0 0 +3.02612 0.005 0 0 0 0.005 0 0 +3.02856 0.005 0 0 0 0.005 0 0 +3.03101 0.005 0 0 0 0.005 0 0 +3.03345 0.005 0 0 0 0.005 0 0 +3.03589 0.005 0 0 0 0.005 0 0 +3.03833 0.005 0 0 0 0.005 0 0 +3.04077 0.005 0 0 0 0.005 0 0 +3.04321 0.005 0 0 0 0.005 0 0 +3.04565 0.005 0 0 0 0.005 0 0 +3.0481 0.005 0 0 0 0.005 0 0 +3.05054 0.005 0 0 0 0.005 0 0 +3.05298 0.005 0 0 0 0.005 0 0 +3.05542 0.005 0 0 0 0.005 0 0 +3.05786 0.005 0 0 0 0.005 0 0 +3.0603 0.005 0 0 0 0.005 0 0 +3.06274 0.005 0 0 0 0.005 0 0 +3.06519 0.005 0 0 0 0.005 0 0 +3.06763 0.005 0 0 0 0.005 0 0 +3.07007 0.005 0 0 0 0.005 0 0 +3.07251 0.005 0 0 0 0.005 0 0 +3.07495 0.005 0 0 0 0.005 0 0 +3.07739 0.005 0 0 0 0.005 0 0 +3.07983 0.005 0 0 0 0.005 0 0 +3.08228 0.005 0 0 0 0.005 0 0 +3.08472 0.005 0 0 0 0.005 0 0 +3.08716 0.005 0 0 0 0.005 0 0 +3.0896 0.005 0 0 0 0.005 0 0 +3.09204 0.005 0 0 0 0.005 0 0 +3.09448 0.005 0 0 0 0.005 0 0 +3.09692 0.005 0 0 0 0.005 0 0 +3.09937 0.005 0 0 0 0.005 0 0 +3.10181 0.005 0 0 0 0.005 0 0 +3.10425 0.005 0 0 0 0.005 0 0 +3.10669 0.005 0 0 0 0.005 0 0 +3.10913 0.005 0 0 0 0.005 0 0 +3.11157 0.005 0 0 0 0.005 0 0 +3.11401 0.005 0 0 0 0.005 0 0 +3.11646 0.005 0 0 0 0.005 0 0 +3.1189 0.005 0 0 0 0.005 0 0 +3.12134 0.005 0 0 0 0.005 0 0 +3.12378 0.005 0 0 0 0.005 0 0 +3.12622 0.005 0 0 0 0.005 0 0 +3.12866 0.005 0 0 0 0.005 0 0 +3.1311 0.005 0 0 0 0.005 0 0 +3.13354 0.005 0 0 0 0.005 0 0 +3.13599 0.005 0 0 0 0.005 0 0 +3.13843 0.005 0 0 0 0.005 0 0 +3.14087 0.005 0 0 0 0.005 0 0 +3.14331 0.005 0 0 0 0.005 0 0 +3.14575 0.005 0 0 0 0.005 0 0 +3.14819 0.005 0 0 0 0.005 0 0 +3.15063 0.005 0 0 0 0.005 0 0 +3.15308 0.005 0 0 0 0.005 0 0 +3.15552 0.005 0 0 0 0.005 0 0 +3.15796 0.005 0 0 0 0.005 0 0 +3.1604 0.005 0 0 0 0.005 0 0 +3.16284 0.005 0 0 0 0.005 0 0 +3.16528 0.005 0 0 0 0.005 0 0 +3.16772 0.005 0 0 0 0.005 0 0 +3.17017 0.005 0 0 0 0.005 0 0 +3.17261 0.005 0 0 0 0.005 0 0 +3.17505 0.005 0 0 0 0.005 0 0 +3.17749 0.005 0 0 0 0.005 0 0 +3.17993 0.005 0 0 0 0.005 0 0 +3.18237 0.005 0 0 0 0.005 0 0 +3.18481 0.005 0 0 0 0.005 0 0 +3.18726 0.005 0 0 0 0.005 0 0 +3.1897 0.005 0 0 0 0.005 0 0 +3.19214 0.005 0 0 0 0.005 0 0 +3.19458 0.005 0 0 0 0.005 0 0 +3.19702 0.005 0 0 0 0.005 0 0 +3.19946 0.005 0 0 0 0.005 0 0 +3.2019 0.005 0 0 0 0.005 0 0 +3.20435 0.005 0 0 0 0.005 0 0 +3.20679 0.005 0 0 0 0.005 0 0 +3.20923 0.005 0 0 0 0.005 0 0 +3.21167 0.005 0 0 0 0.005 0 0 +3.21411 0.005 0 0 0 0.005 0 0 +3.21655 0.005 0 0 0 0.005 0 0 +3.21899 0.005 0 0 0 0.005 0 0 +3.22144 0.005 0 0 0 0.005 0 0 +3.22388 0.005 0 0 0 0.005 0 0 +3.22632 0.005 0 0 0 0.005 0 0 +3.22876 0.005 0 0 0 0.005 0 0 +3.2312 0.005 0 0 0 0.005 0 0 +3.23364 0.005 0 0 0 0.005 0 0 +3.23608 0.005 0 0 0 0.005 0 0 +3.23853 0.005 0 0 0 0.005 0 0 +3.24097 0.005 0 0 0 0.005 0 0 +3.24341 0.005 0 0 0 0.005 0 0 +3.24585 0.005 0 0 0 0.005 0 0 +3.24829 0.005 0 0 0 0.005 0 0 +3.25073 0.005 0 0 0 0.005 0 0 +3.25317 0.005 0 0 0 0.005 0 0 +3.25562 0.005 0 0 0 0.005 0 0 +3.25806 0.005 0 0 0 0.005 0 0 +3.2605 0.005 0 0 0 0.005 0 0 +3.26294 0.005 0 0 0 0.005 0 0 +3.26538 0.005 0 0 0 0.005 0 0 +3.26782 0.005 0 0 0 0.005 0 0 +3.27026 0.005 0 0 0 0.005 0 0 +3.27271 0.005 0 0 0 0.005 0 0 +3.27515 0.005 0 0 0 0.005 0 0 +3.27759 0.005 0 0 0 0.005 0 0 +3.28003 0.005 0 0 0 0.005 0 0 +3.28247 0.005 0 0 0 0.005 0 0 +3.28491 0.005 0 0 0 0.005 0 0 +3.28735 0.005 0 0 0 0.005 0 0 +3.28979 0.005 0 0 0 0.005 0 0 +3.29224 0.005 0 0 0 0.005 0 0 +3.29468 0.005 0 0 0 0.005 0 0 +3.29712 0.005 0 0 0 0.005 0 0 +3.29956 0.005 0 0 0 0.005 0 0 +3.302 0.005 0 0 0 0.005 0 0 +3.30444 0.005 0 0 0 0.005 0 0 +3.30688 0.005 0 0 0 0.005 0 0 +3.30933 0.005 0 0 0 0.005 0 0 +3.31177 0.005 0 0 0 0.005 0 0 +3.31421 0.005 0 0 0 0.005 0 0 +3.31665 0.005 0 0 0 0.005 0 0 +3.31909 0.005 0 0 0 0.005 0 0 +3.32153 0.005 0 0 0 0.005 0 0 +3.32397 0.005 0 0 0 0.005 0 0 +3.32642 0.005 0 0 0 0.005 0 0 +3.32886 0.005 0 0 0 0.005 0 0 +3.3313 0.005 0 0 0 0.005 0 0 +3.33374 0.005 0 0 0 0.005 0 0 +3.33618 0.005 0 0 0 0.005 0 0 +3.33862 0.005 0 0 0 0.005 0 0 +3.34106 0.005 0 0 0 0.005 0 0 +3.34351 0.005 0 0 0 0.005 0 0 +3.34595 0.005 0 0 0 0.005 0 0 +3.34839 0.005 0 0 0 0.005 0 0 +3.35083 0.005 0 0 0 0.005 0 0 +3.35327 0.005 0 0 0 0.005 0 0 +3.35571 0.005 0 0 0 0.005 0 0 +3.35815 0.005 0 0 0 0.005 0 0 +3.3606 0.005 0 0 0 0.005 0 0 +3.36304 0.005 0 0 0 0.005 0 0 +3.36548 0.005 0 0 0 0.005 0 0 +3.36792 0.005 0 0 0 0.005 0 0 +3.37036 0.005 0 0 0 0.005 0 0 +3.3728 0.005 0 0 0 0.005 0 0 +3.37524 0.005 0 0 0 0.005 0 0 +3.37769 0.005 0 0 0 0.005 0 0 +3.38013 0.005 0 0 0 0.005 0 0 +3.38257 0.005 0 0 0 0.005 0 0 +3.38501 0.005 0 0 0 0.005 0 0 +3.38745 0.005 0 0 0 0.005 0 0 +3.38989 0.005 0 0 0 0.005 0 0 +3.39233 0.005 0 0 0 0.005 0 0 +3.39478 0.005 0 0 0 0.005 0 0 +3.39722 0.005 0 0 0 0.005 0 0 +3.39966 0.005 0 0 0 0.005 0 0 +3.4021 0.005 0 0 0 0.005 0 0 +3.40454 0.005 0 0 0 0.005 0 0 +3.40698 0.005 0 0 0 0.005 0 0 +3.40942 0.005 0 0 0 0.005 0 0 +3.41187 0.005 0 0 0 0.005 0 0 +3.41431 0.005 0 0 0 0.005 0 0 +3.41675 0.005 0 0 0 0.005 0 0 +3.41919 0.005 0 0 0 0.005 0 0 +3.42163 0.005 0 0 0 0.005 0 0 +3.42407 0.005 0 0 0 0.005 0 0 +3.42651 0.005 0 0 0 0.005 0 0 +3.42896 0.005 0 0 0 0.005 0 0 +3.4314 0.005 0 0 0 0.005 0 0 +3.43384 0.005 0 0 0 0.005 0 0 +3.43628 0.005 0 0 0 0.005 0 0 +3.43872 0.005 0 0 0 0.005 0 0 +3.44116 0.005 0 0 0 0.005 0 0 +3.4436 0.005 0 0 0 0.005 0 0 +3.44604 0.005 0 0 0 0.005 0 0 +3.44849 0.005 0 0 0 0.005 0 0 +3.45093 0.005 0 0 0 0.005 0 0 +3.45337 0.005 0 0 0 0.005 0 0 +3.45581 0.005 0 0 0 0.005 0 0 +3.45825 0.005 0 0 0 0.005 0 0 +3.46069 0.005 0 0 0 0.005 0 0 +3.46313 0.005 0 0 0 0.005 0 0 +3.46558 0.005 0 0 0 0.005 0 0 +3.46802 0.005 0 0 0 0.005 0 0 +3.47046 0.005 0 0 0 0.005 0 0 +3.4729 0.005 0 0 0 0.005 0 0 +3.47534 0.005 0 0 0 0.005 0 0 +3.47778 0.005 0 0 0 0.005 0 0 +3.48022 0.005 0 0 0 0.005 0 0 +3.48267 0.005 0 0 0 0.005 0 0 +3.48511 0.005 0 0 0 0.005 0 0 +3.48755 0.005 0 0 0 0.005 0 0 +3.48999 0.005 0 0 0 0.005 0 0 +3.49243 0.005 0 0 0 0.005 0 0 +3.49487 0.005 0 0 0 0.005 0 0 +3.49731 0.005 0 0 0 0.005 0 0 +3.49976 0.005 0 0 0 0.005 0 0 +3.5022 0.005 0 0 0 0.005 0 0 +3.50464 0.005 0 0 0 0.005 0 0 +3.50708 0.005 0 0 0 0.005 0 0 +3.50952 0.005 0 0 0 0.005 0 0 +3.51196 0.005 0 0 0 0.005 0 0 +3.5144 0.005 0 0 0 0.005 0 0 +3.51685 0.005 0 0 0 0.005 0 0 +3.51929 0.005 0 0 0 0.005 0 0 +3.52173 0.005 0 0 0 0.005 0 0 +3.52417 0.005 0 0 0 0.005 0 0 +3.52661 0.005 0 0 0 0.005 0 0 +3.52905 0.005 0 0 0 0.005 0 0 +3.53149 0.005 0 0 0 0.005 0 0 +3.53394 0.005 0 0 0 0.005 0 0 +3.53638 0.005 0 0 0 0.005 0 0 +3.53882 0.005 0 0 0 0.005 0 0 +3.54126 0.005 0 0 0 0.005 0 0 +3.5437 0.005 0 0 0 0.005 0 0 +3.54614 0.005 0 0 0 0.005 0 0 +3.54858 0.005 0 0 0 0.005 0 0 +3.55103 0.005 0 0 0 0.005 0 0 +3.55347 0.005 0 0 0 0.005 0 0 +3.55591 0.005 0 0 0 0.005 0 0 +3.55835 0.005 0 0 0 0.005 0 0 +3.56079 0.005 0 0 0 0.005 0 0 +3.56323 0.005 0 0 0 0.005 0 0 +3.56567 0.005 0 0 0 0.005 0 0 +3.56812 0.005 0 0 0 0.005 0 0 +3.57056 0.005 0 0 0 0.005 0 0 +3.573 0.005 0 0 0 0.005 0 0 +3.57544 0.005 0 0 0 0.005 0 0 +3.57788 0.005 0 0 0 0.005 0 0 +3.58032 0.005 0 0 0 0.005 0 0 +3.58276 0.005 0 0 0 0.005 0 0 +3.58521 0.005 0 0 0 0.005 0 0 +3.58765 0.005 0 0 0 0.005 0 0 +3.59009 0.005 0 0 0 0.005 0 0 +3.59253 0.005 0 0 0 0.005 0 0 +3.59497 0.005 0 0 0 0.005 0 0 +3.59741 0.005 0 0 0 0.005 0 0 +3.59985 0.005 0 0 0 0.005 0 0 +3.60229 0.005 0 0 0 0.005 0 0 +3.60474 0.005 0 0 0 0.005 0 0 +3.60718 0.005 0 0 0 0.005 0 0 +3.60962 0.005 0 0 0 0.005 0 0 +3.61206 0.005 0 0 0 0.005 0 0 +3.6145 0.005 0 0 0 0.005 0 0 +3.61694 0.005 0 0 0 0.005 0 0 +3.61938 0.005 0 0 0 0.005 0 0 +3.62183 0.005 0 0 0 0.005 0 0 +3.62427 0.005 0 0 0 0.005 0 0 +3.62671 0.005 0 0 0 0.005 0 0 +3.62915 0.005 0 0 0 0.005 0 0 +3.63159 0.005 0 0 0 0.005 0 0 +3.63403 0.005 0 0 0 0.005 0 0 +3.63647 0.005 0 0 0 0.005 0 0 +3.63892 0.005 0 0 0 0.005 0 0 +3.64136 0.005 0 0 0 0.005 0 0 +3.6438 0.005 0 0 0 0.005 0 0 +3.64624 0.005 0 0 0 0.005 0 0 +3.64868 0.005 0 0 0 0.005 0 0 +3.65112 0.005 0 0 0 0.005 0 0 +3.65356 0.005 0 0 0 0.005 0 0 +3.65601 0.005 0 0 0 0.005 0 0 +3.65845 0.005 0 0 0 0.005 0 0 +3.66089 0.005 0 0 0 0.005 0 0 +3.66333 0.005 0 0 0 0.005 0 0 +3.66577 0.005 0 0 0 0.005 0 0 +3.66821 0.005 0 0 0 0.005 0 0 +3.67065 0.005 0 0 0 0.005 0 0 +3.6731 0.00499516 0.00021442 0 1.07106e-006 0.00499516 0.000968625 4.89012e-005 +3.67554 0.00498904 0.000485687 0 2.42311e-006 0.00498904 0.0021954 8.42738e-005 +3.67798 0.00498293 0.000756955 0 3.77185e-006 0.00498293 0.00342368 0.000113191 +3.68042 0.00497681 0.00102822 0 5.11727e-006 0.00497681 0.00465347 0.000138719 +3.68286 0.00497071 0.00129949 0 6.45938e-006 0.00497071 0.00588477 0.000162021 +3.6853 0.0049646 0.00157076 0 7.79818e-006 0.0049646 0.00711758 0.000183699 +3.68774 0.0049585 0.00184202 0 9.13368e-006 0.0049585 0.00835191 0.000204115 +3.69019 0.0049524 0.00211329 0 1.04659e-005 0.0049524 0.00958775 0.000223508 +3.69263 0.00494631 0.00238456 0 1.17948e-005 0.00494631 0.0108251 0.000242049 +3.69507 0.00494022 0.00265583 0 1.31204e-005 0.00494022 0.012064 0.000259861 +3.69751 0.00493414 0.00292709 0 1.44427e-005 0.00493414 0.0133044 0.00027704 +3.69995 0.00492805 0.00319836 0 1.57617e-005 0.00492805 0.0145464 0.000293661 +3.70239 0.00492198 0.00346963 0 1.70774e-005 0.00492198 0.0157899 0.000309784 +3.70483 0.0049159 0.0037409 0 1.83899e-005 0.0049159 0.0170349 0.000325459 +3.70728 0.00490983 0.00401216 0 1.9699e-005 0.00490983 0.0182815 0.000340727 +3.70972 0.00490376 0.00428343 0 2.10049e-005 0.00490376 0.0195296 0.000355624 +3.71216 0.0048977 0.0045547 0 2.23075e-005 0.0048977 0.0207792 0.000370179 +3.7146 0.00489164 0.00482596 0 2.36069e-005 0.00489164 0.0220304 0.000384417 +3.71704 0.00488559 0.00509723 0 2.4903e-005 0.00488559 0.0232831 0.000398362 +3.71948 0.00487953 0.0053685 0 2.61958e-005 0.00487953 0.0245374 0.000412032 +3.72192 0.00487349 0.00563977 0 2.74853e-005 0.00487349 0.0257933 0.000425446 +3.72437 0.00486744 0.00591103 0 2.87716e-005 0.00486744 0.0270507 0.000438618 +3.72681 0.0048614 0.0061823 0 3.00546e-005 0.0048614 0.0283097 0.000451563 +3.72925 0.00485536 0.00645357 0 3.13344e-005 0.00485536 0.0295702 0.000464292 +3.73169 0.00484933 0.00672484 0 3.2611e-005 0.00484933 0.0308323 0.000476818 +3.73413 0.0048433 0.0069961 0 3.38842e-005 0.0048433 0.032096 0.00048915 +3.73657 0.00483728 0.00726737 0 3.51543e-005 0.00483728 0.0333613 0.000501298 +3.73901 0.00483125 0.00753864 0 3.64211e-005 0.00483125 0.0346281 0.00051327 +3.74146 0.00482524 0.00780991 0 3.76846e-005 0.00482524 0.0358965 0.000525073 +3.7439 0.00481922 0.00808117 0 3.8945e-005 0.00481922 0.0371665 0.000536716 +3.74634 0.00481321 0.00835244 0 4.02021e-005 0.00481321 0.0384381 0.000548204 +3.74878 0.0048072 0.00862371 0 4.14559e-005 0.0048072 0.0397112 0.000559545 +3.75122 0.0048012 0.00889498 0 4.27066e-005 0.0048012 0.040986 0.000570742 +3.75366 0.0047952 0.00916624 0 4.3954e-005 0.0047952 0.0422623 0.000581803 +3.7561 0.00478921 0.00943751 0 4.51982e-005 0.00478921 0.0435403 0.000592731 +3.75854 0.00478322 0.00970878 0 4.64392e-005 0.00478322 0.0448198 0.000603532 +3.76099 0.00477723 0.00998004 0 4.76769e-005 0.00477723 0.046101 0.000614209 +3.76343 0.00477124 0.0102513 0 4.89115e-005 0.00477124 0.0473837 0.000624766 +3.76587 0.00476526 0.0105226 0 5.01428e-005 0.00476526 0.0486681 0.000635209 +3.76831 0.00475929 0.0107938 0 5.1371e-005 0.00475929 0.0499541 0.000645539 +3.77075 0.00475331 0.0110651 0 5.25959e-005 0.00475331 0.0512417 0.00065576 +3.77319 0.00474734 0.0113364 0 5.38177e-005 0.00474734 0.0525309 0.000665877 +3.77563 0.00474138 0.0116076 0 5.50362e-005 0.00474138 0.0538217 0.00067589 +3.77808 0.00473542 0.0118789 0 5.62516e-005 0.00473542 0.0551142 0.000685804 +3.78052 0.00472946 0.0121502 0 5.74638e-005 0.00472946 0.0564083 0.000695622 +3.78296 0.0047235 0.0124215 0 5.86728e-005 0.0047235 0.057704 0.000705345 +3.7854 0.00471755 0.0126927 0 5.98786e-005 0.00471755 0.0590013 0.000714976 +3.78784 0.00471161 0.012964 0 6.10812e-005 0.00471161 0.0603003 0.000724517 +3.79028 0.00470566 0.0132353 0 6.22806e-005 0.00470566 0.0616009 0.000733971 +3.79272 0.00469972 0.0135065 0 6.34769e-005 0.00469972 0.0629032 0.00074334 +3.79517 0.00469379 0.0137778 0 6.467e-005 0.00469379 0.0642071 0.000752626 +3.79761 0.00468786 0.0140491 0 6.58599e-005 0.00468786 0.0655127 0.00076183 +3.80005 0.00468193 0.0143203 0 6.70467e-005 0.00468193 0.0668199 0.000770954 +3.80249 0.004676 0.0145916 0 6.82303e-005 0.004676 0.0681288 0.000780001 +3.80493 0.00467008 0.0148629 0 6.94108e-005 0.00467008 0.0694393 0.000788972 +3.80737 0.00466417 0.0151341 0 7.05881e-005 0.00466417 0.0707515 0.000797868 +3.80981 0.00465825 0.0154054 0 7.17622e-005 0.00465825 0.0720654 0.000806692 +3.81226 0.00465234 0.0156767 0 7.29332e-005 0.00465234 0.0733809 0.000815444 +3.8147 0.00464644 0.0159479 0 7.41011e-005 0.00464644 0.0746981 0.000824125 +3.81714 0.00464054 0.0162192 0 7.52658e-005 0.00464054 0.076017 0.000832738 +3.81958 0.00463464 0.0164905 0 7.64273e-005 0.00463464 0.0773375 0.000841284 +3.82202 0.00462874 0.0167617 0 7.75858e-005 0.00462874 0.0786598 0.000849764 +3.82446 0.00462285 0.017033 0 7.8741e-005 0.00462285 0.0799837 0.000858179 +3.8269 0.00461697 0.0173043 0 7.98932e-005 0.00461697 0.0813093 0.00086653 +3.82935 0.00461108 0.0175755 0 8.10422e-005 0.00461108 0.0826366 0.000874819 +3.83179 0.0046052 0.0178468 0 8.21882e-005 0.0046052 0.0839656 0.000883046 +3.83423 0.00459933 0.0181181 0 8.33309e-005 0.00459933 0.0852963 0.000891212 +3.83667 0.00459346 0.0183893 0 8.44706e-005 0.00459346 0.0866287 0.00089932 +3.83911 0.00458759 0.0186606 0 8.56072e-005 0.00458759 0.0879628 0.000907369 +3.84155 0.00458172 0.0189319 0 8.67406e-005 0.00458172 0.0892986 0.00091536 +3.84399 0.00457586 0.0192031 0 8.78709e-005 0.00457586 0.0906361 0.000923295 +3.84644 0.00457001 0.0194744 0 8.89982e-005 0.00457001 0.0919753 0.000931174 +3.84888 0.00456415 0.0197457 0 9.01223e-005 0.00456415 0.0933162 0.000938999 +3.85132 0.0045583 0.0200169 0 9.12433e-005 0.0045583 0.0946589 0.00094677 +3.85376 0.00455246 0.0202882 0 9.23612e-005 0.00455246 0.0960033 0.000954487 +3.8562 0.00454662 0.0205595 0 9.34761e-005 0.00454662 0.0973494 0.000962153 +3.85864 0.00454078 0.0208307 0 9.45878e-005 0.00454078 0.0986972 0.000969766 +3.86108 0.00453495 0.021102 0 9.56964e-005 0.00453495 0.100047 0.000977329 +3.86353 0.00452911 0.0213733 0 9.6802e-005 0.00452911 0.101398 0.000984842 +3.86597 0.00452329 0.0216445 0 9.79045e-005 0.00452329 0.102751 0.000992306 +3.86841 0.00451746 0.0219158 0 9.90039e-005 0.00451746 0.104106 0.00099972 +3.87085 0.00451165 0.0221871 0 0.0001001 0.00451165 0.105463 0.00100709 +3.87329 0.00450583 0.0224583 0 0.000101193 0.00450583 0.106821 0.00101441 +3.87573 0.00450002 0.0227296 0 0.000102284 0.00450002 0.108181 0.00102168 +3.87817 0.00449421 0.0230009 0 0.000103371 0.00449421 0.109543 0.00102891 +3.88062 0.00448841 0.0232721 0 0.000104455 0.00448841 0.110906 0.00103609 +3.88306 0.00448261 0.0235434 0 0.000105536 0.00448261 0.112272 0.00104322 +3.8855 0.00447681 0.0238147 0 0.000106614 0.00447681 0.113639 0.00105031 +3.88794 0.00447102 0.0240859 0 0.000107689 0.00447102 0.115007 0.00105736 +3.89038 0.00446523 0.0243572 0 0.00010876 0.00446523 0.116378 0.00106437 +3.89282 0.00445944 0.0246285 0 0.000109829 0.00445944 0.117751 0.00107133 +3.89526 0.00445366 0.0248997 0 0.000110895 0.00445366 0.119125 0.00107825 +3.89771 0.00444788 0.025171 0 0.000111958 0.00444788 0.120501 0.00108512 +3.90015 0.00444211 0.0254423 0 0.000113017 0.00444211 0.121879 0.00109196 +3.90259 0.00443634 0.0257136 0 0.000114074 0.00443634 0.123258 0.00109876 +3.90503 0.00443057 0.0259848 0 0.000115128 0.00443057 0.124639 0.00110551 +3.90747 0.00442481 0.0262561 0 0.000116178 0.00442481 0.126023 0.00111223 +3.90991 0.00441905 0.0265274 0 0.000117226 0.00441905 0.127408 0.0011189 +3.91235 0.00441329 0.0267986 0 0.00011827 0.00441329 0.128794 0.00112554 +3.91479 0.00440754 0.0270699 0 0.000119312 0.00440754 0.130183 0.00113214 +3.91724 0.00440179 0.0273412 0 0.00012035 0.00440179 0.131573 0.0011387 +3.91968 0.00439605 0.0276124 0 0.000121386 0.00439605 0.132965 0.00114522 +3.92212 0.00439031 0.0278837 0 0.000122418 0.00439031 0.134359 0.0011517 +3.92456 0.00438457 0.028155 0 0.000123447 0.00438457 0.135755 0.00115815 +3.927 0.00437884 0.0284262 0 0.000124474 0.00437884 0.137153 0.00116456 +3.92944 0.00437311 0.0286975 0 0.000125497 0.00437311 0.138553 0.00117094 +3.93188 0.00436738 0.0289688 0 0.000126518 0.00436738 0.139954 0.00117727 +3.93433 0.00436166 0.02924 0 0.000127535 0.00436166 0.141357 0.00118358 +3.93677 0.00435594 0.0295113 0 0.000128549 0.00435594 0.142762 0.00118985 +3.93921 0.00435023 0.0297826 0 0.000129561 0.00435023 0.144169 0.00119608 +3.94165 0.00434452 0.0300538 0 0.000130569 0.00434452 0.145578 0.00120228 +3.94409 0.00433881 0.0303251 0 0.000131575 0.00433881 0.146988 0.00120844 +3.94653 0.00433311 0.0305964 0 0.000132577 0.00433311 0.148401 0.00121457 +3.94897 0.00432741 0.0308676 0 0.000133577 0.00432741 0.149815 0.00122067 +3.95142 0.00432171 0.0311389 0 0.000134573 0.00432171 0.151231 0.00122673 +3.95386 0.00431602 0.0314102 0 0.000135567 0.00431602 0.152649 0.00123276 +3.9563 0.00431033 0.0316814 0 0.000136558 0.00431033 0.154069 0.00123876 +3.95874 0.00430465 0.0319527 0 0.000137545 0.00430465 0.155491 0.00124473 +3.96118 0.00429897 0.032224 0 0.00013853 0.00429897 0.156914 0.00125066 +3.96362 0.00429329 0.0324952 0 0.000139512 0.00429329 0.15834 0.00125656 +3.96606 0.00428762 0.0327665 0 0.00014049 0.00428762 0.159767 0.00126243 +3.96851 0.00428195 0.0330378 0 0.000141466 0.00428195 0.161196 0.00126827 +3.97095 0.00427628 0.033309 0 0.000142439 0.00427628 0.162628 0.00127408 +3.97339 0.00427062 0.0335803 0 0.000143409 0.00427062 0.164061 0.00127986 +3.97583 0.00426496 0.0338516 0 0.000144376 0.00426496 0.165496 0.0012856 +3.97827 0.00425931 0.0341228 0 0.00014534 0.00425931 0.166933 0.00129132 +3.98071 0.00425366 0.0343941 0 0.000146301 0.00425366 0.168371 0.001297 +3.98315 0.00424801 0.0346654 0 0.000147259 0.00424801 0.169812 0.00130266 +3.9856 0.00424237 0.0349366 0 0.000148214 0.00424237 0.171255 0.00130829 +3.98804 0.00423673 0.0352079 0 0.000149166 0.00423673 0.172699 0.00131389 +3.99048 0.00423109 0.0354792 0 0.000150116 0.00423109 0.174146 0.00131945 +3.99292 0.00422546 0.0357504 0 0.000151062 0.00422546 0.175594 0.00132499 +3.99536 0.00421983 0.0360217 0 0.000152006 0.00421983 0.177044 0.00133051 +3.9978 0.00421421 0.036293 0 0.000152946 0.00421421 0.178497 0.00133599 +4.00024 0.00420859 0.0365642 0 0.000153884 0.00420859 0.179951 0.00134144 +4.00269 0.00420297 0.0368355 0 0.000154819 0.00420297 0.181407 0.00134687 +4.00513 0.00419736 0.0371068 0 0.000155751 0.00419736 0.182865 0.00135227 +4.00757 0.00419175 0.037378 0 0.000156679 0.00419175 0.184325 0.00135764 +4.01001 0.00418615 0.0376493 0 0.000157606 0.00418615 0.185787 0.00136299 +4.01245 0.00418054 0.0379206 0 0.000158529 0.00418054 0.187251 0.0013683 +4.01489 0.00417495 0.0381919 0 0.000159449 0.00417495 0.188717 0.00137359 +4.01733 0.00416935 0.0384631 0 0.000160366 0.00416935 0.190185 0.00137886 +4.01978 0.00416376 0.0387344 0 0.000161281 0.00416376 0.191655 0.00138409 +4.02222 0.00415817 0.0390057 0 0.000162192 0.00415817 0.193126 0.0013893 +4.02466 0.00415259 0.0392769 0 0.000163101 0.00415259 0.1946 0.00139449 +4.0271 0.00414701 0.0395482 0 0.000164007 0.00414701 0.196076 0.00139965 +4.02954 0.00414144 0.0398195 0 0.00016491 0.00414144 0.197554 0.00140478 +4.03198 0.00413586 0.0400907 0 0.00016581 0.00413586 0.199034 0.00140989 +4.03442 0.0041303 0.040362 0 0.000166707 0.0041303 0.200515 0.00141497 +4.03687 0.00412473 0.0406333 0 0.000167601 0.00412473 0.201999 0.00142002 +4.03931 0.00411917 0.0409045 0 0.000168493 0.00411917 0.203485 0.00142505 +4.04175 0.00411361 0.0411758 0 0.000169381 0.00411361 0.204973 0.00143006 +4.04419 0.00410806 0.0414471 0 0.000170267 0.00410806 0.206462 0.00143504 +4.04663 0.00410251 0.0417183 0 0.00017115 0.00410251 0.207954 0.00144 +4.04907 0.00409697 0.0419896 0 0.00017203 0.00409697 0.209448 0.00144493 +4.05151 0.00409143 0.0422609 0 0.000172907 0.00409143 0.210944 0.00144984 +4.05396 0.00408589 0.0425321 0 0.000173781 0.00408589 0.212442 0.00145472 +4.0564 0.00408035 0.0428034 0 0.000174653 0.00408035 0.213941 0.00145958 +4.05884 0.00407482 0.0430747 0 0.000175522 0.00407482 0.215443 0.00146441 +4.06128 0.0040693 0.0433459 0 0.000176387 0.0040693 0.216947 0.00146923 +4.06372 0.00406377 0.0436172 0 0.00017725 0.00406377 0.218453 0.00147401 +4.06616 0.00405825 0.0438885 0 0.000178111 0.00405825 0.219961 0.00147878 +4.0686 0.00405274 0.0441597 0 0.000178968 0.00405274 0.221471 0.00148352 +4.07104 0.00404723 0.044431 0 0.000179822 0.00404723 0.222984 0.00148824 +4.07349 0.00404172 0.0447023 0 0.000180674 0.00404172 0.224498 0.00149293 +4.07593 0.00403621 0.0449735 0 0.000181523 0.00403621 0.226014 0.00149761 +4.07837 0.00403071 0.0452448 0 0.000182369 0.00403071 0.227532 0.00150226 +4.08081 0.00402522 0.0455161 0 0.000183212 0.00402522 0.229053 0.00150688 +4.08325 0.00401972 0.0457873 0 0.000184052 0.00401972 0.230575 0.00151149 +4.08569 0.00401423 0.0460586 0 0.00018489 0.00401423 0.2321 0.00151607 +4.08813 0.00400875 0.0463299 0 0.000185725 0.00400875 0.233627 0.00152063 +4.09058 0.00400327 0.0466011 0 0.000186557 0.00400327 0.235155 0.00152517 +4.09302 0.00399779 0.0468724 0 0.000187386 0.00399779 0.236686 0.00152968 +4.09546 0.00399232 0.0471437 0 0.000188212 0.00399232 0.238219 0.00153418 +4.0979 0.00398684 0.0474149 0 0.000189036 0.00398684 0.239754 0.00153865 +4.10034 0.00398138 0.0476862 0 0.000189857 0.00398138 0.241291 0.0015431 +4.10278 0.00397591 0.0479575 0 0.000190675 0.00397591 0.242831 0.00154753 +4.10522 0.00397046 0.0482287 0 0.00019149 0.00397046 0.244372 0.00155194 +4.10767 0.003965 0.0485 0 0.000192303 0.003965 0.245915 0.00155633 +4.11011 0.00395955 0.0487713 0 0.000193112 0.00395955 0.247461 0.00156069 +4.11255 0.0039541 0.0490425 0 0.000193919 0.0039541 0.249009 0.00156504 +4.11499 0.00394866 0.0493138 0 0.000194723 0.00394866 0.250559 0.00156936 +4.11743 0.00394322 0.0495851 0 0.000195525 0.00394322 0.252111 0.00157366 +4.11987 0.00393778 0.0498563 0 0.000196323 0.00393778 0.253665 0.00157794 +4.12231 0.00393235 0.0501276 0 0.000197119 0.00393235 0.255221 0.00158221 +4.12476 0.00392692 0.0503989 0 0.000197912 0.00392692 0.25678 0.00158645 +4.1272 0.00392149 0.0506701 0 0.000198703 0.00392149 0.25834 0.00159067 +4.12964 0.00391607 0.0509414 0 0.00019949 0.00391607 0.259903 0.00159487 +4.13208 0.00391065 0.0512127 0 0.000200275 0.00391065 0.261468 0.00159905 +4.13452 0.00390524 0.051484 0 0.000201057 0.00390524 0.263035 0.00160321 +4.13696 0.00389983 0.0517552 0 0.000201836 0.00389983 0.264604 0.00160735 +4.1394 0.00389442 0.0520265 0 0.000202613 0.00389442 0.266176 0.00161147 +4.14185 0.00388902 0.0522978 0 0.000203387 0.00388902 0.267749 0.00161557 +4.14429 0.00388362 0.052569 0 0.000204158 0.00388362 0.269325 0.00161965 +4.14673 0.00387822 0.0528403 0 0.000204926 0.00387822 0.270903 0.00162371 +4.14917 0.00387283 0.0531116 0 0.000205692 0.00387283 0.272484 0.00162775 +4.15161 0.00386744 0.0533828 0 0.000206455 0.00386744 0.274066 0.00163178 +4.15405 0.00386206 0.0536541 0 0.000207215 0.00386206 0.275651 0.00163578 +4.15649 0.00385668 0.0539254 0 0.000207973 0.00385668 0.277237 0.00163976 +4.15894 0.0038513 0.0541966 0 0.000208728 0.0038513 0.278826 0.00164373 +4.16138 0.00384593 0.0544679 0 0.00020948 0.00384593 0.280418 0.00164767 +4.16382 0.00384056 0.0547392 0 0.000210229 0.00384056 0.282011 0.0016516 +4.16626 0.00383519 0.0550104 0 0.000210976 0.00383519 0.283607 0.00165551 +4.1687 0.00382983 0.0552817 0 0.00021172 0.00382983 0.285205 0.0016594 +4.17114 0.00382447 0.055553 0 0.000212461 0.00382447 0.286805 0.00166327 +4.17358 0.00381912 0.0558242 0 0.000213199 0.00381912 0.288408 0.00166712 +4.17603 0.00381377 0.0560955 0 0.000213935 0.00381377 0.290012 0.00167096 +4.17847 0.00380842 0.0563668 0 0.000214668 0.00380842 0.291619 0.00167477 +4.18091 0.00380308 0.056638 0 0.000215399 0.00380308 0.293228 0.00167857 +4.18335 0.00379774 0.0569093 0 0.000216127 0.00379774 0.29484 0.00168235 +4.18579 0.0037924 0.0571806 0 0.000216852 0.0037924 0.296454 0.00168611 +4.18823 0.00378707 0.0574518 0 0.000217574 0.00378707 0.29807 0.00168985 +4.19067 0.00378175 0.0577231 0 0.000218294 0.00378175 0.299688 0.00169358 +4.19312 0.00377642 0.0579944 0 0.000219011 0.00377642 0.301308 0.00169728 +4.19556 0.0037711 0.0582656 0 0.000219726 0.0037711 0.302931 0.00170097 +4.198 0.00376578 0.0585369 0 0.000220437 0.00376578 0.304556 0.00170464 +4.20044 0.00376047 0.0588082 0 0.000221146 0.00376047 0.306184 0.0017083 +4.20288 0.00375516 0.0590794 0 0.000221853 0.00375516 0.307813 0.00171193 +4.20532 0.00374986 0.0593507 0 0.000222557 0.00374986 0.309445 0.00171555 +4.20776 0.00374455 0.059622 0 0.000223258 0.00374455 0.31108 0.00171915 +4.21021 0.00373926 0.0598932 0 0.000223956 0.00373926 0.312716 0.00172274 +4.21265 0.00373396 0.0601645 0 0.000224652 0.00373396 0.314355 0.0017263 +4.21509 0.00372867 0.0604358 0 0.000225345 0.00372867 0.315997 0.00172985 +4.21753 0.00372338 0.060707 0 0.000226036 0.00372338 0.31764 0.00173338 +4.21997 0.0037181 0.0609783 0 0.000226724 0.0037181 0.319286 0.0017369 +4.22241 0.00371282 0.0612496 0 0.000227409 0.00371282 0.320935 0.0017404 +4.22485 0.00370755 0.0615208 0 0.000228091 0.00370755 0.322585 0.00174388 +4.22729 0.00370228 0.0617921 0 0.000228771 0.00370228 0.324238 0.00174734 +4.22974 0.00369701 0.0620634 0 0.000229449 0.00369701 0.325894 0.00175079 +4.23218 0.00369174 0.0623346 0 0.000230124 0.00369174 0.327551 0.00175422 +4.23462 0.00368648 0.0626059 0 0.000230796 0.00368648 0.329211 0.00175764 +4.23706 0.00368123 0.0628772 0 0.000231465 0.00368123 0.330874 0.00176103 +4.2395 0.00367597 0.0631484 0 0.000232132 0.00367597 0.332539 0.00176441 +4.24194 0.00367072 0.0634197 0 0.000232796 0.00367072 0.334206 0.00176778 +4.24438 0.00366548 0.063691 0 0.000233458 0.00366548 0.335875 0.00177113 +4.24683 0.00366024 0.0639622 0 0.000234117 0.00366024 0.337547 0.00177446 +4.24927 0.003655 0.0642335 0 0.000234773 0.003655 0.339222 0.00177777 +4.25171 0.00364976 0.0645048 0 0.000235427 0.00364976 0.340898 0.00178107 +4.25415 0.00364453 0.0647761 0 0.000236079 0.00364453 0.342578 0.00178436 +4.25659 0.00363931 0.0650473 0 0.000236727 0.00363931 0.344259 0.00178762 +4.25903 0.00363408 0.0653186 0 0.000237373 0.00363408 0.345943 0.00179087 +4.26147 0.00362887 0.0655899 0 0.000238017 0.00362887 0.34763 0.00179411 +4.26392 0.00362365 0.0658611 0 0.000238658 0.00362365 0.349318 0.00179733 +4.26636 0.00361844 0.0661324 0 0.000239296 0.00361844 0.35101 0.00180053 +4.2688 0.00361323 0.0664037 0 0.000239932 0.00361323 0.352703 0.00180372 +4.27124 0.00360803 0.0666749 0 0.000240565 0.00360803 0.3544 0.00180689 +4.27368 0.00360283 0.0669462 0 0.000241195 0.00360283 0.356098 0.00181005 +4.27612 0.00359763 0.0672175 0 0.000241823 0.00359763 0.357799 0.00181319 +4.27856 0.00359244 0.0674887 0 0.000242449 0.00359244 0.359503 0.00181631 +4.28101 0.00358725 0.06776 0 0.000243072 0.00358725 0.361209 0.00181942 +4.28345 0.00358206 0.0680313 0 0.000243692 0.00358206 0.362917 0.00182252 +4.28589 0.00357688 0.0683025 0 0.00024431 0.00357688 0.364628 0.00182559 +4.28833 0.0035717 0.0685738 0 0.000244925 0.0035717 0.366341 0.00182866 +4.29077 0.00356653 0.0688451 0 0.000245538 0.00356653 0.368057 0.00183171 +4.29321 0.00356136 0.0691163 0 0.000246148 0.00356136 0.369776 0.00183474 +4.29565 0.00355619 0.0693876 0 0.000246755 0.00355619 0.371497 0.00183776 +4.2981 0.00355103 0.0696589 0 0.000247361 0.00355103 0.37322 0.00184076 +4.30054 0.00354587 0.0699301 0 0.000247963 0.00354587 0.374946 0.00184375 +4.30298 0.00354071 0.0702014 0 0.000248563 0.00354071 0.376674 0.00184672 +4.30542 0.00353556 0.0704727 0 0.00024916 0.00353556 0.378405 0.00184968 +4.30786 0.00353041 0.0707439 0 0.000249755 0.00353041 0.380139 0.00185262 +4.3103 0.00352527 0.0710152 0 0.000250348 0.00352527 0.381875 0.00185555 +4.31274 0.00352013 0.0712865 0 0.000250938 0.00352013 0.383613 0.00185846 +4.31519 0.00351499 0.0715577 0 0.000251525 0.00351499 0.385354 0.00186136 +4.31763 0.00350986 0.071829 0 0.00025211 0.00350986 0.387098 0.00186425 +4.32007 0.00350473 0.0721003 0 0.000252692 0.00350473 0.388844 0.00186712 +4.32251 0.0034996 0.0723715 0 0.000253272 0.0034996 0.390592 0.00186997 +4.32495 0.00349448 0.0726428 0 0.000253849 0.00349448 0.392344 0.00187281 +4.32739 0.00348936 0.0729141 0 0.000254424 0.00348936 0.394098 0.00187564 +4.32983 0.00348425 0.0731853 0 0.000254996 0.00348425 0.395854 0.00187845 +4.33228 0.00347914 0.0734566 0 0.000255566 0.00347914 0.397613 0.00188125 +4.33472 0.00347403 0.0737279 0 0.000256133 0.00347403 0.399374 0.00188403 +4.33716 0.00346893 0.0739991 0 0.000256698 0.00346893 0.401139 0.0018868 +4.3396 0.00346383 0.0742704 0 0.00025726 0.00346383 0.402905 0.00188955 +4.34204 0.00345874 0.0745417 0 0.00025782 0.00345874 0.404675 0.00189229 +4.34448 0.00345364 0.0748129 0 0.000258377 0.00345364 0.406447 0.00189502 +4.34692 0.00344856 0.0750842 0 0.000258932 0.00344856 0.408221 0.00189773 +4.34937 0.00344347 0.0753555 0 0.000259484 0.00344347 0.409998 0.00190043 +4.35181 0.00343839 0.0756267 0 0.000260034 0.00343839 0.411778 0.00190311 +4.35425 0.00343331 0.075898 0 0.000260582 0.00343331 0.413561 0.00190578 +4.35669 0.00342824 0.0761693 0 0.000261127 0.00342824 0.415346 0.00190844 +4.35913 0.00342317 0.0764405 0 0.000261669 0.00342317 0.417133 0.00191108 +4.36157 0.00341811 0.0767118 0 0.000262209 0.00341811 0.418924 0.00191371 +4.36401 0.00341305 0.0769831 0 0.000262747 0.00341305 0.420717 0.00191632 +4.36646 0.00340799 0.0772544 0 0.000263282 0.00340799 0.422512 0.00191892 +4.3689 0.00340293 0.0775256 0 0.000263815 0.00340293 0.424311 0.00192151 +4.37134 0.00339788 0.0777969 0 0.000264345 0.00339788 0.426112 0.00192408 +4.37378 0.00339284 0.0780682 0 0.000264872 0.00339284 0.427915 0.00192664 +4.37622 0.00338779 0.0783394 0 0.000265398 0.00338779 0.429722 0.00192919 +4.37866 0.00338275 0.0786107 0 0.000265921 0.00338275 0.431531 0.00193172 +4.3811 0.00337772 0.078882 0 0.000266441 0.00337772 0.433343 0.00193424 +4.38354 0.00337269 0.0791532 0 0.000266959 0.00337269 0.435157 0.00193675 +4.38599 0.00336766 0.0794245 0 0.000267475 0.00336766 0.436974 0.00193924 +4.38843 0.00336263 0.0796958 0 0.000267988 0.00336263 0.438794 0.00194172 +4.39087 0.00335761 0.079967 0 0.000268498 0.00335761 0.440617 0.00194419 +4.39331 0.0033526 0.0802383 0 0.000269007 0.0033526 0.442442 0.00194664 +4.39575 0.00334758 0.0805096 0 0.000269513 0.00334758 0.44427 0.00194908 +4.39819 0.00334258 0.0807808 0 0.000270016 0.00334258 0.446101 0.00195151 +4.40063 0.00333757 0.0810521 0 0.000270517 0.00333757 0.447935 0.00195392 +4.40308 0.00333257 0.0813234 0 0.000271016 0.00333257 0.449771 0.00195632 +4.40552 0.00332757 0.0815946 0 0.000271512 0.00332757 0.45161 0.00195871 +4.40796 0.00332258 0.0818659 0 0.000272006 0.00332258 0.453452 0.00196108 +4.4104 0.00331759 0.0821372 0 0.000272497 0.00331759 0.455296 0.00196344 +4.41284 0.0033126 0.0824084 0 0.000272986 0.0033126 0.457144 0.00196579 +4.41528 0.00330762 0.0826797 0 0.000273473 0.00330762 0.458994 0.00196813 +4.41772 0.00330264 0.082951 0 0.000273957 0.00330264 0.460847 0.00197045 +4.42017 0.00329766 0.0832222 0 0.000274439 0.00329766 0.462702 0.00197276 +4.42261 0.00329269 0.0834935 0 0.000274918 0.00329269 0.464561 0.00197506 +4.42505 0.00328772 0.0837648 0 0.000275395 0.00328772 0.466422 0.00197734 +4.42749 0.00328276 0.084036 0 0.00027587 0.00328276 0.468286 0.00197961 +4.42993 0.0032778 0.0843073 0 0.000276342 0.0032778 0.470153 0.00198187 +4.43237 0.00327284 0.0845786 0 0.000276812 0.00327284 0.472023 0.00198412 +4.43481 0.00326789 0.0848498 0 0.00027728 0.00326789 0.473896 0.00198635 +4.43726 0.00326294 0.0851211 0 0.000277745 0.00326294 0.475771 0.00198857 +4.4397 0.00325799 0.0853924 0 0.000278208 0.00325799 0.47765 0.00199078 +4.44214 0.00325305 0.0856636 0 0.000278668 0.00325305 0.479531 0.00199298 +4.44458 0.00324811 0.0859349 0 0.000279126 0.00324811 0.481415 0.00199516 +4.44702 0.00324318 0.0862062 0 0.000279582 0.00324318 0.483302 0.00199733 +4.44946 0.00323825 0.0864774 0 0.000280036 0.00323825 0.485192 0.00199949 +4.4519 0.00323332 0.0867487 0 0.000280487 0.00323332 0.487084 0.00200164 +4.45435 0.0032284 0.08702 0 0.000280935 0.0032284 0.48898 0.00200377 +4.45679 0.00322348 0.0872912 0 0.000281382 0.00322348 0.490878 0.00200589 +4.45923 0.00321857 0.0875625 0 0.000281826 0.00321857 0.492779 0.002008 +4.46167 0.00321365 0.0878338 0 0.000282267 0.00321365 0.494684 0.0020101 +4.46411 0.00320875 0.088105 0 0.000282707 0.00320875 0.496591 0.00201218 +4.46655 0.00320384 0.0883763 0 0.000283144 0.00320384 0.498501 0.00201426 +4.46899 0.00319894 0.0886476 0 0.000283578 0.00319894 0.500414 0.00201632 +4.47144 0.00319404 0.0889188 0 0.000284011 0.00319404 0.50233 0.00201837 +4.47388 0.00318915 0.0891901 0 0.000284441 0.00318915 0.504249 0.0020204 +4.47632 0.00318426 0.0894614 0 0.000284869 0.00318426 0.50617 0.00202243 +4.47876 0.00317938 0.0897326 0 0.000285294 0.00317938 0.508095 0.00202444 +4.4812 0.0031745 0.0900039 0 0.000285717 0.0031745 0.510023 0.00202644 +4.48364 0.00316962 0.0902752 0 0.000286138 0.00316962 0.511953 0.00202843 +4.48608 0.00316474 0.0905465 0 0.000286556 0.00316474 0.513887 0.00203041 +4.48853 0.00315987 0.0908177 0 0.000286972 0.00315987 0.515824 0.00203237 +4.49097 0.00315501 0.091089 0 0.000287386 0.00315501 0.517763 0.00203433 +4.49341 0.00315014 0.0913603 0 0.000287798 0.00315014 0.519706 0.00203627 +4.49585 0.00314528 0.0916315 0 0.000288207 0.00314528 0.521651 0.0020382 +4.49829 0.00314043 0.0919028 0 0.000288614 0.00314043 0.5236 0.00204012 +4.50073 0.00313558 0.0921741 0 0.000289019 0.00313558 0.525552 0.00204203 +4.50317 0.00313073 0.0924453 0 0.000289421 0.00313073 0.527506 0.00204392 +4.50562 0.00312588 0.0927166 0 0.000289821 0.00312588 0.529464 0.0020458 +4.50806 0.00312104 0.0929879 0 0.000290219 0.00312104 0.531425 0.00204768 +4.5105 0.00311621 0.0932591 0 0.000290615 0.00311621 0.533388 0.00204954 +4.51294 0.00311137 0.0935304 0 0.000291008 0.00311137 0.535355 0.00205138 +4.51538 0.00310655 0.0938017 0 0.000291399 0.00310655 0.537325 0.00205322 +4.51782 0.00310172 0.0940729 0 0.000291788 0.00310172 0.539298 0.00205505 +4.52026 0.0030969 0.0943442 0 0.000292174 0.0030969 0.541274 0.00205686 +4.52271 0.00309208 0.0946155 0 0.000292559 0.00309208 0.543253 0.00205866 +4.52515 0.00308727 0.0948867 0 0.000292941 0.00308727 0.545235 0.00206046 +4.52759 0.00308246 0.095158 0 0.00029332 0.00308246 0.54722 0.00206224 +4.53003 0.00307765 0.0954293 0 0.000293698 0.00307765 0.549209 0.00206401 +4.53247 0.00307285 0.0957005 0 0.000294073 0.00307285 0.5512 0.00206576 +4.53491 0.00306805 0.0959718 0 0.000294446 0.00306805 0.553195 0.00206751 +4.53735 0.00306325 0.0962431 0 0.000294817 0.00306325 0.555192 0.00206924 +4.53979 0.00305846 0.0965143 0 0.000295185 0.00305846 0.557193 0.00207097 +4.54224 0.00305367 0.0967856 0 0.000295552 0.00305367 0.559197 0.00207268 +4.54468 0.00304889 0.0970569 0 0.000295916 0.00304889 0.561204 0.00207438 +4.54712 0.00304411 0.0973281 0 0.000296277 0.00304411 0.563215 0.00207607 +4.54956 0.00303933 0.0975994 0 0.000296637 0.00303933 0.565228 0.00207775 +4.552 0.00303456 0.0978707 0 0.000296994 0.00303456 0.567244 0.00207942 +4.55444 0.00302979 0.0981419 0 0.000297349 0.00302979 0.569264 0.00208108 +4.55688 0.00302502 0.0984132 0 0.000297702 0.00302502 0.571287 0.00208272 +4.55933 0.00302026 0.0986845 0 0.000298053 0.00302026 0.573313 0.00208436 +4.56177 0.0030155 0.0989557 0 0.000298401 0.0030155 0.575343 0.00208598 +4.56421 0.00301075 0.099227 0 0.000298748 0.00301075 0.577375 0.0020876 +4.56665 0.003006 0.0994983 0 0.000299092 0.003006 0.579411 0.0020892 +4.56909 0.00300125 0.0997695 0 0.000299434 0.00300125 0.58145 0.00209079 +4.57153 0.00299651 0.100041 0 0.000299773 0.00299651 0.583492 0.00209237 +4.57397 0.00299177 0.100312 0 0.000300111 0.00299177 0.585537 0.00209394 +4.57642 0.00298704 0.100583 0 0.000300446 0.00298704 0.587586 0.0020955 +4.57886 0.0029823 0.100855 0 0.000300779 0.0029823 0.589638 0.00209705 +4.5813 0.00297758 0.101126 0 0.00030111 0.00297758 0.591693 0.00209859 +4.58374 0.00297285 0.101397 0 0.000301439 0.00297285 0.593751 0.00210011 +4.58618 0.00296813 0.101668 0 0.000301765 0.00296813 0.595813 0.00210163 +4.58862 0.00296342 0.10194 0 0.00030209 0.00296342 0.597878 0.00210314 +4.59106 0.0029587 0.102211 0 0.000302412 0.0029587 0.599946 0.00210463 +4.59351 0.00295399 0.102482 0 0.000302732 0.00295399 0.602018 0.00210612 +4.59595 0.00294929 0.102753 0 0.00030305 0.00294929 0.604093 0.00210759 +4.59839 0.00294459 0.103025 0 0.000303365 0.00294459 0.606171 0.00210905 +4.60083 0.00293989 0.103296 0 0.000303679 0.00293989 0.608252 0.00211051 +4.60327 0.00293519 0.103567 0 0.00030399 0.00293519 0.610337 0.00211195 +4.60571 0.0029305 0.103839 0 0.000304299 0.0029305 0.612425 0.00211338 +4.60815 0.00292582 0.10411 0 0.000304606 0.00292582 0.614517 0.0021148 +4.6106 0.00292113 0.104381 0 0.000304911 0.00292113 0.616611 0.00211621 +4.61304 0.00291646 0.104652 0 0.000305214 0.00291646 0.61871 0.00211761 +4.61548 0.00291178 0.104924 0 0.000305514 0.00291178 0.620811 0.002119 +4.61792 0.00290711 0.105195 0 0.000305813 0.00290711 0.622916 0.00212038 +4.62036 0.00290244 0.105466 0 0.000306109 0.00290244 0.625024 0.00212175 +4.6228 0.00289778 0.105737 0 0.000306403 0.00289778 0.627136 0.00212311 +4.62524 0.00289312 0.106009 0 0.000306695 0.00289312 0.629251 0.00212446 +4.62769 0.00288846 0.10628 0 0.000306985 0.00288846 0.63137 0.0021258 +4.63013 0.00288381 0.106551 0 0.000307273 0.00288381 0.633492 0.00212713 +4.63257 0.00287916 0.106822 0 0.000307559 0.00287916 0.635617 0.00212844 +4.63501 0.00287451 0.107094 0 0.000307842 0.00287451 0.637746 0.00212975 +4.63745 0.00286987 0.107365 0 0.000308124 0.00286987 0.639878 0.00213105 +4.63989 0.00286523 0.107636 0 0.000308403 0.00286523 0.642014 0.00213234 +4.64233 0.0028606 0.107908 0 0.00030868 0.0028606 0.644153 0.00213362 +4.64478 0.00285597 0.108179 0 0.000308955 0.00285597 0.646295 0.00213488 +4.64722 0.00285134 0.10845 0 0.000309228 0.00285134 0.648442 0.00213614 +4.64966 0.00284672 0.108721 0 0.000309499 0.00284672 0.650591 0.00213739 +4.6521 0.0028421 0.108993 0 0.000309768 0.0028421 0.652744 0.00213862 +4.65454 0.00283748 0.109264 0 0.000310035 0.00283748 0.654901 0.00213985 +4.65698 0.00283287 0.109535 0 0.000310299 0.00283287 0.657061 0.00214107 +4.65942 0.00282826 0.109806 0 0.000310562 0.00282826 0.659224 0.00214228 +4.66187 0.00282366 0.110078 0 0.000310822 0.00282366 0.661392 0.00214347 +4.66431 0.00281906 0.110349 0 0.00031108 0.00281906 0.663562 0.00214466 +4.66675 0.00281446 0.11062 0 0.000311337 0.00281446 0.665736 0.00214584 +4.66919 0.00280987 0.110892 0 0.000311591 0.00280987 0.667914 0.00214701 +4.67163 0.00280528 0.111163 0 0.000311843 0.00280528 0.670095 0.00214816 +4.67407 0.0028007 0.111434 0 0.000312093 0.0028007 0.67228 0.00214931 +4.67651 0.00279611 0.111705 0 0.000312341 0.00279611 0.674469 0.00215045 +4.67896 0.00279154 0.111977 0 0.000312587 0.00279154 0.676661 0.00215158 +4.6814 0.00278696 0.112248 0 0.000312831 0.00278696 0.678856 0.0021527 +4.68384 0.00278239 0.112519 0 0.000313072 0.00278239 0.681056 0.00215381 +4.68628 0.00277783 0.11279 0 0.000313312 0.00277783 0.683259 0.00215491 +4.68872 0.00277326 0.113062 0 0.00031355 0.00277326 0.685465 0.002156 +4.69116 0.0027687 0.113333 0 0.000313785 0.0027687 0.687675 0.00215708 +4.6936 0.00276415 0.113604 0 0.000314019 0.00276415 0.689889 0.00215815 +4.69604 0.0027596 0.113875 0 0.00031425 0.0027596 0.692106 0.00215921 +4.69849 0.00275505 0.114147 0 0.00031448 0.00275505 0.694327 0.00216026 +4.70093 0.0027505 0.114418 0 0.000314707 0.0027505 0.696552 0.0021613 +4.70337 0.00274596 0.114689 0 0.000314933 0.00274596 0.69878 0.00216233 +4.70581 0.00274143 0.114961 0 0.000315156 0.00274143 0.701013 0.00216335 +4.70825 0.00273689 0.115232 0 0.000315377 0.00273689 0.703248 0.00216437 +4.71069 0.00273237 0.115503 0 0.000315597 0.00273237 0.705488 0.00216537 +4.71313 0.00272784 0.115774 0 0.000315814 0.00272784 0.707731 0.00216636 +4.71558 0.00272332 0.116046 0 0.000316029 0.00272332 0.709978 0.00216735 +4.71802 0.0027188 0.116317 0 0.000316242 0.0027188 0.712229 0.00216832 +4.72046 0.00271429 0.116588 0 0.000316454 0.00271429 0.714483 0.00216929 +4.7229 0.00270978 0.116859 0 0.000316663 0.00270978 0.716741 0.00217024 +4.72534 0.00270527 0.117131 0 0.00031687 0.00270527 0.719003 0.00217119 +4.72778 0.00270077 0.117402 0 0.000317075 0.00270077 0.721269 0.00217213 +4.73022 0.00269627 0.117673 0 0.000317278 0.00269627 0.723538 0.00217305 +4.73267 0.00269177 0.117944 0 0.00031748 0.00269177 0.725811 0.00217397 +4.73511 0.00268728 0.118216 0 0.000317679 0.00268728 0.728089 0.00217488 +4.73755 0.00268279 0.118487 0 0.000317876 0.00268279 0.730369 0.00217578 +4.73999 0.00267831 0.118758 0 0.000318071 0.00267831 0.732654 0.00217667 +4.74243 0.00267383 0.11903 0 0.000318265 0.00267383 0.734943 0.00217756 +4.74487 0.00266935 0.119301 0 0.000318456 0.00266935 0.737235 0.00217843 +4.74731 0.00266488 0.119572 0 0.000318645 0.00266488 0.739531 0.00217929 +4.74976 0.00266041 0.119843 0 0.000318832 0.00266041 0.741831 0.00218014 +4.7522 0.00265594 0.120115 0 0.000319018 0.00265594 0.744135 0.00218099 +4.75464 0.00265148 0.120386 0 0.000319201 0.00265148 0.746443 0.00218182 +4.75708 0.00264703 0.120657 0 0.000319382 0.00264703 0.748754 0.00218265 +4.75952 0.00264257 0.120928 0 0.000319562 0.00264257 0.75107 0.00218347 +4.76196 0.00263812 0.1212 0 0.000319739 0.00263812 0.753389 0.00218428 +4.7644 0.00263367 0.121471 0 0.000319915 0.00263367 0.755713 0.00218508 +4.76685 0.00262923 0.121742 0 0.000320088 0.00262923 0.75804 0.00218587 +4.76929 0.00262479 0.122013 0 0.00032026 0.00262479 0.760371 0.00218665 +4.77173 0.00262036 0.122285 0 0.00032043 0.00262036 0.762707 0.00218742 +4.77417 0.00261593 0.122556 0 0.000320597 0.00261593 0.765046 0.00218818 +4.77661 0.0026115 0.122827 0 0.000320763 0.0026115 0.767389 0.00218894 +4.77905 0.00260707 0.123099 0 0.000320927 0.00260707 0.769736 0.00218968 +4.78149 0.00260265 0.12337 0 0.000321089 0.00260265 0.772087 0.00219042 +4.78394 0.00259824 0.123641 0 0.000321249 0.00259824 0.774442 0.00219115 +4.78638 0.00259382 0.123912 0 0.000321407 0.00259382 0.776801 0.00219186 +4.78882 0.00258941 0.124184 0 0.000321563 0.00258941 0.779164 0.00219257 +4.79126 0.00258501 0.124455 0 0.000321717 0.00258501 0.781531 0.00219327 +4.7937 0.00258061 0.124726 0 0.000321869 0.00258061 0.783902 0.00219397 +4.79614 0.00257621 0.124997 0 0.00032202 0.00257621 0.786278 0.00219465 +4.79858 0.00257182 0.125269 0 0.000322168 0.00257182 0.788657 0.00219532 +4.80103 0.00256743 0.12554 0 0.000322314 0.00256743 0.79104 0.00219599 +4.80347 0.00256304 0.125811 0 0.000322459 0.00256304 0.793427 0.00219665 +4.80591 0.00255866 0.126082 0 0.000322602 0.00255866 0.795819 0.00219729 +4.80835 0.00255428 0.126354 0 0.000322742 0.00255428 0.798214 0.00219793 +4.81079 0.0025499 0.126625 0 0.000322881 0.0025499 0.800614 0.00219856 +4.81323 0.00254553 0.126896 0 0.000323018 0.00254553 0.803018 0.00219918 +4.81567 0.00254116 0.127168 0 0.000323153 0.00254116 0.805426 0.0021998 +4.81812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.82788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.83765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.84985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8645 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.86938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.88892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.8938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.90845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.92798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.94995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.96948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.97925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.98901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9939 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0061 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01099 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01343 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01587 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.01831 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02075 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02319 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02563 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.02808 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03052 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03296 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.03784 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04028 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04272 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04517 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04761 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05005 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05249 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05493 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05737 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.05981 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06226 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06714 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06958 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07202 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07446 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.07935 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08179 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08423 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08667 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08911 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09155 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09399 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09644 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.09888 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10132 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10376 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10864 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11108 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11353 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11597 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.11841 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12085 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12329 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12573 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12817 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13062 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13306 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.1355 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.13794 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14038 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14282 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14526 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15015 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15259 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15503 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15747 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.15991 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16235 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16724 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16968 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17212 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17456 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.17944 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18433 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18677 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18921 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19165 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19409 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19653 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.19897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20142 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20386 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20874 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21118 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21362 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.21851 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22095 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22339 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22583 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22827 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23071 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23315 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.23804 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24048 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24292 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24536 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25024 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25269 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25513 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.25757 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26001 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26245 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26489 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26733 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26978 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27222 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27466 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2771 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.27954 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28198 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28442 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28687 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.28931 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29175 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29419 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29663 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.29907 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30151 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30396 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3064 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30884 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31128 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31372 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.31616 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3186 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32104 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32349 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32593 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32837 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33081 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33325 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33569 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.33813 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34058 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34302 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.34546 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3479 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35034 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35278 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35522 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.35767 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36011 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36255 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36499 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36743 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36987 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37231 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37476 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.37964 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38208 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38452 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38696 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39185 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39429 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39673 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.39917 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40161 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40405 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40649 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40894 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41138 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41382 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41626 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42114 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42358 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.42847 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43091 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43335 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43579 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43823 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44067 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.44556 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45044 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45288 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45532 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45776 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46265 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46509 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46753 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.46997 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47241 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47485 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47974 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48218 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48462 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.48706 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.4895 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49194 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49683 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49927 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50171 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50415 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50659 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.50903 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51147 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51392 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51636 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5188 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52124 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52368 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52612 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.52856 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53101 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53345 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53589 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53833 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54077 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54321 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.54565 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55054 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55298 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55542 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55786 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5603 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56274 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56519 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.56763 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57007 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57251 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57495 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57739 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57983 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58228 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58472 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.58716 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59204 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59448 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59692 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.59937 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60181 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60425 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60669 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.60913 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61157 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61401 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61646 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6189 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62134 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62378 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62622 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.62866 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6311 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63354 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63599 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63843 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64087 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64331 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64575 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.64819 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65063 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65308 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65552 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.65796 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66284 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66528 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.66772 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67017 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67261 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67505 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67749 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67993 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68237 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68481 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.68726 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6897 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69214 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69458 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69702 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69946 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70435 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70679 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.70923 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71167 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71411 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71655 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71899 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72388 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72632 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.72876 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7312 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73364 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73608 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74097 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74341 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74585 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.74829 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75317 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75562 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75806 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.7605 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76294 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76538 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.76782 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77026 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77271 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77515 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77759 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78003 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78247 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78491 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78735 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.78979 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79224 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79468 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79712 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79956 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80444 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80688 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.80933 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81177 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81421 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81665 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81909 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82153 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82397 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82642 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.82886 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83374 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83618 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83862 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84106 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84351 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84595 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84839 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85083 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85327 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85571 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.85815 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8606 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86304 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86548 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86792 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87036 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.8728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87524 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.87769 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88013 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88257 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88501 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88745 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88989 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89233 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89478 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89722 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.89966 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9021 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90454 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90698 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.90942 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91187 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91431 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91675 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.91919 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92163 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92407 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92651 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92896 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9314 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93384 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93628 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.93872 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94116 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9436 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94604 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94849 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95093 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95337 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95581 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.95825 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96069 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96313 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96558 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.96802 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97046 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9729 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97534 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.97778 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98267 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98511 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98755 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98999 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99243 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99487 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.99976 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0022 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00464 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00708 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00952 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01196 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0144 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01685 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.01929 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02173 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02417 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02661 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02905 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03149 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03394 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03638 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.03882 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04126 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.0437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04614 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04858 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05103 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05347 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05591 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.05835 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06079 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06323 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06567 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06812 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07056 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.073 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07544 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.07788 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08032 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08276 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08521 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08765 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09009 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09253 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09497 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09741 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.09985 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10229 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10474 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10718 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10962 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11206 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1145 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11694 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.11938 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12183 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12427 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12671 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12915 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13159 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13403 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13647 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.13892 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14136 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1438 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14624 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14868 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15112 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15356 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15601 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.15845 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16089 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16333 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16577 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16821 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17065 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1731 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17554 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.17798 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18042 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18286 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.1853 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18774 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19019 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19263 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19507 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19751 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.19995 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20239 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20483 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20728 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20972 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21216 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21704 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.21948 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22192 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22437 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22681 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.22925 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23169 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23413 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23657 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.23901 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24146 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2439 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24634 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24878 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25122 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25366 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2561 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25854 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.26099 0.001 0 0 0 0.001 0 0 +6.26343 0.001 0 0 0 0.001 0 0 +6.26587 0.001 0 0 0 0.001 0 0 +6.26831 0.001 0 0 0 0.001 0 0 +6.27075 0.001 0 0 0 0.001 0 0 +6.27319 0.001 0 0 0 0.001 0 0 +6.27563 0.001 0 0 0 0.001 0 0 +6.27808 0.001 0 0 0 0.001 0 0 +6.28052 0.001 0 0 0 0.001 0 0 +6.28296 0.001 0 0 0 0.001 0 0 +6.2854 0.001 0 0 0 0.001 0 0 +6.28784 0.001 0 0 0 0.001 0 0 +6.29028 0.001 0 0 0 0.001 0 0 +6.29272 0.001 0 0 0 0.001 0 0 +6.29517 0.001 0 0 0 0.001 0 0 +6.29761 0.001 0 0 0 0.001 0 0 +6.30005 0.001 0 0 0 0.001 0 0 +6.30249 0.001 0 0 0 0.001 0 0 +6.30493 0.001 0 0 0 0.001 0 0 +6.30737 0.001 0 0 0 0.001 0 0 +6.30981 0.001 0 0 0 0.001 0 0 +6.31226 0.001 0 0 0 0.001 0 0 +6.3147 0.001 0 0 0 0.001 0 0 +6.31714 0.001 0 0 0 0.001 0 0 +6.31958 0.001 0 0 0 0.001 0 0 +6.32202 0.001 0 0 0 0.001 0 0 +6.32446 0.001 0 0 0 0.001 0 0 +6.3269 0.001 0 0 0 0.001 0 0 +6.32935 0.001 0 0 0 0.001 0 0 +6.33179 0.001 0 0 0 0.001 0 0 +6.33423 0.001 0 0 0 0.001 0 0 +6.33667 0.001 0 0 0 0.001 0 0 +6.33911 0.001 0 0 0 0.001 0 0 +6.34155 0.001 0 0 0 0.001 0 0 +6.34399 0.001 0 0 0 0.001 0 0 +6.34644 0.001 0 0 0 0.001 0 0 +6.34888 0.001 0 0 0 0.001 0 0 +6.35132 0.001 0 0 0 0.001 0 0 +6.35376 0.001 0 0 0 0.001 0 0 +6.3562 0.001 0 0 0 0.001 0 0 +6.35864 0.001 0 0 0 0.001 0 0 +6.36108 0.001 0 0 0 0.001 0 0 +6.36353 0.001 0 0 0 0.001 0 0 +6.36597 0.001 0 0 0 0.001 0 0 +6.36841 0.001 0 0 0 0.001 0 0 +6.37085 0.001 0 0 0 0.001 0 0 +6.37329 0.001 0 0 0 0.001 0 0 +6.37573 0.001 0 0 0 0.001 0 0 +6.37817 0.001 0 0 0 0.001 0 0 +6.38062 0.001 0 0 0 0.001 0 0 +6.38306 0.001 0 0 0 0.001 0 0 +6.3855 0.001 0 0 0 0.001 0 0 +6.38794 0.001 0 0 0 0.001 0 0 +6.39038 0.001 0 0 0 0.001 0 0 +6.39282 0.001 0 0 0 0.001 0 0 +6.39526 0.001 0 0 0 0.001 0 0 +6.39771 0.001 0 0 0 0.001 0 0 +6.40015 0.001 0 0 0 0.001 0 0 +6.40259 0.001 0 0 0 0.001 0 0 +6.40503 0.001 0 0 0 0.001 0 0 +6.40747 0.001 0 0 0 0.001 0 0 +6.40991 0.001 0 0 0 0.001 0 0 +6.41235 0.001 0 0 0 0.001 0 0 +6.41479 0.001 0 0 0 0.001 0 0 +6.41724 0.001 0 0 0 0.001 0 0 +6.41968 0.001 0 0 0 0.001 0 0 +6.42212 0.001 0 0 0 0.001 0 0 +6.42456 0.001 0 0 0 0.001 0 0 +6.427 0.001 0 0 0 0.001 0 0 +6.42944 0.001 0 0 0 0.001 0 0 +6.43188 0.001 0 0 0 0.001 0 0 +6.43433 0.001 0 0 0 0.001 0 0 +6.43677 0.001 0 0 0 0.001 0 0 +6.43921 0.001 0 0 0 0.001 0 0 +6.44165 0.001 0 0 0 0.001 0 0 +6.44409 0.001 0 0 0 0.001 0 0 +6.44653 0.001 0 0 0 0.001 0 0 +6.44897 0.001 0 0 0 0.001 0 0 +6.45142 0.001 0 0 0 0.001 0 0 +6.45386 0.001 0 0 0 0.001 0 0 +6.4563 0.001 0 0 0 0.001 0 0 +6.45874 0.001 0 0 0 0.001 0 0 +6.46118 0.001 0 0 0 0.001 0 0 +6.46362 0.001 0 0 0 0.001 0 0 +6.46606 0.001 0 0 0 0.001 0 0 +6.46851 0.001 0 0 0 0.001 0 0 +6.47095 0.001 0 0 0 0.001 0 0 +6.47339 0.001 0 0 0 0.001 0 0 +6.47583 0.001 0 0 0 0.001 0 0 +6.47827 0.001 0 0 0 0.001 0 0 +6.48071 0.001 0 0 0 0.001 0 0 +6.48315 0.001 0 0 0 0.001 0 0 +6.4856 0.001 0 0 0 0.001 0 0 +6.48804 0.001 0 0 0 0.001 0 0 +6.49048 0.001 0 0 0 0.001 0 0 +6.49292 0.001 0 0 0 0.001 0 0 +6.49536 0.001 0 0 0 0.001 0 0 +6.4978 0.001 0 0 0 0.001 0 0 +6.50024 0.001 0 0 0 0.001 0 0 +6.50269 0.001 0 0 0 0.001 0 0 +6.50513 0.001 0 0 0 0.001 0 0 +6.50757 0.001 0 0 0 0.001 0 0 +6.51001 0.001 0 0 0 0.001 0 0 +6.51245 0.001 0 0 0 0.001 0 0 +6.51489 0.001 0 0 0 0.001 0 0 +6.51733 0.001 0 0 0 0.001 0 0 +6.51978 0.001 0 0 0 0.001 0 0 +6.52222 0.001 0 0 0 0.001 0 0 +6.52466 0.001 0 0 0 0.001 0 0 +6.5271 0.001 0 0 0 0.001 0 0 +6.52954 0.001 0 0 0 0.001 0 0 +6.53198 0.001 0 0 0 0.001 0 0 +6.53442 0.001 0 0 0 0.001 0 0 +6.53687 0.001 0 0 0 0.001 0 0 +6.53931 0.001 0 0 0 0.001 0 0 +6.54175 0.001 0 0 0 0.001 0 0 +6.54419 0.001 0 0 0 0.001 0 0 +6.54663 0.001 0 0 0 0.001 0 0 +6.54907 0.001 0 0 0 0.001 0 0 +6.55151 0.001 0 0 0 0.001 0 0 +6.55396 0.001 0 0 0 0.001 0 0 +6.5564 0.001 0 0 0 0.001 0 0 +6.55884 0.001 0 0 0 0.001 0 0 +6.56128 0.001 0 0 0 0.001 0 0 +6.56372 0.001 0 0 0 0.001 0 0 +6.56616 0.001 0 0 0 0.001 0 0 +6.5686 0.001 0 0 0 0.001 0 0 +6.57104 0.001 0 0 0 0.001 0 0 +6.57349 0.001 0 0 0 0.001 0 0 +6.57593 0.001 0 0 0 0.001 0 0 +6.57837 0.001 0 0 0 0.001 0 0 +6.58081 0.001 0 0 0 0.001 0 0 +6.58325 0.001 0 0 0 0.001 0 0 +6.58569 0.001 0 0 0 0.001 0 0 +6.58813 0.001 0 0 0 0.001 0 0 +6.59058 0.001 0 0 0 0.001 0 0 +6.59302 0.001 0 0 0 0.001 0 0 +6.59546 0.001 0 0 0 0.001 0 0 +6.5979 0.001 0 0 0 0.001 0 0 +6.60034 0.001 0 0 0 0.001 0 0 +6.60278 0.001 0 0 0 0.001 0 0 +6.60522 0.001 0 0 0 0.001 0 0 +6.60767 0.001 0 0 0 0.001 0 0 +6.61011 0.001 0 0 0 0.001 0 0 +6.61255 0.001 0 0 0 0.001 0 0 +6.61499 0.001 0 0 0 0.001 0 0 +6.61743 0.001 0 0 0 0.001 0 0 +6.61987 0.001 0 0 0 0.001 0 0 +6.62231 0.001 0 0 0 0.001 0 0 +6.62476 0.001 0 0 0 0.001 0 0 +6.6272 0.001 0 0 0 0.001 0 0 +6.62964 0.001 0 0 0 0.001 0 0 +6.63208 0.001 0 0 0 0.001 0 0 +6.63452 0.001 0 0 0 0.001 0 0 +6.63696 0.001 0 0 0 0.001 0 0 +6.6394 0.001 0 0 0 0.001 0 0 +6.64185 0.001 0 0 0 0.001 0 0 +6.64429 0.001 0 0 0 0.001 0 0 +6.64673 0.001 0 0 0 0.001 0 0 +6.64917 0.001 0 0 0 0.001 0 0 +6.65161 0.001 0 0 0 0.001 0 0 +6.65405 0.001 0 0 0 0.001 0 0 +6.65649 0.001 0 0 0 0.001 0 0 +6.65894 0.001 0 0 0 0.001 0 0 +6.66138 0.001 0 0 0 0.001 0 0 +6.66382 0.001 0 0 0 0.001 0 0 +6.66626 0.001 0 0 0 0.001 0 0 +6.6687 0.001 0 0 0 0.001 0 0 +6.67114 0.001 0 0 0 0.001 0 0 +6.67358 0.001 0 0 0 0.001 0 0 +6.67603 0.001 0 0 0 0.001 0 0 +6.67847 0.001 0 0 0 0.001 0 0 +6.68091 0.001 0 0 0 0.001 0 0 +6.68335 0.001 0 0 0 0.001 0 0 +6.68579 0.001 0 0 0 0.001 0 0 +6.68823 0.001 0 0 0 0.001 0 0 +6.69067 0.001 0 0 0 0.001 0 0 +6.69312 0.001 0 0 0 0.001 0 0 +6.69556 0.001 0 0 0 0.001 0 0 +6.698 0.001 0 0 0 0.001 0 0 +6.70044 0.001 0 0 0 0.001 0 0 +6.70288 0.001 0 0 0 0.001 0 0 +6.70532 0.001 0 0 0 0.001 0 0 +6.70776 0.001 0 0 0 0.001 0 0 +6.71021 0.001 0 0 0 0.001 0 0 +6.71265 0.001 0 0 0 0.001 0 0 +6.71509 0.001 0 0 0 0.001 0 0 +6.71753 0.001 0 0 0 0.001 0 0 +6.71997 0.001 0 0 0 0.001 0 0 +6.72241 0.001 0 0 0 0.001 0 0 +6.72485 0.001 0 0 0 0.001 0 0 +6.72729 0.001 0 0 0 0.001 0 0 +6.72974 0.001 0 0 0 0.001 0 0 +6.73218 0.001 0 0 0 0.001 0 0 +6.73462 0.001 0 0 0 0.001 0 0 +6.73706 0.001 0 0 0 0.001 0 0 +6.7395 0.001 0 0 0 0.001 0 0 +6.74194 0.001 0 0 0 0.001 0 0 +6.74438 0.001 0 0 0 0.001 0 0 +6.74683 0.001 0 0 0 0.001 0 0 +6.74927 0.001 0 0 0 0.001 0 0 +6.75171 0.001 0 0 0 0.001 0 0 +6.75415 0.001 0 0 0 0.001 0 0 +6.75659 0.001 0 0 0 0.001 0 0 +6.75903 0.001 0 0 0 0.001 0 0 +6.76147 0.001 0 0 0 0.001 0 0 +6.76392 0.001 0 0 0 0.001 0 0 +6.76636 0.001 0 0 0 0.001 0 0 +6.7688 0.001 0 0 0 0.001 0 0 +6.77124 0.001 0 0 0 0.001 0 0 +6.77368 0.001 0 0 0 0.001 0 0 +6.77612 0.001 0 0 0 0.001 0 0 +6.77856 0.001 0 0 0 0.001 0 0 +6.78101 0.001 0 0 0 0.001 0 0 +6.78345 0.001 0 0 0 0.001 0 0 +6.78589 0.001 0 0 0 0.001 0 0 +6.78833 0.001 0 0 0 0.001 0 0 +6.79077 0.001 0 0 0 0.001 0 0 +6.79321 0.001 0 0 0 0.001 0 0 +6.79565 0.001 0 0 0 0.001 0 0 +6.7981 0.001 0 0 0 0.001 0 0 +6.80054 0.001 0 0 0 0.001 0 0 +6.80298 0.001 0 0 0 0.001 0 0 +6.80542 0.001 0 0 0 0.001 0 0 +6.80786 0.001 0 0 0 0.001 0 0 +6.8103 0.001 0 0 0 0.001 0 0 +6.81274 0.001 0 0 0 0.001 0 0 +6.81519 0.001 0 0 0 0.001 0 0 +6.81763 0.001 0 0 0 0.001 0 0 +6.82007 0.001 0 0 0 0.001 0 0 +6.82251 0.001 0 0 0 0.001 0 0 +6.82495 0.001 0 0 0 0.001 0 0 +6.82739 0.001 0 0 0 0.001 0 0 +6.82983 0.001 0 0 0 0.001 0 0 +6.83228 0.001 0 0 0 0.001 0 0 +6.83472 0.001 0 0 0 0.001 0 0 +6.83716 0.001 0 0 0 0.001 0 0 +6.8396 0.001 0 0 0 0.001 0 0 +6.84204 0.001 0 0 0 0.001 0 0 +6.84448 0.001 0 0 0 0.001 0 0 +6.84692 0.001 0 0 0 0.001 0 0 +6.84937 0.001 0 0 0 0.001 0 0 +6.85181 0.001 0 0 0 0.001 0 0 +6.85425 0.001 0 0 0 0.001 0 0 +6.85669 0.001 0 0 0 0.001 0 0 +6.85913 0.001 0 0 0 0.001 0 0 +6.86157 0.001 0 0 0 0.001 0 0 +6.86401 0.001 0 0 0 0.001 0 0 +6.86646 0.001 0 0 0 0.001 0 0 +6.8689 0.001 0 0 0 0.001 0 0 +6.87134 0.001 0 0 0 0.001 0 0 +6.87378 0.001 0 0 0 0.001 0 0 +6.87622 0.001 0 0 0 0.001 0 0 +6.87866 0.001 0 0 0 0.001 0 0 +6.8811 0.001 0 0 0 0.001 0 0 +6.88354 0.001 0 0 0 0.001 0 0 +6.88599 0.001 0 0 0 0.001 0 0 +6.88843 0.001 0 0 0 0.001 0 0 +6.89087 0.001 0 0 0 0.001 0 0 +6.89331 0.001 0 0 0 0.001 0 0 +6.89575 0.001 0 0 0 0.001 0 0 +6.89819 0.001 0 0 0 0.001 0 0 +6.90063 0.001 0 0 0 0.001 0 0 +6.90308 0.001 0 0 0 0.001 0 0 +6.90552 0.001 0 0 0 0.001 0 0 +6.90796 0.001 0 0 0 0.001 0 0 +6.9104 0.001 0 0 0 0.001 0 0 +6.91284 0.001 0 0 0 0.001 0 0 +6.91528 0.001 0 0 0 0.001 0 0 +6.91772 0.001 0 0 0 0.001 0 0 +6.92017 0.001 0 0 0 0.001 0 0 +6.92261 0.001 0 0 0 0.001 0 0 +6.92505 0.001 0 0 0 0.001 0 0 +6.92749 0.001 0 0 0 0.001 0 0 +6.92993 0.001 0 0 0 0.001 0 0 +6.93237 0.001 0 0 0 0.001 0 0 +6.93481 0.001 0 0 0 0.001 0 0 +6.93726 0.001 0 0 0 0.001 0 0 +6.9397 0.001 0 0 0 0.001 0 0 +6.94214 0.001 0 0 0 0.001 0 0 +6.94458 0.001 0 0 0 0.001 0 0 +6.94702 0.001 0 0 0 0.001 0 0 +6.94946 0.001 0 0 0 0.001 0 0 +6.9519 0.001 0 0 0 0.001 0 0 +6.95435 0.001 0 0 0 0.001 0 0 +6.95679 0.001 0 0 0 0.001 0 0 +6.95923 0.001 0 0 0 0.001 0 0 +6.96167 0.001 0 0 0 0.001 0 0 +6.96411 0.001 0 0 0 0.001 0 0 +6.96655 0.001 0 0 0 0.001 0 0 +6.96899 0.001 0 0 0 0.001 0 0 +6.97144 0.001 0 0 0 0.001 0 0 +6.97388 0.001 0 0 0 0.001 0 0 +6.97632 0.001 0 0 0 0.001 0 0 +6.97876 0.001 0 0 0 0.001 0 0 +6.9812 0.001 0 0 0 0.001 0 0 +6.98364 0.001 0 0 0 0.001 0 0 +6.98608 0.001 0 0 0 0.001 0 0 +6.98853 0.001 0 0 0 0.001 0 0 +6.99097 0.001 0 0 0 0.001 0 0 +6.99341 0.001 0 0 0 0.001 0 0 +6.99585 0.001 0 0 0 0.001 0 0 +6.99829 0.001 0 0 0 0.001 0 0 +7.00073 0.001 0 0 0 0.001 0 0 +7.00317 0.001 0 0 0 0.001 0 0 +7.00562 0.001 0 0 0 0.001 0 0 +7.00806 0.001 0 0 0 0.001 0 0 +7.0105 0.001 0 0 0 0.001 0 0 +7.01294 0.001 0 0 0 0.001 0 0 +7.01538 0.001 0 0 0 0.001 0 0 +7.01782 0.001 0 0 0 0.001 0 0 +7.02026 0.001 0 0 0 0.001 0 0 +7.02271 0.001 0 0 0 0.001 0 0 +7.02515 0.001 0 0 0 0.001 0 0 +7.02759 0.001 0 0 0 0.001 0 0 +7.03003 0.001 0 0 0 0.001 0 0 +7.03247 0.001 0 0 0 0.001 0 0 +7.03491 0.001 0 0 0 0.001 0 0 +7.03735 0.001 0 0 0 0.001 0 0 +7.03979 0.001 0 0 0 0.001 0 0 +7.04224 0.001 0 0 0 0.001 0 0 +7.04468 0.001 0 0 0 0.001 0 0 +7.04712 0.001 0 0 0 0.001 0 0 +7.04956 0.001 0 0 0 0.001 0 0 +7.052 0.001 0 0 0 0.001 0 0 +7.05444 0.001 0 0 0 0.001 0 0 +7.05688 0.001 0 0 0 0.001 0 0 +7.05933 0.001 0 0 0 0.001 0 0 +7.06177 0.001 0 0 0 0.001 0 0 +7.06421 0.001 0 0 0 0.001 0 0 +7.06665 0.001 0 0 0 0.001 0 0 +7.06909 0.001 0 0 0 0.001 0 0 +7.07153 0.001 0 0 0 0.001 0 0 +7.07397 0.001 0 0 0 0.001 0 0 +7.07642 0.001 0 0 0 0.001 0 0 +7.07886 0.001 0 0 0 0.001 0 0 +7.0813 0.001 0 0 0 0.001 0 0 +7.08374 0.001 0 0 0 0.001 0 0 +7.08618 0.001 0 0 0 0.001 0 0 +7.08862 0.001 0 0 0 0.001 0 0 +7.09106 0.001 0 0 0 0.001 0 0 +7.09351 0.001 0 0 0 0.001 0 0 +7.09595 0.001 0 0 0 0.001 0 0 +7.09839 0.001 0 0 0 0.001 0 0 +7.10083 0.001 0 0 0 0.001 0 0 +7.10327 0.001 0 0 0 0.001 0 0 +7.10571 0.001 0 0 0 0.001 0 0 +7.10815 0.001 0 0 0 0.001 0 0 +7.1106 0.001 0 0 0 0.001 0 0 +7.11304 0.001 0 0 0 0.001 0 0 +7.11548 0.001 0 0 0 0.001 0 0 +7.11792 0.001 0 0 0 0.001 0 0 +7.12036 0.001 0 0 0 0.001 0 0 +7.1228 0.001 0 0 0 0.001 0 0 +7.12524 0.001 0 0 0 0.001 0 0 +7.12769 0.001 0 0 0 0.001 0 0 +7.13013 0.001 0 0 0 0.001 0 0 +7.13257 0.001 0 0 0 0.001 0 0 +7.13501 0.001 0 0 0 0.001 0 0 +7.13745 0.001 0 0 0 0.001 0 0 +7.13989 0.001 0 0 0 0.001 0 0 +7.14233 0.001 0 0 0 0.001 0 0 +7.14478 0.001 0 0 0 0.001 0 0 +7.14722 0.001 0 0 0 0.001 0 0 +7.14966 0.001 0 0 0 0.001 0 0 +7.1521 0.001 0 0 0 0.001 0 0 +7.15454 0.001 0 0 0 0.001 0 0 +7.15698 0.001 0 0 0 0.001 0 0 +7.15942 0.001 0 0 0 0.001 0 0 +7.16187 0.001 0 0 0 0.001 0 0 +7.16431 0.001 0 0 0 0.001 0 0 +7.16675 0.001 0 0 0 0.001 0 0 +7.16919 0.001 0 0 0 0.001 0 0 +7.17163 0.001 0 0 0 0.001 0 0 +7.17407 0.001 0 0 0 0.001 0 0 +7.17651 0.001 0 0 0 0.001 0 0 +7.17896 0.001 0 0 0 0.001 0 0 +7.1814 0.001 0 0 0 0.001 0 0 +7.18384 0.001 0 0 0 0.001 0 0 +7.18628 0.001 0 0 0 0.001 0 0 +7.18872 0.001 0 0 0 0.001 0 0 +7.19116 0.001 0 0 0 0.001 0 0 +7.1936 0.001 0 0 0 0.001 0 0 +7.19604 0.001 0 0 0 0.001 0 0 +7.19849 0.001 0 0 0 0.001 0 0 +7.20093 0.001 0 0 0 0.001 0 0 +7.20337 0.001 0 0 0 0.001 0 0 +7.20581 0.001 0 0 0 0.001 0 0 +7.20825 0.001 0 0 0 0.001 0 0 +7.21069 0.001 0 0 0 0.001 0 0 +7.21313 0.001 0 0 0 0.001 0 0 +7.21558 0.001 0 0 0 0.001 0 0 +7.21802 0.001 0 0 0 0.001 0 0 +7.22046 0.001 0 0 0 0.001 0 0 +7.2229 0.001 0 0 0 0.001 0 0 +7.22534 0.001 0 0 0 0.001 0 0 +7.22778 0.001 0 0 0 0.001 0 0 +7.23022 0.001 0 0 0 0.001 0 0 +7.23267 0.001 0 0 0 0.001 0 0 +7.23511 0.001 0 0 0 0.001 0 0 +7.23755 0.001 0 0 0 0.001 0 0 +7.23999 0.001 0 0 0 0.001 0 0 +7.24243 0.001 0 0 0 0.001 0 0 +7.24487 0.001 0 0 0 0.001 0 0 +7.24731 0.001 0 0 0 0.001 0 0 +7.24976 0.001 0 0 0 0.001 0 0 +7.2522 0.001 0 0 0 0.001 0 0 +7.25464 0.001 0 0 0 0.001 0 0 +7.25708 0.001 0 0 0 0.001 0 0 +7.25952 0.001 0 0 0 0.001 0 0 +7.26196 0.001 0 0 0 0.001 0 0 +7.2644 0.001 0 0 0 0.001 0 0 +7.26685 0.001 0 0 0 0.001 0 0 +7.26929 0.001 0 0 0 0.001 0 0 +7.27173 0.001 0 0 0 0.001 0 0 +7.27417 0.001 0 0 0 0.001 0 0 +7.27661 0.001 0 0 0 0.001 0 0 +7.27905 0.001 0 0 0 0.001 0 0 +7.28149 0.001 0 0 0 0.001 0 0 +7.28394 0.001 0 0 0 0.001 0 0 +7.28638 0.001 0 0 0 0.001 0 0 +7.28882 0.001 0 0 0 0.001 0 0 +7.29126 0.001 0 0 0 0.001 0 0 +7.2937 0.001 0 0 0 0.001 0 0 +7.29614 0.001 0 0 0 0.001 0 0 +7.29858 0.001 0 0 0 0.001 0 0 +7.30103 0.001 0 0 0 0.001 0 0 +7.30347 0.001 0 0 0 0.001 0 0 +7.30591 0.001 0 0 0 0.001 0 0 +7.30835 0.001 0 0 0 0.001 0 0 +7.31079 0.001 0 0 0 0.001 0 0 +7.31323 0.001 0 0 0 0.001 0 0 +7.31567 0.001 0 0 0 0.001 0 0 +7.31812 0.001 0 0 0 0.001 0 0 +7.32056 0.001 0 0 0 0.001 0 0 +7.323 0.001 0 0 0 0.001 0 0 +7.32544 0.001 0 0 0 0.001 0 0 +7.32788 0.001 0 0 0 0.001 0 0 +7.33032 0.001 0 0 0 0.001 0 0 +7.33276 0.001 0 0 0 0.001 0 0 +7.33521 0.001 0 0 0 0.001 0 0 +7.33765 0.001 0 0 0 0.001 0 0 +7.34009 0.001 0 0 0 0.001 0 0 +7.34253 0.001 0 0 0 0.001 0 0 +7.34497 0.001 0 0 0 0.001 0 0 +7.34741 0.001 0 0 0 0.001 0 0 +7.34985 0.001 0 0 0 0.001 0 0 +7.35229 0.001 0 0 0 0.001 0 0 +7.35474 0.001 0 0 0 0.001 0 0 +7.35718 0.001 0 0 0 0.001 0 0 +7.35962 0.001 0 0 0 0.001 0 0 +7.36206 0.001 0 0 0 0.001 0 0 +7.3645 0.001 0 0 0 0.001 0 0 +7.36694 0.001 0 0 0 0.001 0 0 +7.36938 0.001 0 0 0 0.001 0 0 +7.37183 0.001 0 0 0 0.001 0 0 +7.37427 0.001 0 0 0 0.001 0 0 +7.37671 0.001 0 0 0 0.001 0 0 +7.37915 0.001 0 0 0 0.001 0 0 +7.38159 0.001 0 0 0 0.001 0 0 +7.38403 0.001 0 0 0 0.001 0 0 +7.38647 0.001 0 0 0 0.001 0 0 +7.38892 0.001 0 0 0 0.001 0 0 +7.39136 0.001 0 0 0 0.001 0 0 +7.3938 0.001 0 0 0 0.001 0 0 +7.39624 0.001 0 0 0 0.001 0 0 +7.39868 0.001 0 0 0 0.001 0 0 +7.40112 0.001 0 0 0 0.001 0 0 +7.40356 0.001 0 0 0 0.001 0 0 +7.40601 0.001 0 0 0 0.001 0 0 +7.40845 0.001 0 0 0 0.001 0 0 +7.41089 0.001 0 0 0 0.001 0 0 +7.41333 0.001 0 0 0 0.001 0 0 +7.41577 0.001 0 0 0 0.001 0 0 +7.41821 0.001 0 0 0 0.001 0 0 +7.42065 0.001 0 0 0 0.001 0 0 +7.4231 0.001 0 0 0 0.001 0 0 +7.42554 0.001 0 0 0 0.001 0 0 +7.42798 0.001 0 0 0 0.001 0 0 +7.43042 0.001 0 0 0 0.001 0 0 +7.43286 0.001 0 0 0 0.001 0 0 +7.4353 0.001 0 0 0 0.001 0 0 +7.43774 0.001 0 0 0 0.001 0 0 +7.44019 0.001 0 0 0 0.001 0 0 +7.44263 0.001 0 0 0 0.001 0 0 +7.44507 0.001 0 0 0 0.001 0 0 +7.44751 0.001 0 0 0 0.001 0 0 +7.44995 0.001 0 0 0 0.001 0 0 +7.45239 0.001 0 0 0 0.001 0 0 +7.45483 0.001 0 0 0 0.001 0 0 +7.45728 0.001 0 0 0 0.001 0 0 +7.45972 0.001 0 0 0 0.001 0 0 +7.46216 0.001 0 0 0 0.001 0 0 +7.4646 0.001 0 0 0 0.001 0 0 +7.46704 0.001 0 0 0 0.001 0 0 +7.46948 0.001 0 0 0 0.001 0 0 +7.47192 0.001 0 0 0 0.001 0 0 +7.47437 0.001 0 0 0 0.001 0 0 +7.47681 0.001 0 0 0 0.001 0 0 +7.47925 0.001 0 0 0 0.001 0 0 +7.48169 0.001 0 0 0 0.001 0 0 +7.48413 0.001 0 0 0 0.001 0 0 +7.48657 0.001 0 0 0 0.001 0 0 +7.48901 0.001 0 0 0 0.001 0 0 +7.49146 0.001 0 0 0 0.001 0 0 +7.4939 0.001 0 0 0 0.001 0 0 +7.49634 0.001 0 0 0 0.001 0 0 +7.49878 0.001 0 0 0 0.001 0 0 +7.50122 0.001 0 0 0 0.001 0 0 +7.50366 0.001 0 0 0 0.001 0 0 +7.5061 0.001 0 0 0 0.001 0 0 +7.50854 0.001 0 0 0 0.001 0 0 +7.51099 0.001 0 0 0 0.001 0 0 +7.51343 0.001 0 0 0 0.001 0 0 +7.51587 0.001 0 0 0 0.001 0 0 +7.51831 0.001 0 0 0 0.001 0 0 +7.52075 0.001 0 0 0 0.001 0 0 +7.52319 0.001 0 0 0 0.001 0 0 +7.52563 0.001 0 0 0 0.001 0 0 +7.52808 0.001 0 0 0 0.001 0 0 +7.53052 0.001 0 0 0 0.001 0 0 +7.53296 0.001 0 0 0 0.001 0 0 +7.5354 0.001 0 0 0 0.001 0 0 +7.53784 0.001 0 0 0 0.001 0 0 +7.54028 0.001 0 0 0 0.001 0 0 +7.54272 0.001 0 0 0 0.001 0 0 +7.54517 0.001 0 0 0 0.001 0 0 +7.54761 0.001 0 0 0 0.001 0 0 +7.55005 0.001 0 0 0 0.001 0 0 +7.55249 0.001 0 0 0 0.001 0 0 +7.55493 0.001 0 0 0 0.001 0 0 +7.55737 0.001 0 0 0 0.001 0 0 +7.55981 0.001 0 0 0 0.001 0 0 +7.56226 0.001 0 0 0 0.001 0 0 +7.5647 0.001 0 0 0 0.001 0 0 +7.56714 0.001 0 0 0 0.001 0 0 +7.56958 0.001 0 0 0 0.001 0 0 +7.57202 0.001 0 0 0 0.001 0 0 +7.57446 0.001 0 0 0 0.001 0 0 +7.5769 0.001 0 0 0 0.001 0 0 +7.57935 0.001 0 0 0 0.001 0 0 +7.58179 0.001 0 0 0 0.001 0 0 +7.58423 0.001 0 0 0 0.001 0 0 +7.58667 0.001 0 0 0 0.001 0 0 +7.58911 0.001 0 0 0 0.001 0 0 +7.59155 0.001 0 0 0 0.001 0 0 +7.59399 0.001 0 0 0 0.001 0 0 +7.59644 0.001 0 0 0 0.001 0 0 +7.59888 0.001 0 0 0 0.001 0 0 +7.60132 0.001 0 0 0 0.001 0 0 +7.60376 0.001 0 0 0 0.001 0 0 +7.6062 0.001 0 0 0 0.001 0 0 +7.60864 0.001 0 0 0 0.001 0 0 +7.61108 0.001 0 0 0 0.001 0 0 +7.61353 0.001 0 0 0 0.001 0 0 +7.61597 0.001 0 0 0 0.001 0 0 +7.61841 0.001 0 0 0 0.001 0 0 +7.62085 0.001 0 0 0 0.001 0 0 +7.62329 0.001 0 0 0 0.001 0 0 +7.62573 0.001 0 0 0 0.001 0 0 +7.62817 0.001 0 0 0 0.001 0 0 +7.63062 0.001 0 0 0 0.001 0 0 +7.63306 0.001 0 0 0 0.001 0 0 +7.6355 0.001 0 0 0 0.001 0 0 +7.63794 0.001 0 0 0 0.001 0 0 +7.64038 0.001 0 0 0 0.001 0 0 +7.64282 0.001 0 0 0 0.001 0 0 +7.64526 0.001 0 0 0 0.001 0 0 +7.64771 0.001 0 0 0 0.001 0 0 +7.65015 0.001 0 0 0 0.001 0 0 +7.65259 0.001 0 0 0 0.001 0 0 +7.65503 0.001 0 0 0 0.001 0 0 +7.65747 0.001 0 0 0 0.001 0 0 +7.65991 0.001 0 0 0 0.001 0 0 +7.66235 0.001 0 0 0 0.001 0 0 +7.66479 0.001 0 0 0 0.001 0 0 +7.66724 0.001 0 0 0 0.001 0 0 +7.66968 0.001 0 0 0 0.001 0 0 +7.67212 0.001 0 0 0 0.001 0 0 +7.67456 0.001 0 0 0 0.001 0 0 +7.677 0.001 0 0 0 0.001 0 0 +7.67944 0.001 0 0 0 0.001 0 0 +7.68188 0.001 0 0 0 0.001 0 0 +7.68433 0.001 0 0 0 0.001 0 0 +7.68677 0.001 0 0 0 0.001 0 0 +7.68921 0.001 0 0 0 0.001 0 0 +7.69165 0.001 0 0 0 0.001 0 0 +7.69409 0.001 0 0 0 0.001 0 0 +7.69653 0.001 0 0 0 0.001 0 0 +7.69897 0.001 0 0 0 0.001 0 0 +7.70142 0.001 0 0 0 0.001 0 0 +7.70386 0.001 0 0 0 0.001 0 0 +7.7063 0.001 0 0 0 0.001 0 0 +7.70874 0.001 0 0 0 0.001 0 0 +7.71118 0.001 0 0 0 0.001 0 0 +7.71362 0.001 0 0 0 0.001 0 0 +7.71606 0.001 0 0 0 0.001 0 0 +7.71851 0.001 0 0 0 0.001 0 0 +7.72095 0.001 0 0 0 0.001 0 0 +7.72339 0.001 0 0 0 0.001 0 0 +7.72583 0.001 0 0 0 0.001 0 0 +7.72827 0.001 0 0 0 0.001 0 0 +7.73071 0.001 0 0 0 0.001 0 0 +7.73315 0.001 0 0 0 0.001 0 0 +7.7356 0.001 0 0 0 0.001 0 0 +7.73804 0.001 0 0 0 0.001 0 0 +7.74048 0.001 0 0 0 0.001 0 0 +7.74292 0.001 0 0 0 0.001 0 0 +7.74536 0.001 0 0 0 0.001 0 0 +7.7478 0.001 0 0 0 0.001 0 0 +7.75024 0.001 0 0 0 0.001 0 0 +7.75269 0.001 0 0 0 0.001 0 0 +7.75513 0.001 0 0 0 0.001 0 0 +7.75757 0.001 0 0 0 0.001 0 0 +7.76001 0.001 0 0 0 0.001 0 0 +7.76245 0.001 0 0 0 0.001 0 0 +7.76489 0.001 0 0 0 0.001 0 0 +7.76733 0.001 0 0 0 0.001 0 0 +7.76978 0.001 0 0 0 0.001 0 0 +7.77222 0.001 0 0 0 0.001 0 0 +7.77466 0.001 0 0 0 0.001 0 0 +7.7771 0.001 0 0 0 0.001 0 0 +7.77954 0.001 0 0 0 0.001 0 0 +7.78198 0.001 0 0 0 0.001 0 0 +7.78442 0.001 0 0 0 0.001 0 0 +7.78687 0.001 0 0 0 0.001 0 0 +7.78931 0.001 0 0 0 0.001 0 0 +7.79175 0.001 0 0 0 0.001 0 0 +7.79419 0.001 0 0 0 0.001 0 0 +7.79663 0.001 0 0 0 0.001 0 0 +7.79907 0.001 0 0 0 0.001 0 0 +7.80151 0.001 0 0 0 0.001 0 0 +7.80396 0.001 0 0 0 0.001 0 0 +7.8064 0.001 0 0 0 0.001 0 0 +7.80884 0.001 0 0 0 0.001 0 0 +7.81128 0.001 0 0 0 0.001 0 0 +7.81372 0.001 0 0 0 0.001 0 0 +7.81616 0.001 0 0 0 0.001 0 0 +7.8186 0.001 0 0 0 0.001 0 0 +7.82104 0.001 0 0 0 0.001 0 0 +7.82349 0.001 0 0 0 0.001 0 0 +7.82593 0.001 0 0 0 0.001 0 0 +7.82837 0.001 0 0 0 0.001 0 0 +7.83081 0.001 0 0 0 0.001 0 0 +7.83325 0.001 0 0 0 0.001 0 0 +7.83569 0.001 0 0 0 0.001 0 0 +7.83813 0.001 0 0 0 0.001 0 0 +7.84058 0.001 0 0 0 0.001 0 0 +7.84302 0.001 0 0 0 0.001 0 0 +7.84546 0.001 0 0 0 0.001 0 0 +7.8479 0.001 0 0 0 0.001 0 0 +7.85034 0.001 0 0 0 0.001 0 0 +7.85278 0.001 0 0 0 0.001 0 0 +7.85522 0.001 0 0 0 0.001 0 0 +7.85767 0.001 0 0 0 0.001 0 0 +7.86011 0.001 0 0 0 0.001 0 0 +7.86255 0.001 0 0 0 0.001 0 0 +7.86499 0.001 0 0 0 0.001 0 0 +7.86743 0.001 0 0 0 0.001 0 0 +7.86987 0.001 0 0 0 0.001 0 0 +7.87231 0.001 0 0 0 0.001 0 0 +7.87476 0.001 0 0 0 0.001 0 0 +7.8772 0.001 0 0 0 0.001 0 0 +7.87964 0.001 0 0 0 0.001 0 0 +7.88208 0.001 0 0 0 0.001 0 0 +7.88452 0.001 0 0 0 0.001 0 0 +7.88696 0.001 0 0 0 0.001 0 0 +7.8894 0.001 0 0 0 0.001 0 0 +7.89185 0.001 0 0 0 0.001 0 0 +7.89429 0.001 0 0 0 0.001 0 0 +7.89673 0.001 0 0 0 0.001 0 0 +7.89917 0.001 0 0 0 0.001 0 0 +7.90161 0.001 0 0 0 0.001 0 0 +7.90405 0.001 0 0 0 0.001 0 0 +7.90649 0.001 0 0 0 0.001 0 0 +7.90894 0.001 0 0 0 0.001 0 0 +7.91138 0.001 0 0 0 0.001 0 0 +7.91382 0.001 0 0 0 0.001 0 0 +7.91626 0.001 0 0 0 0.001 0 0 +7.9187 0.001 0 0 0 0.001 0 0 +7.92114 0.001 0 0 0 0.001 0 0 +7.92358 0.001 0 0 0 0.001 0 0 +7.92603 0.001 0 0 0 0.001 0 0 +7.92847 0.001 0 0 0 0.001 0 0 +7.93091 0.001 0 0 0 0.001 0 0 +7.93335 0.001 0 0 0 0.001 0 0 +7.93579 0.001 0 0 0 0.001 0 0 +7.93823 0.001 0 0 0 0.001 0 0 +7.94067 0.001 0 0 0 0.001 0 0 +7.94312 0.001 0 0 0 0.001 0 0 +7.94556 0.001 0 0 0 0.001 0 0 +7.948 0.001 0 0 0 0.001 0 0 +7.95044 0.001 0 0 0 0.001 0 0 +7.95288 0.001 0 0 0 0.001 0 0 +7.95532 0.001 0 0 0 0.001 0 0 +7.95776 0.001 0 0 0 0.001 0 0 +7.96021 0.001 0 0 0 0.001 0 0 +7.96265 0.001 0 0 0 0.001 0 0 +7.96509 0.001 0 0 0 0.001 0 0 +7.96753 0.001 0 0 0 0.001 0 0 +7.96997 0.001 0 0 0 0.001 0 0 +7.97241 0.001 0 0 0 0.001 0 0 +7.97485 0.001 0 0 0 0.001 0 0 +7.97729 0.001 0 0 0 0.001 0 0 +7.97974 0.001 0 0 0 0.001 0 0 +7.98218 0.001 0 0 0 0.001 0 0 +7.98462 0.001 0 0 0 0.001 0 0 +7.98706 0.001 0 0 0 0.001 0 0 +7.9895 0.001 0 0 0 0.001 0 0 +7.99194 0.001 0 0 0 0.001 0 0 +7.99438 0.001 0 0 0 0.001 0 0 +7.99683 0.001 0 0 0 0.001 0 0 +7.99927 0.001 0 0 0 0.001 0 0 +8.00171 0.001 0 0 0 0.001 0 0 +8.00415 0.001 0 0 0 0.001 0 0 +8.00659 0.001 0 0 0 0.001 0 0 +8.00903 0.001 0 0 0 0.001 0 0 +8.01147 0.001 0 0 0 0.001 0 0 +8.01392 0.001 0 0 0 0.001 0 0 +8.01636 0.001 0 0 0 0.001 0 0 +8.0188 0.001 0 0 0 0.001 0 0 +8.02124 0.001 0 0 0 0.001 0 0 +8.02368 0.001 0 0 0 0.001 0 0 +8.02612 0.001 0 0 0 0.001 0 0 +8.02856 0.001 0 0 0 0.001 0 0 +8.03101 0.001 0 0 0 0.001 0 0 +8.03345 0.001 0 0 0 0.001 0 0 +8.03589 0.001 0 0 0 0.001 0 0 +8.03833 0.001 0 0 0 0.001 0 0 +8.04077 0.001 0 0 0 0.001 0 0 +8.04321 0.001 0 0 0 0.001 0 0 +8.04565 0.001 0 0 0 0.001 0 0 +8.0481 0.001 0 0 0 0.001 0 0 +8.05054 0.001 0 0 0 0.001 0 0 +8.05298 0.001 0 0 0 0.001 0 0 +8.05542 0.001 0 0 0 0.001 0 0 +8.05786 0.001 0 0 0 0.001 0 0 +8.0603 0.001 0 0 0 0.001 0 0 +8.06274 0.001 0 0 0 0.001 0 0 +8.06519 0.001 0 0 0 0.001 0 0 +8.06763 0.001 0 0 0 0.001 0 0 +8.07007 0.001 0 0 0 0.001 0 0 +8.07251 0.001 0 0 0 0.001 0 0 +8.07495 0.001 0 0 0 0.001 0 0 +8.07739 0.001 0 0 0 0.001 0 0 +8.07983 0.001 0 0 0 0.001 0 0 +8.08228 0.001 0 0 0 0.001 0 0 +8.08472 0.001 0 0 0 0.001 0 0 +8.08716 0.001 0 0 0 0.001 0 0 +8.0896 0.001 0 0 0 0.001 0 0 +8.09204 0.001 0 0 0 0.001 0 0 +8.09448 0.001 0 0 0 0.001 0 0 +8.09692 0.001 0 0 0 0.001 0 0 +8.09937 0.001 0 0 0 0.001 0 0 +8.10181 0.001 0 0 0 0.001 0 0 +8.10425 0.001 0 0 0 0.001 0 0 +8.10669 0.001 0 0 0 0.001 0 0 +8.10913 0.001 0 0 0 0.001 0 0 +8.11157 0.001 0 0 0 0.001 0 0 +8.11401 0.001 0 0 0 0.001 0 0 +8.11646 0.001 0 0 0 0.001 0 0 +8.1189 0.001 0 0 0 0.001 0 0 +8.12134 0.001 0 0 0 0.001 0 0 +8.12378 0.001 0 0 0 0.001 0 0 +8.12622 0.001 0 0 0 0.001 0 0 +8.12866 0.001 0 0 0 0.001 0 0 +8.1311 0.001 0 0 0 0.001 0 0 +8.13354 0.001 0 0 0 0.001 0 0 +8.13599 0.001 0 0 0 0.001 0 0 +8.13843 0.001 0 0 0 0.001 0 0 +8.14087 0.001 0 0 0 0.001 0 0 +8.14331 0.001 0 0 0 0.001 0 0 +8.14575 0.001 0 0 0 0.001 0 0 +8.14819 0.001 0 0 0 0.001 0 0 +8.15063 0.001 0 0 0 0.001 0 0 +8.15308 0.001 0 0 0 0.001 0 0 +8.15552 0.001 0 0 0 0.001 0 0 +8.15796 0.001 0 0 0 0.001 0 0 +8.1604 0.001 0 0 0 0.001 0 0 +8.16284 0.001 0 0 0 0.001 0 0 +8.16528 0.001 0 0 0 0.001 0 0 +8.16772 0.001 0 0 0 0.001 0 0 +8.17017 0.001 0 0 0 0.001 0 0 +8.17261 0.001 0 0 0 0.001 0 0 +8.17505 0.001 0 0 0 0.001 0 0 +8.17749 0.001 0 0 0 0.001 0 0 +8.17993 0.001 0 0 0 0.001 0 0 +8.18237 0.001 0 0 0 0.001 0 0 +8.18481 0.001 0 0 0 0.001 0 0 +8.18726 0.001 0 0 0 0.001 0 0 +8.1897 0.001 0 0 0 0.001 0 0 +8.19214 0.001 0 0 0 0.001 0 0 +8.19458 0.001 0 0 0 0.001 0 0 +8.19702 0.001 0 0 0 0.001 0 0 +8.19946 0.001 0 0 0 0.001 0 0 +8.2019 0.001 0 0 0 0.001 0 0 +8.20435 0.001 0 0 0 0.001 0 0 +8.20679 0.001 0 0 0 0.001 0 0 +8.20923 0.001 0 0 0 0.001 0 0 +8.21167 0.001 0 0 0 0.001 0 0 +8.21411 0.001 0 0 0 0.001 0 0 +8.21655 0.001 0 0 0 0.001 0 0 +8.21899 0.001 0 0 0 0.001 0 0 +8.22144 0.001 0 0 0 0.001 0 0 +8.22388 0.001 0 0 0 0.001 0 0 +8.22632 0.001 0 0 0 0.001 0 0 +8.22876 0.001 0 0 0 0.001 0 0 +8.2312 0.001 0 0 0 0.001 0 0 +8.23364 0.001 0 0 0 0.001 0 0 +8.23608 0.001 0 0 0 0.001 0 0 +8.23853 0.001 0 0 0 0.001 0 0 +8.24097 0.001 0 0 0 0.001 0 0 +8.24341 0.001 0 0 0 0.001 0 0 +8.24585 0.001 0 0 0 0.001 0 0 +8.24829 0.001 0 0 0 0.001 0 0 +8.25073 0.001 0 0 0 0.001 0 0 +8.25317 0.001 0 0 0 0.001 0 0 +8.25562 0.001 0 0 0 0.001 0 0 +8.25806 0.001 0 0 0 0.001 0 0 +8.2605 0.001 0 0 0 0.001 0 0 +8.26294 0.001 0 0 0 0.001 0 0 +8.26538 0.001 0 0 0 0.001 0 0 +8.26782 0.001 0 0 0 0.001 0 0 +8.27026 0.001 0 0 0 0.001 0 0 +8.27271 0.001 0 0 0 0.001 0 0 +8.27515 0.001 0 0 0 0.001 0 0 +8.27759 0.001 0 0 0 0.001 0 0 +8.28003 0.001 0 0 0 0.001 0 0 +8.28247 0.001 0 0 0 0.001 0 0 +8.28491 0.001 0 0 0 0.001 0 0 +8.28735 0.001 0 0 0 0.001 0 0 +8.28979 0.001 0 0 0 0.001 0 0 +8.29224 0.001 0 0 0 0.001 0 0 +8.29468 0.001 0 0 0 0.001 0 0 +8.29712 0.001 0 0 0 0.001 0 0 +8.29956 0.001 0 0 0 0.001 0 0 +8.302 0.001 0 0 0 0.001 0 0 +8.30444 0.001 0 0 0 0.001 0 0 +8.30688 0.001 0 0 0 0.001 0 0 +8.30933 0.001 0 0 0 0.001 0 0 +8.31177 0.001 0 0 0 0.001 0 0 +8.31421 0.001 0 0 0 0.001 0 0 +8.31665 0.001 0 0 0 0.001 0 0 +8.31909 0.001 0 0 0 0.001 0 0 +8.32153 0.001 0 0 0 0.001 0 0 +8.32397 0.001 0 0 0 0.001 0 0 +8.32642 0.001 0 0 0 0.001 0 0 +8.32886 0.001 0 0 0 0.001 0 0 +8.3313 0.001 0 0 0 0.001 0 0 +8.33374 0.001 0 0 0 0.001 0 0 +8.33618 0.001 0 0 0 0.001 0 0 +8.33862 0.001 0 0 0 0.001 0 0 +8.34106 0.001 0 0 0 0.001 0 0 +8.34351 0.001 0 0 0 0.001 0 0 +8.34595 0.001 0 0 0 0.001 0 0 +8.34839 0.001 0 0 0 0.001 0 0 +8.35083 0.001 0 0 0 0.001 0 0 +8.35327 0.001 0 0 0 0.001 0 0 +8.35571 0.001 0 0 0 0.001 0 0 +8.35815 0.001 0 0 0 0.001 0 0 +8.3606 0.001 0 0 0 0.001 0 0 +8.36304 0.001 0 0 0 0.001 0 0 +8.36548 0.001 0 0 0 0.001 0 0 +8.36792 0.001 0 0 0 0.001 0 0 +8.37036 0.001 0 0 0 0.001 0 0 +8.3728 0.001 0 0 0 0.001 0 0 +8.37524 0.001 0 0 0 0.001 0 0 +8.37769 0.001 0 0 0 0.001 0 0 +8.38013 0.001 0 0 0 0.001 0 0 +8.38257 0.001 0 0 0 0.001 0 0 +8.38501 0.001 0 0 0 0.001 0 0 +8.38745 0.001 0 0 0 0.001 0 0 +8.38989 0.001 0 0 0 0.001 0 0 +8.39233 0.001 0 0 0 0.001 0 0 +8.39478 0.001 0 0 0 0.001 0 0 +8.39722 0.001 0 0 0 0.001 0 0 +8.39966 0.001 0 0 0 0.001 0 0 +8.4021 0.001 0 0 0 0.001 0 0 +8.40454 0.001 0 0 0 0.001 0 0 +8.40698 0.001 0 0 0 0.001 0 0 +8.40942 0.001 0 0 0 0.001 0 0 +8.41187 0.001 0 0 0 0.001 0 0 +8.41431 0.001 0 0 0 0.001 0 0 +8.41675 0.001 0 0 0 0.001 0 0 +8.41919 0.001 0 0 0 0.001 0 0 +8.42163 0.001 0 0 0 0.001 0 0 +8.42407 0.001 0 0 0 0.001 0 0 +8.42651 0.001 0 0 0 0.001 0 0 +8.42896 0.001 0 0 0 0.001 0 0 +8.4314 0.001 0 0 0 0.001 0 0 +8.43384 0.001 0 0 0 0.001 0 0 +8.43628 0.001 0 0 0 0.001 0 0 +8.43872 0.001 0 0 0 0.001 0 0 +8.44116 0.001 0 0 0 0.001 0 0 +8.4436 0.001 0 0 0 0.001 0 0 +8.44604 0.001 0 0 0 0.001 0 0 +8.44849 0.001 0 0 0 0.001 0 0 +8.45093 0.001 0 0 0 0.001 0 0 +8.45337 0.001 0 0 0 0.001 0 0 +8.45581 0.001 0 0 0 0.001 0 0 +8.45825 0.001 0 0 0 0.001 0 0 +8.46069 0.001 0 0 0 0.001 0 0 +8.46313 0.001 0 0 0 0.001 0 0 +8.46558 0.001 0 0 0 0.001 0 0 +8.46802 0.001 0 0 0 0.001 0 0 +8.47046 0.001 0 0 0 0.001 0 0 +8.4729 0.001 0 0 0 0.001 0 0 +8.47534 0.001 0 0 0 0.001 0 0 +8.47778 0.001 0 0 0 0.001 0 0 +8.48022 0.001 0 0 0 0.001 0 0 +8.48267 0.001 0 0 0 0.001 0 0 +8.48511 0.001 0 0 0 0.001 0 0 +8.48755 0.001 0 0 0 0.001 0 0 +8.48999 0.001 0 0 0 0.001 0 0 +8.49243 0.001 0 0 0 0.001 0 0 +8.49487 0.001 0 0 0 0.001 0 0 +8.49731 0.001 0 0 0 0.001 0 0 +8.49976 0.001 0 0 0 0.001 0 0 +8.5022 0.001 0 0 0 0.001 0 0 +8.50464 0.001 0 0 0 0.001 0 0 +8.50708 0.001 0 0 0 0.001 0 0 +8.50952 0.001 0 0 0 0.001 0 0 +8.51196 0.001 0 0 0 0.001 0 0 +8.5144 0.001 0 0 0 0.001 0 0 +8.51685 0.001 0 0 0 0.001 0 0 +8.51929 0.001 0 0 0 0.001 0 0 +8.52173 0.001 0 0 0 0.001 0 0 +8.52417 0.001 0 0 0 0.001 0 0 +8.52661 0.001 0 0 0 0.001 0 0 +8.52905 0.001 0 0 0 0.001 0 0 +8.53149 0.001 0 0 0 0.001 0 0 +8.53394 0.001 0 0 0 0.001 0 0 +8.53638 0.001 0 0 0 0.001 0 0 +8.53882 0.001 0 0 0 0.001 0 0 +8.54126 0.001 0 0 0 0.001 0 0 +8.5437 0.001 0 0 0 0.001 0 0 +8.54614 0.001 0 0 0 0.001 0 0 +8.54858 0.001 0 0 0 0.001 0 0 +8.55103 0.001 0 0 0 0.001 0 0 +8.55347 0.001 0 0 0 0.001 0 0 +8.55591 0.001 0 0 0 0.001 0 0 +8.55835 0.001 0 0 0 0.001 0 0 +8.56079 0.001 0 0 0 0.001 0 0 +8.56323 0.001 0 0 0 0.001 0 0 +8.56567 0.001 0 0 0 0.001 0 0 +8.56812 0.001 0 0 0 0.001 0 0 +8.57056 0.001 0 0 0 0.001 0 0 +8.573 0.001 0 0 0 0.001 0 0 +8.57544 0.001 0 0 0 0.001 0 0 +8.57788 0.001 0 0 0 0.001 0 0 +8.58032 0.001 0 0 0 0.001 0 0 +8.58276 0.001 0 0 0 0.001 0 0 +8.58521 0.001 0 0 0 0.001 0 0 +8.58765 0.001 0 0 0 0.001 0 0 +8.59009 0.001 0 0 0 0.001 0 0 +8.59253 0.001 0 0 0 0.001 0 0 +8.59497 0.001 0 0 0 0.001 0 0 +8.59741 0.001 0 0 0 0.001 0 0 +8.59985 0.001 0 0 0 0.001 0 0 +8.60229 0.001 0 0 0 0.001 0 0 +8.60474 0.001 0 0 0 0.001 0 0 +8.60718 0.001 0 0 0 0.001 0 0 +8.60962 0.001 0 0 0 0.001 0 0 +8.61206 0.001 0 0 0 0.001 0 0 +8.6145 0.001 0 0 0 0.001 0 0 +8.61694 0.001 0 0 0 0.001 0 0 +8.61938 0.001 0 0 0 0.001 0 0 +8.62183 0.001 0 0 0 0.001 0 0 +8.62427 0.001 0 0 0 0.001 0 0 +8.62671 0.001 0 0 0 0.001 0 0 +8.62915 0.001 0 0 0 0.001 0 0 +8.63159 0.001 0 0 0 0.001 0 0 +8.63403 0.001 0 0 0 0.001 0 0 +8.63647 0.001 0 0 0 0.001 0 0 +8.63892 0.001 0 0 0 0.001 0 0 +8.64136 0.001 0 0 0 0.001 0 0 +8.6438 0.001 0 0 0 0.001 0 0 +8.64624 0.001 0 0 0 0.001 0 0 +8.64868 0.001 0 0 0 0.001 0 0 +8.65112 0.001 0 0 0 0.001 0 0 +8.65356 0.001 0 0 0 0.001 0 0 +8.65601 0.001 0 0 0 0.001 0 0 +8.65845 0.001 0 0 0 0.001 0 0 +8.66089 0.001 0 0 0 0.001 0 0 +8.66333 0.001 0 0 0 0.001 0 0 +8.66577 0.001 0 0 0 0.001 0 0 +8.66821 0.001 0 0 0 0.001 0 0 +8.67065 0.001 0 0 0 0.001 0 0 +8.6731 0.001 0 0 0 0.001 0 0 +8.67554 0.001 0 0 0 0.001 0 0 +8.67798 0.001 0 0 0 0.001 0 0 +8.68042 0.001 0 0 0 0.001 0 0 +8.68286 0.001 0 0 0 0.001 0 0 +8.6853 0.001 0 0 0 0.001 0 0 +8.68774 0.001 0 0 0 0.001 0 0 +8.69019 0.001 0 0 0 0.001 0 0 +8.69263 0.001 0 0 0 0.001 0 0 +8.69507 0.001 0 0 0 0.001 0 0 +8.69751 0.001 0 0 0 0.001 0 0 +8.69995 0.001 0 0 0 0.001 0 0 +8.70239 0.001 0 0 0 0.001 0 0 +8.70483 0.001 0 0 0 0.001 0 0 +8.70728 0.001 0 0 0 0.001 0 0 +8.70972 0.001 0 0 0 0.001 0 0 +8.71216 0.001 0 0 0 0.001 0 0 +8.7146 0.001 0 0 0 0.001 0 0 +8.71704 0.001 0 0 0 0.001 0 0 +8.71948 0.001 0 0 0 0.001 0 0 +8.72192 0.001 0 0 0 0.001 0 0 +8.72437 0.001 0 0 0 0.001 0 0 +8.72681 0.001 0 0 0 0.001 0 0 +8.72925 0.001 0 0 0 0.001 0 0 +8.73169 0.001 0 0 0 0.001 0 0 +8.73413 0.001 0 0 0 0.001 0 0 +8.73657 0.001 0 0 0 0.001 0 0 +8.73901 0.001 0 0 0 0.001 0 0 +8.74146 0.001 0 0 0 0.001 0 0 +8.7439 0.001 0 0 0 0.001 0 0 +8.74634 0.001 0 0 0 0.001 0 0 +8.74878 0.001 0 0 0 0.001 0 0 +8.75122 0.001 0 0 0 0.001 0 0 +8.75366 0.001 0 0 0 0.001 0 0 +8.7561 0.001 0 0 0 0.001 0 0 +8.75854 0.001 0 0 0 0.001 0 0 +8.76099 0.001 0 0 0 0.001 0 0 +8.76343 0.001 0 0 0 0.001 0 0 +8.76587 0.001 0 0 0 0.001 0 0 +8.76831 0.001 0 0 0 0.001 0 0 +8.77075 0.001 0 0 0 0.001 0 0 +8.77319 0.001 0 0 0 0.001 0 0 +8.77563 0.001 0 0 0 0.001 0 0 +8.77808 0.001 0 0 0 0.001 0 0 +8.78052 0.001 0 0 0 0.001 0 0 +8.78296 0.001 0 0 0 0.001 0 0 +8.7854 0.001 0 0 0 0.001 0 0 +8.78784 0.001 0 0 0 0.001 0 0 +8.79028 0.001 0 0 0 0.001 0 0 +8.79272 0.001 0 0 0 0.001 0 0 +8.79517 0.001 0 0 0 0.001 0 0 +8.79761 0.001 0 0 0 0.001 0 0 +8.80005 0.001 0 0 0 0.001 0 0 +8.80249 0.001 0 0 0 0.001 0 0 +8.80493 0.001 0 0 0 0.001 0 0 +8.80737 0.001 0 0 0 0.001 0 0 +8.80981 0.001 0 0 0 0.001 0 0 +8.81226 0.001 0 0 0 0.001 0 0 +8.8147 0.001 0 0 0 0.001 0 0 +8.81714 0.001 0 0 0 0.001 0 0 +8.81958 0.001 0 0 0 0.001 0 0 +8.82202 0.001 0 0 0 0.001 0 0 +8.82446 0.001 0 0 0 0.001 0 0 +8.8269 0.001 0 0 0 0.001 0 0 +8.82935 0.001 0 0 0 0.001 0 0 +8.83179 0.001 0 0 0 0.001 0 0 +8.83423 0.001 0 0 0 0.001 0 0 +8.83667 0.001 0 0 0 0.001 0 0 +8.83911 0.001 0 0 0 0.001 0 0 +8.84155 0.001 0 0 0 0.001 0 0 +8.84399 0.001 0 0 0 0.001 0 0 +8.84644 0.001 0 0 0 0.001 0 0 +8.84888 0.001 0 0 0 0.001 0 0 +8.85132 0.001 0 0 0 0.001 0 0 +8.85376 0.001 0 0 0 0.001 0 0 +8.8562 0.001 0 0 0 0.001 0 0 +8.85864 0.001 0 0 0 0.001 0 0 +8.86108 0.001 0 0 0 0.001 0 0 +8.86353 0.001 0 0 0 0.001 0 0 +8.86597 0.001 0 0 0 0.001 0 0 +8.86841 0.001 0 0 0 0.001 0 0 +8.87085 0.001 0 0 0 0.001 0 0 +8.87329 0.001 0 0 0 0.001 0 0 +8.87573 0.001 0 0 0 0.001 0 0 +8.87817 0.001 0 0 0 0.001 0 0 +8.88062 0.001 0 0 0 0.001 0 0 +8.88306 0.001 0 0 0 0.001 0 0 +8.8855 0.001 0 0 0 0.001 0 0 +8.88794 0.001 0 0 0 0.001 0 0 +8.89038 0.001 0 0 0 0.001 0 0 +8.89282 0.001 0 0 0 0.001 0 0 +8.89526 0.001 0 0 0 0.001 0 0 +8.89771 0.001 0 0 0 0.001 0 0 +8.90015 0.001 0 0 0 0.001 0 0 +8.90259 0.001 0 0 0 0.001 0 0 +8.90503 0.001 0 0 0 0.001 0 0 +8.90747 0.001 0 0 0 0.001 0 0 +8.90991 0.001 0 0 0 0.001 0 0 +8.91235 0.001 0 0 0 0.001 0 0 +8.91479 0.001 0 0 0 0.001 0 0 +8.91724 0.001 0 0 0 0.001 0 0 +8.91968 0.001 0 0 0 0.001 0 0 +8.92212 0.001 0 0 0 0.001 0 0 +8.92456 0.001 0 0 0 0.001 0 0 +8.927 0.001 0 0 0 0.001 0 0 +8.92944 0.001 0 0 0 0.001 0 0 +8.93188 0.001 0 0 0 0.001 0 0 +8.93433 0.001 0 0 0 0.001 0 0 +8.93677 0.001 0 0 0 0.001 0 0 +8.93921 0.001 0 0 0 0.001 0 0 +8.94165 0.001 0 0 0 0.001 0 0 +8.94409 0.001 0 0 0 0.001 0 0 +8.94653 0.001 0 0 0 0.001 0 0 +8.94897 0.001 0 0 0 0.001 0 0 +8.95142 0.001 0 0 0 0.001 0 0 +8.95386 0.001 0 0 0 0.001 0 0 +8.9563 0.001 0 0 0 0.001 0 0 +8.95874 0.001 0 0 0 0.001 0 0 +8.96118 0.001 0 0 0 0.001 0 0 +8.96362 0.001 0 0 0 0.001 0 0 +8.96606 0.001 0 0 0 0.001 0 0 +8.96851 0.001 0 0 0 0.001 0 0 +8.97095 0.001 0 0 0 0.001 0 0 +8.97339 0.001 0 0 0 0.001 0 0 +8.97583 0.001 0 0 0 0.001 0 0 +8.97827 0.001 0 0 0 0.001 0 0 +8.98071 0.001 0 0 0 0.001 0 0 +8.98315 0.001 0 0 0 0.001 0 0 +8.9856 0.001 0 0 0 0.001 0 0 +8.98804 0.001 0 0 0 0.001 0 0 +8.99048 0.001 0 0 0 0.001 0 0 +8.99292 0.001 0 0 0 0.001 0 0 +8.99536 0.001 0 0 0 0.001 0 0 +8.9978 0.001 0 0 0 0.001 0 0 +9.00024 0.001 0 0 0 0.001 0 0 +9.00269 0.001 0 0 0 0.001 0 0 +9.00513 0.001 0 0 0 0.001 0 0 +9.00757 0.001 0 0 0 0.001 0 0 +9.01001 0.001 0 0 0 0.001 0 0 +9.01245 0.001 0 0 0 0.001 0 0 +9.01489 0.001 0 0 0 0.001 0 0 +9.01733 0.001 0 0 0 0.001 0 0 +9.01978 0.001 0 0 0 0.001 0 0 +9.02222 0.001 0 0 0 0.001 0 0 +9.02466 0.001 0 0 0 0.001 0 0 +9.0271 0.001 0 0 0 0.001 0 0 +9.02954 0.001 0 0 0 0.001 0 0 +9.03198 0.001 0 0 0 0.001 0 0 +9.03442 0.001 0 0 0 0.001 0 0 +9.03687 0.001 0 0 0 0.001 0 0 +9.03931 0.001 0 0 0 0.001 0 0 +9.04175 0.001 0 0 0 0.001 0 0 +9.04419 0.001 0 0 0 0.001 0 0 +9.04663 0.001 0 0 0 0.001 0 0 +9.04907 0.001 0 0 0 0.001 0 0 +9.05151 0.001 0 0 0 0.001 0 0 +9.05396 0.001 0 0 0 0.001 0 0 +9.0564 0.001 0 0 0 0.001 0 0 +9.05884 0.001 0 0 0 0.001 0 0 +9.06128 0.001 0 0 0 0.001 0 0 +9.06372 0.001 0 0 0 0.001 0 0 +9.06616 0.001 0 0 0 0.001 0 0 +9.0686 0.001 0 0 0 0.001 0 0 +9.07104 0.001 0 0 0 0.001 0 0 +9.07349 0.001 0 0 0 0.001 0 0 +9.07593 0.001 0 0 0 0.001 0 0 +9.07837 0.001 0 0 0 0.001 0 0 +9.08081 0.001 0 0 0 0.001 0 0 +9.08325 0.001 0 0 0 0.001 0 0 +9.08569 0.001 0 0 0 0.001 0 0 +9.08813 0.001 0 0 0 0.001 0 0 +9.09058 0.001 0 0 0 0.001 0 0 +9.09302 0.001 0 0 0 0.001 0 0 +9.09546 0.001 0 0 0 0.001 0 0 +9.0979 0.001 0 0 0 0.001 0 0 +9.10034 0.001 0 0 0 0.001 0 0 +9.10278 0.001 0 0 0 0.001 0 0 +9.10522 0.001 0 0 0 0.001 0 0 +9.10767 0.001 0 0 0 0.001 0 0 +9.11011 0.001 0 0 0 0.001 0 0 +9.11255 0.001 0 0 0 0.001 0 0 +9.11499 0.001 0 0 0 0.001 0 0 +9.11743 0.001 0 0 0 0.001 0 0 +9.11987 0.001 0 0 0 0.001 0 0 +9.12231 0.001 0 0 0 0.001 0 0 +9.12476 0.001 0 0 0 0.001 0 0 +9.1272 0.001 0 0 0 0.001 0 0 +9.12964 0.001 0 0 0 0.001 0 0 +9.13208 0.001 0 0 0 0.001 0 0 +9.13452 0.001 0 0 0 0.001 0 0 +9.13696 0.001 0 0 0 0.001 0 0 +9.1394 0.001 0 0 0 0.001 0 0 +9.14185 0.001 0 0 0 0.001 0 0 +9.14429 0.001 0 0 0 0.001 0 0 +9.14673 0.001 0 0 0 0.001 0 0 +9.14917 0.001 0 0 0 0.001 0 0 +9.15161 0.001 0 0 0 0.001 0 0 +9.15405 0.001 0 0 0 0.001 0 0 +9.15649 0.001 0 0 0 0.001 0 0 +9.15894 0.001 0 0 0 0.001 0 0 +9.16138 0.001 0 0 0 0.001 0 0 +9.16382 0.001 0 0 0 0.001 0 0 +9.16626 0.001 0 0 0 0.001 0 0 +9.1687 0.001 0 0 0 0.001 0 0 +9.17114 0.001 0 0 0 0.001 0 0 +9.17358 0.001 0 0 0 0.001 0 0 +9.17603 0.001 0 0 0 0.001 0 0 +9.17847 0.001 0 0 0 0.001 0 0 +9.18091 0.001 0 0 0 0.001 0 0 +9.18335 0.001 0 0 0 0.001 0 0 +9.18579 0.001 0 0 0 0.001 0 0 +9.18823 0.001 0 0 0 0.001 0 0 +9.19067 0.001 0 0 0 0.001 0 0 +9.19312 0.001 0 0 0 0.001 0 0 +9.19556 0.001 0 0 0 0.001 0 0 +9.198 0.001 0 0 0 0.001 0 0 +9.20044 0.001 0 0 0 0.001 0 0 +9.20288 0.001 0 0 0 0.001 0 0 +9.20532 0.001 0 0 0 0.001 0 0 +9.20776 0.001 0 0 0 0.001 0 0 +9.21021 0.001 0 0 0 0.001 0 0 +9.21265 0.001 0 0 0 0.001 0 0 +9.21509 0.001 0 0 0 0.001 0 0 +9.21753 0.001 0 0 0 0.001 0 0 +9.21997 0.001 0 0 0 0.001 0 0 +9.22241 0.001 0 0 0 0.001 0 0 +9.22485 0.001 0 0 0 0.001 0 0 +9.22729 0.001 0 0 0 0.001 0 0 +9.22974 0.001 0 0 0 0.001 0 0 +9.23218 0.001 0 0 0 0.001 0 0 +9.23462 0.001 0 0 0 0.001 0 0 +9.23706 0.001 0 0 0 0.001 0 0 +9.2395 0.001 0 0 0 0.001 0 0 +9.24194 0.001 0 0 0 0.001 0 0 +9.24438 0.001 0 0 0 0.001 0 0 +9.24683 0.001 0 0 0 0.001 0 0 +9.24927 0.001 0 0 0 0.001 0 0 +9.25171 0.001 0 0 0 0.001 0 0 +9.25415 0.001 0 0 0 0.001 0 0 +9.25659 0.001 0 0 0 0.001 0 0 +9.25903 0.001 0 0 0 0.001 0 0 +9.26147 0.001 0 0 0 0.001 0 0 +9.26392 0.001 0 0 0 0.001 0 0 +9.26636 0.001 0 0 0 0.001 0 0 +9.2688 0.001 0 0 0 0.001 0 0 +9.27124 0.001 0 0 0 0.001 0 0 +9.27368 0.001 0 0 0 0.001 0 0 +9.27612 0.001 0 0 0 0.001 0 0 +9.27856 0.001 0 0 0 0.001 0 0 +9.28101 0.001 0 0 0 0.001 0 0 +9.28345 0.001 0 0 0 0.001 0 0 +9.28589 0.001 0 0 0 0.001 0 0 +9.28833 0.001 0 0 0 0.001 0 0 +9.29077 0.001 0 0 0 0.001 0 0 +9.29321 0.001 0 0 0 0.001 0 0 +9.29565 0.001 0 0 0 0.001 0 0 +9.2981 0.001 0 0 0 0.001 0 0 +9.30054 0.001 0 0 0 0.001 0 0 +9.30298 0.001 0 0 0 0.001 0 0 +9.30542 0.001 0 0 0 0.001 0 0 +9.30786 0.001 0 0 0 0.001 0 0 +9.3103 0.001 0 0 0 0.001 0 0 +9.31274 0.001 0 0 0 0.001 0 0 +9.31519 0.001 0 0 0 0.001 0 0 +9.31763 0.001 0 0 0 0.001 0 0 +9.32007 0.001 0 0 0 0.001 0 0 +9.32251 0.001 0 0 0 0.001 0 0 +9.32495 0.001 0 0 0 0.001 0 0 +9.32739 0.001 0 0 0 0.001 0 0 +9.32983 0.001 0 0 0 0.001 0 0 +9.33228 0.001 0 0 0 0.001 0 0 +9.33472 0.001 0 0 0 0.001 0 0 +9.33716 0.001 0 0 0 0.001 0 0 +9.3396 0.001 0 0 0 0.001 0 0 +9.34204 0.001 0 0 0 0.001 0 0 +9.34448 0.001 0 0 0 0.001 0 0 +9.34692 0.001 0 0 0 0.001 0 0 +9.34937 0.001 0 0 0 0.001 0 0 +9.35181 0.001 0 0 0 0.001 0 0 +9.35425 0.001 0 0 0 0.001 0 0 +9.35669 0.001 0 0 0 0.001 0 0 +9.35913 0.001 0 0 0 0.001 0 0 +9.36157 0.001 0 0 0 0.001 0 0 +9.36401 0.001 0 0 0 0.001 0 0 +9.36646 0.001 0 0 0 0.001 0 0 +9.3689 0.001 0 0 0 0.001 0 0 +9.37134 0.001 0 0 0 0.001 0 0 +9.37378 0.001 0 0 0 0.001 0 0 +9.37622 0.001 0 0 0 0.001 0 0 +9.37866 0.001 0 0 0 0.001 0 0 +9.3811 0.001 0 0 0 0.001 0 0 +9.38354 0.001 0 0 0 0.001 0 0 +9.38599 0.001 0 0 0 0.001 0 0 +9.38843 0.001 0 0 0 0.001 0 0 +9.39087 0.001 0 0 0 0.001 0 0 +9.39331 0.001 0 0 0 0.001 0 0 +9.39575 0.001 0 0 0 0.001 0 0 +9.39819 0.001 0 0 0 0.001 0 0 +9.40063 0.001 0 0 0 0.001 0 0 +9.40308 0.001 0 0 0 0.001 0 0 +9.40552 0.001 0 0 0 0.001 0 0 +9.40796 0.001 0 0 0 0.001 0 0 +9.4104 0.001 0 0 0 0.001 0 0 +9.41284 0.001 0 0 0 0.001 0 0 +9.41528 0.001 0 0 0 0.001 0 0 +9.41772 0.001 0 0 0 0.001 0 0 +9.42017 0.001 0 0 0 0.001 0 0 +9.42261 0.001 0 0 0 0.001 0 0 +9.42505 0.001 0 0 0 0.001 0 0 +9.42749 0.001 0 0 0 0.001 0 0 +9.42993 0.001 0 0 0 0.001 0 0 +9.43237 0.001 0 0 0 0.001 0 0 +9.43481 0.001 0 0 0 0.001 0 0 +9.43726 0.001 0 0 0 0.001 0 0 +9.4397 0.001 0 0 0 0.001 0 0 +9.44214 0.001 0 0 0 0.001 0 0 +9.44458 0.001 0 0 0 0.001 0 0 +9.44702 0.001 0 0 0 0.001 0 0 +9.44946 0.001 0 0 0 0.001 0 0 +9.4519 0.001 0 0 0 0.001 0 0 +9.45435 0.001 0 0 0 0.001 0 0 +9.45679 0.001 0 0 0 0.001 0 0 +9.45923 0.001 0 0 0 0.001 0 0 +9.46167 0.001 0 0 0 0.001 0 0 +9.46411 0.001 0 0 0 0.001 0 0 +9.46655 0.001 0 0 0 0.001 0 0 +9.46899 0.001 0 0 0 0.001 0 0 +9.47144 0.001 0 0 0 0.001 0 0 +9.47388 0.001 0 0 0 0.001 0 0 +9.47632 0.001 0 0 0 0.001 0 0 +9.47876 0.001 0 0 0 0.001 0 0 +9.4812 0.001 0 0 0 0.001 0 0 +9.48364 0.001 0 0 0 0.001 0 0 +9.48608 0.001 0 0 0 0.001 0 0 +9.48853 0.001 0 0 0 0.001 0 0 +9.49097 0.001 0 0 0 0.001 0 0 +9.49341 0.001 0 0 0 0.001 0 0 +9.49585 0.001 0 0 0 0.001 0 0 +9.49829 0.001 0 0 0 0.001 0 0 +9.50073 0.001 0 0 0 0.001 0 0 +9.50317 0.001 0 0 0 0.001 0 0 +9.50562 0.001 0 0 0 0.001 0 0 +9.50806 0.001 0 0 0 0.001 0 0 +9.5105 0.001 0 0 0 0.001 0 0 +9.51294 0.001 0 0 0 0.001 0 0 +9.51538 0.001 0 0 0 0.001 0 0 +9.51782 0.001 0 0 0 0.001 0 0 +9.52026 0.001 0 0 0 0.001 0 0 +9.52271 0.001 0 0 0 0.001 0 0 +9.52515 0.001 0 0 0 0.001 0 0 +9.52759 0.001 0 0 0 0.001 0 0 +9.53003 0.001 0 0 0 0.001 0 0 +9.53247 0.001 0 0 0 0.001 0 0 +9.53491 0.001 0 0 0 0.001 0 0 +9.53735 0.001 0 0 0 0.001 0 0 +9.53979 0.001 0 0 0 0.001 0 0 +9.54224 0.001 0 0 0 0.001 0 0 +9.54468 0.001 0 0 0 0.001 0 0 +9.54712 0.001 0 0 0 0.001 0 0 +9.54956 0.001 0 0 0 0.001 0 0 +9.552 0.001 0 0 0 0.001 0 0 +9.55444 0.001 0 0 0 0.001 0 0 +9.55688 0.001 0 0 0 0.001 0 0 +9.55933 0.001 0 0 0 0.001 0 0 +9.56177 0.001 0 0 0 0.001 0 0 +9.56421 0.001 0 0 0 0.001 0 0 +9.56665 0.001 0 0 0 0.001 0 0 +9.56909 0.001 0 0 0 0.001 0 0 +9.57153 0.001 0 0 0 0.001 0 0 +9.57397 0.001 0 0 0 0.001 0 0 +9.57642 0.001 0 0 0 0.001 0 0 +9.57886 0.001 0 0 0 0.001 0 0 +9.5813 0.001 0 0 0 0.001 0 0 +9.58374 0.001 0 0 0 0.001 0 0 +9.58618 0.001 0 0 0 0.001 0 0 +9.58862 0.001 0 0 0 0.001 0 0 +9.59106 0.001 0 0 0 0.001 0 0 +9.59351 0.001 0 0 0 0.001 0 0 +9.59595 0.001 0 0 0 0.001 0 0 +9.59839 0.001 0 0 0 0.001 0 0 +9.60083 0.001 0 0 0 0.001 0 0 +9.60327 0.001 0 0 0 0.001 0 0 +9.60571 0.001 0 0 0 0.001 0 0 +9.60815 0.001 0 0 0 0.001 0 0 +9.6106 0.001 0 0 0 0.001 0 0 +9.61304 0.001 0 0 0 0.001 0 0 +9.61548 0.001 0 0 0 0.001 0 0 +9.61792 0.001 0 0 0 0.001 0 0 +9.62036 0.001 0 0 0 0.001 0 0 +9.6228 0.001 0 0 0 0.001 0 0 +9.62524 0.001 0 0 0 0.001 0 0 +9.62769 0.001 0 0 0 0.001 0 0 +9.63013 0.001 0 0 0 0.001 0 0 +9.63257 0.001 0 0 0 0.001 0 0 +9.63501 0.001 0 0 0 0.001 0 0 +9.63745 0.001 0 0 0 0.001 0 0 +9.63989 0.001 0 0 0 0.001 0 0 +9.64233 0.001 0 0 0 0.001 0 0 +9.64478 0.001 0 0 0 0.001 0 0 +9.64722 0.001 0 0 0 0.001 0 0 +9.64966 0.001 0 0 0 0.001 0 0 +9.6521 0.001 0 0 0 0.001 0 0 +9.65454 0.001 0 0 0 0.001 0 0 +9.65698 0.001 0 0 0 0.001 0 0 +9.65942 0.001 0 0 0 0.001 0 0 +9.66187 0.001 0 0 0 0.001 0 0 +9.66431 0.001 0 0 0 0.001 0 0 +9.66675 0.001 0 0 0 0.001 0 0 +9.66919 0.001 0 0 0 0.001 0 0 +9.67163 0.001 0 0 0 0.001 0 0 +9.67407 0.001 0 0 0 0.001 0 0 +9.67651 0.001 0 0 0 0.001 0 0 +9.67896 0.001 0 0 0 0.001 0 0 +9.6814 0.001 0 0 0 0.001 0 0 +9.68384 0.001 0 0 0 0.001 0 0 +9.68628 0.001 0 0 0 0.001 0 0 +9.68872 0.001 0 0 0 0.001 0 0 +9.69116 0.001 0 0 0 0.001 0 0 +9.6936 0.001 0 0 0 0.001 0 0 +9.69604 0.001 0 0 0 0.001 0 0 +9.69849 0.001 0 0 0 0.001 0 0 +9.70093 0.001 0 0 0 0.001 0 0 +9.70337 0.001 0 0 0 0.001 0 0 +9.70581 0.001 0 0 0 0.001 0 0 +9.70825 0.001 0 0 0 0.001 0 0 +9.71069 0.001 0 0 0 0.001 0 0 +9.71313 0.001 0 0 0 0.001 0 0 +9.71558 0.001 0 0 0 0.001 0 0 +9.71802 0.001 0 0 0 0.001 0 0 +9.72046 0.001 0 0 0 0.001 0 0 +9.7229 0.001 0 0 0 0.001 0 0 +9.72534 0.001 0 0 0 0.001 0 0 +9.72778 0.001 0 0 0 0.001 0 0 +9.73022 0.001 0 0 0 0.001 0 0 +9.73267 0.001 0 0 0 0.001 0 0 +9.73511 0.001 0 0 0 0.001 0 0 +9.73755 0.001 0 0 0 0.001 0 0 +9.73999 0.001 0 0 0 0.001 0 0 +9.74243 0.001 0 0 0 0.001 0 0 +9.74487 0.001 0 0 0 0.001 0 0 +9.74731 0.001 0 0 0 0.001 0 0 +9.74976 0.001 0 0 0 0.001 0 0 +9.7522 0.001 0 0 0 0.001 0 0 +9.75464 0.001 0 0 0 0.001 0 0 +9.75708 0.001 0 0 0 0.001 0 0 +9.75952 0.001 0 0 0 0.001 0 0 +9.76196 0.001 0 0 0 0.001 0 0 +9.7644 0.001 0 0 0 0.001 0 0 +9.76685 0.001 0 0 0 0.001 0 0 +9.76929 0.001 0 0 0 0.001 0 0 +9.77173 0.001 0 0 0 0.001 0 0 +9.77417 0.001 0 0 0 0.001 0 0 +9.77661 0.001 0 0 0 0.001 0 0 +9.77905 0.001 0 0 0 0.001 0 0 +9.78149 0.001 0 0 0 0.001 0 0 +9.78394 0.001 0 0 0 0.001 0 0 +9.78638 0.001 0 0 0 0.001 0 0 +9.78882 0.001 0 0 0 0.001 0 0 +9.79126 0.001 0 0 0 0.001 0 0 +9.7937 0.001 0 0 0 0.001 0 0 +9.79614 0.001 0 0 0 0.001 0 0 +9.79858 0.001 0 0 0 0.001 0 0 +9.80103 0.001 0 0 0 0.001 0 0 +9.80347 0.001 0 0 0 0.001 0 0 +9.80591 0.001 0 0 0 0.001 0 0 +9.80835 0.001 0 0 0 0.001 0 0 +9.81079 0.001 0 0 0 0.001 0 0 +9.81323 0.001 0 0 0 0.001 0 0 +9.81567 0.001 0 0 0 0.001 0 0 +9.81812 0.001 0 0 0 0.001 0 0 +9.82056 0.001 0 0 0 0.001 0 0 +9.823 0.001 0 0 0 0.001 0 0 +9.82544 0.001 0 0 0 0.001 0 0 +9.82788 0.001 0 0 0 0.001 0 0 +9.83032 0.001 0 0 0 0.001 0 0 +9.83276 0.001 0 0 0 0.001 0 0 +9.83521 0.001 0 0 0 0.001 0 0 +9.83765 0.001 0 0 0 0.001 0 0 +9.84009 0.001 0 0 0 0.001 0 0 +9.84253 0.001 0 0 0 0.001 0 0 +9.84497 0.001 0 0 0 0.001 0 0 +9.84741 0.001 0 0 0 0.001 0 0 +9.84985 0.001 0 0 0 0.001 0 0 +9.85229 0.001 0 0 0 0.001 0 0 +9.85474 0.001 0 0 0 0.001 0 0 +9.85718 0.001 0 0 0 0.001 0 0 +9.85962 0.001 0 0 0 0.001 0 0 +9.86206 0.001 0 0 0 0.001 0 0 +9.8645 0.001 0 0 0 0.001 0 0 +9.86694 0.001 0 0 0 0.001 0 0 +9.86938 0.001 0 0 0 0.001 0 0 +9.87183 0.001 0 0 0 0.001 0 0 +9.87427 0.001 0 0 0 0.001 0 0 +9.87671 0.001 0 0 0 0.001 0 0 +9.87915 0.001 0 0 0 0.001 0 0 +9.88159 0.001 0 0 0 0.001 0 0 +9.88403 0.001 0 0 0 0.001 0 0 +9.88647 0.001 0 0 0 0.001 0 0 +9.88892 0.001 0 0 0 0.001 0 0 +9.89136 0.001 0 0 0 0.001 0 0 +9.8938 0.001 0 0 0 0.001 0 0 +9.89624 0.001 0 0 0 0.001 0 0 +9.89868 0.001 0 0 0 0.001 0 0 +9.90112 0.001 0 0 0 0.001 0 0 +9.90356 0.001 0 0 0 0.001 0 0 +9.90601 0.001 0 0 0 0.001 0 0 +9.90845 0.001 0 0 0 0.001 0 0 +9.91089 0.001 0 0 0 0.001 0 0 +9.91333 0.001 0 0 0 0.001 0 0 +9.91577 0.001 0 0 0 0.001 0 0 +9.91821 0.001 0 0 0 0.001 0 0 +9.92065 0.001 0 0 0 0.001 0 0 +9.9231 0.001 0 0 0 0.001 0 0 +9.92554 0.001 0 0 0 0.001 0 0 +9.92798 0.001 0 0 0 0.001 0 0 +9.93042 0.001 0 0 0 0.001 0 0 +9.93286 0.001 0 0 0 0.001 0 0 +9.9353 0.001 0 0 0 0.001 0 0 +9.93774 0.001 0 0 0 0.001 0 0 +9.94019 0.001 0 0 0 0.001 0 0 +9.94263 0.001 0 0 0 0.001 0 0 +9.94507 0.001 0 0 0 0.001 0 0 +9.94751 0.001 0 0 0 0.001 0 0 +9.94995 0.001 0 0 0 0.001 0 0 +9.95239 0.001 0 0 0 0.001 0 0 +9.95483 0.001 0 0 0 0.001 0 0 +9.95728 0.001 0 0 0 0.001 0 0 +9.95972 0.001 0 0 0 0.001 0 0 +9.96216 0.001 0 0 0 0.001 0 0 +9.9646 0.001 0 0 0 0.001 0 0 +9.96704 0.001 0 0 0 0.001 0 0 +9.96948 0.001 0 0 0 0.001 0 0 +9.97192 0.001 0 0 0 0.001 0 0 +9.97437 0.001 0 0 0 0.001 0 0 +9.97681 0.001 0 0 0 0.001 0 0 +9.97925 0.001 0 0 0 0.001 0 0 +9.98169 0.001 0 0 0 0.001 0 0 +9.98413 0.001 0 0 0 0.001 0 0 +9.98657 0.001 0 0 0 0.001 0 0 +9.98901 0.001 0 0 0 0.001 0 0 +9.99146 0.001 0 0 0 0.001 0 0 +9.9939 0.001 0 0 0 0.001 0 0 +9.99634 0.001 0 0 0 0.001 0 0 +9.99878 0.001 0 0 0 0.001 0 0 diff --git a/reference/swashes_1_nx=512.csv b/reference/swashes_1_nx=512.csv new file mode 100644 index 0000000..0a2916f --- /dev/null +++ b/reference/swashes_1_nx=512.csv @@ -0,0 +1,530 @@ +############################################################################## +# Generated by SWASHES version 1.03.00, 2016-01-29 +############################################################################## +# Dimension: 1 +# Type: 3 (=Dam break) +# Domain: 1 +# Choice: 1 (=on a wet domain without friction (Stoker's solution)) +############################################################################## +# PARAMETERS OF THE SOLUTION +# +# Length of the domain: 10 meters +# Space step: 0.0195312 meters +# Number of cells: 512 +# Position of the dam: x=5 meters +# Time value: 6 seconds +############################################################################## +# +#(i-0.5)*dx h[i] u[i] topo[i] q[i] topo[i]+h[i] Fr[i]=Froude topo[i]+hc[i] +0.00976562 0.005 0 0 0 0.005 0 0 +0.0292969 0.005 0 0 0 0.005 0 0 +0.0488281 0.005 0 0 0 0.005 0 0 +0.0683594 0.005 0 0 0 0.005 0 0 +0.0878906 0.005 0 0 0 0.005 0 0 +0.107422 0.005 0 0 0 0.005 0 0 +0.126953 0.005 0 0 0 0.005 0 0 +0.146484 0.005 0 0 0 0.005 0 0 +0.166016 0.005 0 0 0 0.005 0 0 +0.185547 0.005 0 0 0 0.005 0 0 +0.205078 0.005 0 0 0 0.005 0 0 +0.224609 0.005 0 0 0 0.005 0 0 +0.244141 0.005 0 0 0 0.005 0 0 +0.263672 0.005 0 0 0 0.005 0 0 +0.283203 0.005 0 0 0 0.005 0 0 +0.302734 0.005 0 0 0 0.005 0 0 +0.322266 0.005 0 0 0 0.005 0 0 +0.341797 0.005 0 0 0 0.005 0 0 +0.361328 0.005 0 0 0 0.005 0 0 +0.380859 0.005 0 0 0 0.005 0 0 +0.400391 0.005 0 0 0 0.005 0 0 +0.419922 0.005 0 0 0 0.005 0 0 +0.439453 0.005 0 0 0 0.005 0 0 +0.458984 0.005 0 0 0 0.005 0 0 +0.478516 0.005 0 0 0 0.005 0 0 +0.498047 0.005 0 0 0 0.005 0 0 +0.517578 0.005 0 0 0 0.005 0 0 +0.537109 0.005 0 0 0 0.005 0 0 +0.556641 0.005 0 0 0 0.005 0 0 +0.576172 0.005 0 0 0 0.005 0 0 +0.595703 0.005 0 0 0 0.005 0 0 +0.615234 0.005 0 0 0 0.005 0 0 +0.634766 0.005 0 0 0 0.005 0 0 +0.654297 0.005 0 0 0 0.005 0 0 +0.673828 0.005 0 0 0 0.005 0 0 +0.693359 0.005 0 0 0 0.005 0 0 +0.712891 0.005 0 0 0 0.005 0 0 +0.732422 0.005 0 0 0 0.005 0 0 +0.751953 0.005 0 0 0 0.005 0 0 +0.771484 0.005 0 0 0 0.005 0 0 +0.791016 0.005 0 0 0 0.005 0 0 +0.810547 0.005 0 0 0 0.005 0 0 +0.830078 0.005 0 0 0 0.005 0 0 +0.849609 0.005 0 0 0 0.005 0 0 +0.869141 0.005 0 0 0 0.005 0 0 +0.888672 0.005 0 0 0 0.005 0 0 +0.908203 0.005 0 0 0 0.005 0 0 +0.927734 0.005 0 0 0 0.005 0 0 +0.947266 0.005 0 0 0 0.005 0 0 +0.966797 0.005 0 0 0 0.005 0 0 +0.986328 0.005 0 0 0 0.005 0 0 +1.00586 0.005 0 0 0 0.005 0 0 +1.02539 0.005 0 0 0 0.005 0 0 +1.04492 0.005 0 0 0 0.005 0 0 +1.06445 0.005 0 0 0 0.005 0 0 +1.08398 0.005 0 0 0 0.005 0 0 +1.10352 0.005 0 0 0 0.005 0 0 +1.12305 0.005 0 0 0 0.005 0 0 +1.14258 0.005 0 0 0 0.005 0 0 +1.16211 0.005 0 0 0 0.005 0 0 +1.18164 0.005 0 0 0 0.005 0 0 +1.20117 0.005 0 0 0 0.005 0 0 +1.2207 0.005 0 0 0 0.005 0 0 +1.24023 0.005 0 0 0 0.005 0 0 +1.25977 0.005 0 0 0 0.005 0 0 +1.2793 0.005 0 0 0 0.005 0 0 +1.29883 0.005 0 0 0 0.005 0 0 +1.31836 0.005 0 0 0 0.005 0 0 +1.33789 0.005 0 0 0 0.005 0 0 +1.35742 0.005 0 0 0 0.005 0 0 +1.37695 0.005 0 0 0 0.005 0 0 +1.39648 0.005 0 0 0 0.005 0 0 +1.41602 0.005 0 0 0 0.005 0 0 +1.43555 0.005 0 0 0 0.005 0 0 +1.45508 0.005 0 0 0 0.005 0 0 +1.47461 0.005 0 0 0 0.005 0 0 +1.49414 0.005 0 0 0 0.005 0 0 +1.51367 0.005 0 0 0 0.005 0 0 +1.5332 0.005 0 0 0 0.005 0 0 +1.55273 0.005 0 0 0 0.005 0 0 +1.57227 0.005 0 0 0 0.005 0 0 +1.5918 0.005 0 0 0 0.005 0 0 +1.61133 0.005 0 0 0 0.005 0 0 +1.63086 0.005 0 0 0 0.005 0 0 +1.65039 0.005 0 0 0 0.005 0 0 +1.66992 0.005 0 0 0 0.005 0 0 +1.68945 0.005 0 0 0 0.005 0 0 +1.70898 0.005 0 0 0 0.005 0 0 +1.72852 0.005 0 0 0 0.005 0 0 +1.74805 0.005 0 0 0 0.005 0 0 +1.76758 0.005 0 0 0 0.005 0 0 +1.78711 0.005 0 0 0 0.005 0 0 +1.80664 0.005 0 0 0 0.005 0 0 +1.82617 0.005 0 0 0 0.005 0 0 +1.8457 0.005 0 0 0 0.005 0 0 +1.86523 0.005 0 0 0 0.005 0 0 +1.88477 0.005 0 0 0 0.005 0 0 +1.9043 0.005 0 0 0 0.005 0 0 +1.92383 0.005 0 0 0 0.005 0 0 +1.94336 0.005 0 0 0 0.005 0 0 +1.96289 0.005 0 0 0 0.005 0 0 +1.98242 0.005 0 0 0 0.005 0 0 +2.00195 0.005 0 0 0 0.005 0 0 +2.02148 0.005 0 0 0 0.005 0 0 +2.04102 0.005 0 0 0 0.005 0 0 +2.06055 0.005 0 0 0 0.005 0 0 +2.08008 0.005 0 0 0 0.005 0 0 +2.09961 0.005 0 0 0 0.005 0 0 +2.11914 0.005 0 0 0 0.005 0 0 +2.13867 0.005 0 0 0 0.005 0 0 +2.1582 0.005 0 0 0 0.005 0 0 +2.17773 0.005 0 0 0 0.005 0 0 +2.19727 0.005 0 0 0 0.005 0 0 +2.2168 0.005 0 0 0 0.005 0 0 +2.23633 0.005 0 0 0 0.005 0 0 +2.25586 0.005 0 0 0 0.005 0 0 +2.27539 0.005 0 0 0 0.005 0 0 +2.29492 0.005 0 0 0 0.005 0 0 +2.31445 0.005 0 0 0 0.005 0 0 +2.33398 0.005 0 0 0 0.005 0 0 +2.35352 0.005 0 0 0 0.005 0 0 +2.37305 0.005 0 0 0 0.005 0 0 +2.39258 0.005 0 0 0 0.005 0 0 +2.41211 0.005 0 0 0 0.005 0 0 +2.43164 0.005 0 0 0 0.005 0 0 +2.45117 0.005 0 0 0 0.005 0 0 +2.4707 0.005 0 0 0 0.005 0 0 +2.49023 0.005 0 0 0 0.005 0 0 +2.50977 0.005 0 0 0 0.005 0 0 +2.5293 0.005 0 0 0 0.005 0 0 +2.54883 0.005 0 0 0 0.005 0 0 +2.56836 0.005 0 0 0 0.005 0 0 +2.58789 0.005 0 0 0 0.005 0 0 +2.60742 0.005 0 0 0 0.005 0 0 +2.62695 0.005 0 0 0 0.005 0 0 +2.64648 0.005 0 0 0 0.005 0 0 +2.66602 0.005 0 0 0 0.005 0 0 +2.68555 0.005 0 0 0 0.005 0 0 +2.70508 0.005 0 0 0 0.005 0 0 +2.72461 0.005 0 0 0 0.005 0 0 +2.74414 0.005 0 0 0 0.005 0 0 +2.76367 0.005 0 0 0 0.005 0 0 +2.7832 0.005 0 0 0 0.005 0 0 +2.80273 0.005 0 0 0 0.005 0 0 +2.82227 0.005 0 0 0 0.005 0 0 +2.8418 0.005 0 0 0 0.005 0 0 +2.86133 0.005 0 0 0 0.005 0 0 +2.88086 0.005 0 0 0 0.005 0 0 +2.90039 0.005 0 0 0 0.005 0 0 +2.91992 0.005 0 0 0 0.005 0 0 +2.93945 0.005 0 0 0 0.005 0 0 +2.95898 0.005 0 0 0 0.005 0 0 +2.97852 0.005 0 0 0 0.005 0 0 +2.99805 0.005 0 0 0 0.005 0 0 +3.01758 0.005 0 0 0 0.005 0 0 +3.03711 0.005 0 0 0 0.005 0 0 +3.05664 0.005 0 0 0 0.005 0 0 +3.07617 0.005 0 0 0 0.005 0 0 +3.0957 0.005 0 0 0 0.005 0 0 +3.11523 0.005 0 0 0 0.005 0 0 +3.13477 0.005 0 0 0 0.005 0 0 +3.1543 0.005 0 0 0 0.005 0 0 +3.17383 0.005 0 0 0 0.005 0 0 +3.19336 0.005 0 0 0 0.005 0 0 +3.21289 0.005 0 0 0 0.005 0 0 +3.23242 0.005 0 0 0 0.005 0 0 +3.25195 0.005 0 0 0 0.005 0 0 +3.27148 0.005 0 0 0 0.005 0 0 +3.29102 0.005 0 0 0 0.005 0 0 +3.31055 0.005 0 0 0 0.005 0 0 +3.33008 0.005 0 0 0 0.005 0 0 +3.34961 0.005 0 0 0 0.005 0 0 +3.36914 0.005 0 0 0 0.005 0 0 +3.38867 0.005 0 0 0 0.005 0 0 +3.4082 0.005 0 0 0 0.005 0 0 +3.42773 0.005 0 0 0 0.005 0 0 +3.44727 0.005 0 0 0 0.005 0 0 +3.4668 0.005 0 0 0 0.005 0 0 +3.48633 0.005 0 0 0 0.005 0 0 +3.50586 0.005 0 0 0 0.005 0 0 +3.52539 0.005 0 0 0 0.005 0 0 +3.54492 0.005 0 0 0 0.005 0 0 +3.56445 0.005 0 0 0 0.005 0 0 +3.58398 0.005 0 0 0 0.005 0 0 +3.60352 0.005 0 0 0 0.005 0 0 +3.62305 0.005 0 0 0 0.005 0 0 +3.64258 0.005 0 0 0 0.005 0 0 +3.66211 0.005 0 0 0 0.005 0 0 +3.68164 0.00497376 0.00116386 0 5.78874e-006 0.00497376 0.00526893 0.000150603 +3.70117 0.00492501 0.00333399 0 1.642e-005 0.00492501 0.0151679 0.000301781 +3.7207 0.00487651 0.00550413 0 2.6841e-005 0.00487651 0.0251652 0.00041877 +3.74023 0.00482825 0.00767427 0 3.70533e-005 0.00482825 0.0352621 0.000519192 +3.75977 0.00478022 0.00984441 0 4.70585e-005 0.00478022 0.0454602 0.000608885 +3.7793 0.00473244 0.0120146 0 5.68581e-005 0.00473244 0.055761 0.000690725 +3.79883 0.00468489 0.0141847 0 6.64537e-005 0.00468489 0.0661661 0.000766402 +3.81836 0.00463759 0.0163548 0 7.58469e-005 0.00463759 0.0766771 0.00083702 +3.83789 0.00459052 0.018525 0 8.50393e-005 0.00459052 0.0872955 0.000903351 +3.85742 0.0045437 0.0206951 0 9.40323e-005 0.0045437 0.0980231 0.000965966 +3.87695 0.00449711 0.0228652 0 0.000102828 0.00449711 0.108862 0.0010253 +3.89648 0.00445077 0.0250354 0 0.000111427 0.00445077 0.119813 0.00108169 +3.91602 0.00440467 0.0272055 0 0.000119831 0.00440467 0.130878 0.00113542 +3.93555 0.0043588 0.0293757 0 0.000128043 0.0043588 0.142059 0.00118672 +3.95508 0.00431318 0.0315458 0 0.000136063 0.00431318 0.153359 0.00123577 +3.97461 0.00426779 0.0337159 0 0.000143893 0.00426779 0.164778 0.00128273 +3.99414 0.00422265 0.0358861 0 0.000151534 0.00422265 0.176319 0.00132775 +4.01367 0.00417774 0.0380562 0 0.000158989 0.00417774 0.187984 0.00137095 +4.0332 0.00413308 0.0402264 0 0.000166259 0.00413308 0.199774 0.00141243 +4.05273 0.00408866 0.0423965 0 0.000173345 0.00408866 0.211692 0.00145228 +4.07227 0.00404447 0.0445666 0 0.000180248 0.00404447 0.22374 0.00149059 +4.0918 0.00400053 0.0467368 0 0.000186972 0.00400053 0.23592 0.00152743 +4.11133 0.00395682 0.0489069 0 0.000193516 0.00395682 0.248235 0.00156287 +4.13086 0.00391336 0.0510771 0 0.000199883 0.00391336 0.260685 0.00159696 +4.15039 0.00387014 0.0532472 0 0.000206074 0.00387014 0.273274 0.00162977 +4.16992 0.00382715 0.0554173 0 0.000212091 0.00382715 0.286005 0.00166134 +4.18945 0.00378441 0.0575875 0 0.000217935 0.00378441 0.298878 0.00169172 +4.20898 0.0037419 0.0597576 0 0.000223607 0.0037419 0.311898 0.00172095 +4.22852 0.00369964 0.0619277 0 0.00022911 0.00369964 0.325066 0.00174907 +4.24805 0.00365762 0.0640979 0 0.000234446 0.00365762 0.338384 0.00177612 +4.26758 0.00361583 0.066268 0 0.000239614 0.00361583 0.351856 0.00180213 +4.28711 0.00357429 0.0684382 0 0.000244618 0.00357429 0.365484 0.00182713 +4.30664 0.00353299 0.0706083 0 0.000249458 0.00353299 0.379272 0.00185115 +4.32617 0.00349192 0.0727784 0 0.000254137 0.00349192 0.39322 0.00187423 +4.3457 0.0034511 0.0749486 0 0.000258655 0.0034511 0.407334 0.00189638 +4.36523 0.00341052 0.0771187 0 0.000263015 0.00341052 0.421614 0.00191762 +4.38477 0.00337017 0.0792889 0 0.000267217 0.00337017 0.436065 0.001938 +4.4043 0.00333007 0.081459 0 0.000271264 0.00333007 0.45069 0.00195752 +4.42383 0.00329021 0.0836291 0 0.000275157 0.00329021 0.465491 0.0019762 +4.44336 0.00325058 0.0857993 0 0.000278898 0.00325058 0.480472 0.00199407 +4.46289 0.0032112 0.0879694 0 0.000282487 0.0032112 0.495637 0.00201114 +4.48242 0.00317206 0.0901396 0 0.000285928 0.00317206 0.510988 0.00202744 +4.50195 0.00313315 0.0923097 0 0.00028922 0.00313315 0.526529 0.00204297 +4.52148 0.00309449 0.0944798 0 0.000292367 0.00309449 0.542263 0.00205776 +4.54102 0.00305607 0.09665 0 0.000295369 0.00305607 0.558195 0.00207183 +4.56055 0.00301788 0.0988201 0 0.000298228 0.00301788 0.574327 0.00208517 +4.58008 0.00297994 0.10099 0 0.000300945 0.00297994 0.590665 0.00209782 +4.59961 0.00294224 0.10316 0 0.000303522 0.00294224 0.607211 0.00210978 +4.61914 0.00290477 0.105331 0 0.000305961 0.00290477 0.62397 0.00212107 +4.63867 0.00286755 0.107501 0 0.000308264 0.00286755 0.640945 0.0021317 +4.6582 0.00283057 0.109671 0 0.000310431 0.00283057 0.658142 0.00214167 +4.67773 0.00279383 0.111841 0 0.000312464 0.00279383 0.675564 0.00215102 +4.69727 0.00275732 0.114011 0 0.000314365 0.00275732 0.693216 0.00215973 +4.7168 0.00272106 0.116181 0 0.000316136 0.00272106 0.711103 0.00216784 +4.73633 0.00268504 0.118351 0 0.000317778 0.00268504 0.729228 0.00217533 +4.75586 0.00264925 0.120521 0 0.000319292 0.00264925 0.747598 0.00218224 +4.77539 0.00261371 0.122692 0 0.00032068 0.00261371 0.766217 0.00218856 +4.79492 0.00257841 0.124862 0 0.000321945 0.00257841 0.785089 0.00219431 +4.81445 0.00254335 0.127032 0 0.000323086 0.00254335 0.804221 0.00219949 +4.83398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.85352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.87305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.89258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.91211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.93164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.95117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.9707 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +4.99023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.00977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.0293 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.04883 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.06836 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.08789 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.10742 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.12695 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.14648 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.16602 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.18555 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.20508 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.22461 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.24414 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.26367 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.2832 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.30273 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.32227 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.3418 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.36133 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.38086 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.40039 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.41992 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.43945 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.45898 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.47852 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.49805 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.51758 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.53711 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.55664 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.57617 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.5957 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.61523 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.63477 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.6543 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.67383 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.69336 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.71289 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.73242 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.75195 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.77148 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.79102 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.81055 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.83008 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.84961 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.86914 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.88867 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9082 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.92773 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.94727 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.9668 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +5.98633 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.00586 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.02539 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.04492 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.06445 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.08398 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.10352 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.12305 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.14258 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.16211 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.18164 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.20117 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2207 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.24023 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.25977 0.00253936 0.127279 0 0.000323208 0.00253936 0.806419 0.00220005 +6.2793 0.001 0 0 0 0.001 0 0 +6.29883 0.001 0 0 0 0.001 0 0 +6.31836 0.001 0 0 0 0.001 0 0 +6.33789 0.001 0 0 0 0.001 0 0 +6.35742 0.001 0 0 0 0.001 0 0 +6.37695 0.001 0 0 0 0.001 0 0 +6.39648 0.001 0 0 0 0.001 0 0 +6.41602 0.001 0 0 0 0.001 0 0 +6.43555 0.001 0 0 0 0.001 0 0 +6.45508 0.001 0 0 0 0.001 0 0 +6.47461 0.001 0 0 0 0.001 0 0 +6.49414 0.001 0 0 0 0.001 0 0 +6.51367 0.001 0 0 0 0.001 0 0 +6.5332 0.001 0 0 0 0.001 0 0 +6.55273 0.001 0 0 0 0.001 0 0 +6.57227 0.001 0 0 0 0.001 0 0 +6.5918 0.001 0 0 0 0.001 0 0 +6.61133 0.001 0 0 0 0.001 0 0 +6.63086 0.001 0 0 0 0.001 0 0 +6.65039 0.001 0 0 0 0.001 0 0 +6.66992 0.001 0 0 0 0.001 0 0 +6.68945 0.001 0 0 0 0.001 0 0 +6.70898 0.001 0 0 0 0.001 0 0 +6.72852 0.001 0 0 0 0.001 0 0 +6.74805 0.001 0 0 0 0.001 0 0 +6.76758 0.001 0 0 0 0.001 0 0 +6.78711 0.001 0 0 0 0.001 0 0 +6.80664 0.001 0 0 0 0.001 0 0 +6.82617 0.001 0 0 0 0.001 0 0 +6.8457 0.001 0 0 0 0.001 0 0 +6.86523 0.001 0 0 0 0.001 0 0 +6.88477 0.001 0 0 0 0.001 0 0 +6.9043 0.001 0 0 0 0.001 0 0 +6.92383 0.001 0 0 0 0.001 0 0 +6.94336 0.001 0 0 0 0.001 0 0 +6.96289 0.001 0 0 0 0.001 0 0 +6.98242 0.001 0 0 0 0.001 0 0 +7.00195 0.001 0 0 0 0.001 0 0 +7.02148 0.001 0 0 0 0.001 0 0 +7.04102 0.001 0 0 0 0.001 0 0 +7.06055 0.001 0 0 0 0.001 0 0 +7.08008 0.001 0 0 0 0.001 0 0 +7.09961 0.001 0 0 0 0.001 0 0 +7.11914 0.001 0 0 0 0.001 0 0 +7.13867 0.001 0 0 0 0.001 0 0 +7.1582 0.001 0 0 0 0.001 0 0 +7.17773 0.001 0 0 0 0.001 0 0 +7.19727 0.001 0 0 0 0.001 0 0 +7.2168 0.001 0 0 0 0.001 0 0 +7.23633 0.001 0 0 0 0.001 0 0 +7.25586 0.001 0 0 0 0.001 0 0 +7.27539 0.001 0 0 0 0.001 0 0 +7.29492 0.001 0 0 0 0.001 0 0 +7.31445 0.001 0 0 0 0.001 0 0 +7.33398 0.001 0 0 0 0.001 0 0 +7.35352 0.001 0 0 0 0.001 0 0 +7.37305 0.001 0 0 0 0.001 0 0 +7.39258 0.001 0 0 0 0.001 0 0 +7.41211 0.001 0 0 0 0.001 0 0 +7.43164 0.001 0 0 0 0.001 0 0 +7.45117 0.001 0 0 0 0.001 0 0 +7.4707 0.001 0 0 0 0.001 0 0 +7.49023 0.001 0 0 0 0.001 0 0 +7.50977 0.001 0 0 0 0.001 0 0 +7.5293 0.001 0 0 0 0.001 0 0 +7.54883 0.001 0 0 0 0.001 0 0 +7.56836 0.001 0 0 0 0.001 0 0 +7.58789 0.001 0 0 0 0.001 0 0 +7.60742 0.001 0 0 0 0.001 0 0 +7.62695 0.001 0 0 0 0.001 0 0 +7.64648 0.001 0 0 0 0.001 0 0 +7.66602 0.001 0 0 0 0.001 0 0 +7.68555 0.001 0 0 0 0.001 0 0 +7.70508 0.001 0 0 0 0.001 0 0 +7.72461 0.001 0 0 0 0.001 0 0 +7.74414 0.001 0 0 0 0.001 0 0 +7.76367 0.001 0 0 0 0.001 0 0 +7.7832 0.001 0 0 0 0.001 0 0 +7.80273 0.001 0 0 0 0.001 0 0 +7.82227 0.001 0 0 0 0.001 0 0 +7.8418 0.001 0 0 0 0.001 0 0 +7.86133 0.001 0 0 0 0.001 0 0 +7.88086 0.001 0 0 0 0.001 0 0 +7.90039 0.001 0 0 0 0.001 0 0 +7.91992 0.001 0 0 0 0.001 0 0 +7.93945 0.001 0 0 0 0.001 0 0 +7.95898 0.001 0 0 0 0.001 0 0 +7.97852 0.001 0 0 0 0.001 0 0 +7.99805 0.001 0 0 0 0.001 0 0 +8.01758 0.001 0 0 0 0.001 0 0 +8.03711 0.001 0 0 0 0.001 0 0 +8.05664 0.001 0 0 0 0.001 0 0 +8.07617 0.001 0 0 0 0.001 0 0 +8.0957 0.001 0 0 0 0.001 0 0 +8.11523 0.001 0 0 0 0.001 0 0 +8.13477 0.001 0 0 0 0.001 0 0 +8.1543 0.001 0 0 0 0.001 0 0 +8.17383 0.001 0 0 0 0.001 0 0 +8.19336 0.001 0 0 0 0.001 0 0 +8.21289 0.001 0 0 0 0.001 0 0 +8.23242 0.001 0 0 0 0.001 0 0 +8.25195 0.001 0 0 0 0.001 0 0 +8.27148 0.001 0 0 0 0.001 0 0 +8.29102 0.001 0 0 0 0.001 0 0 +8.31055 0.001 0 0 0 0.001 0 0 +8.33008 0.001 0 0 0 0.001 0 0 +8.34961 0.001 0 0 0 0.001 0 0 +8.36914 0.001 0 0 0 0.001 0 0 +8.38867 0.001 0 0 0 0.001 0 0 +8.4082 0.001 0 0 0 0.001 0 0 +8.42773 0.001 0 0 0 0.001 0 0 +8.44727 0.001 0 0 0 0.001 0 0 +8.4668 0.001 0 0 0 0.001 0 0 +8.48633 0.001 0 0 0 0.001 0 0 +8.50586 0.001 0 0 0 0.001 0 0 +8.52539 0.001 0 0 0 0.001 0 0 +8.54492 0.001 0 0 0 0.001 0 0 +8.56445 0.001 0 0 0 0.001 0 0 +8.58398 0.001 0 0 0 0.001 0 0 +8.60352 0.001 0 0 0 0.001 0 0 +8.62305 0.001 0 0 0 0.001 0 0 +8.64258 0.001 0 0 0 0.001 0 0 +8.66211 0.001 0 0 0 0.001 0 0 +8.68164 0.001 0 0 0 0.001 0 0 +8.70117 0.001 0 0 0 0.001 0 0 +8.7207 0.001 0 0 0 0.001 0 0 +8.74023 0.001 0 0 0 0.001 0 0 +8.75977 0.001 0 0 0 0.001 0 0 +8.7793 0.001 0 0 0 0.001 0 0 +8.79883 0.001 0 0 0 0.001 0 0 +8.81836 0.001 0 0 0 0.001 0 0 +8.83789 0.001 0 0 0 0.001 0 0 +8.85742 0.001 0 0 0 0.001 0 0 +8.87695 0.001 0 0 0 0.001 0 0 +8.89648 0.001 0 0 0 0.001 0 0 +8.91602 0.001 0 0 0 0.001 0 0 +8.93555 0.001 0 0 0 0.001 0 0 +8.95508 0.001 0 0 0 0.001 0 0 +8.97461 0.001 0 0 0 0.001 0 0 +8.99414 0.001 0 0 0 0.001 0 0 +9.01367 0.001 0 0 0 0.001 0 0 +9.0332 0.001 0 0 0 0.001 0 0 +9.05273 0.001 0 0 0 0.001 0 0 +9.07227 0.001 0 0 0 0.001 0 0 +9.0918 0.001 0 0 0 0.001 0 0 +9.11133 0.001 0 0 0 0.001 0 0 +9.13086 0.001 0 0 0 0.001 0 0 +9.15039 0.001 0 0 0 0.001 0 0 +9.16992 0.001 0 0 0 0.001 0 0 +9.18945 0.001 0 0 0 0.001 0 0 +9.20898 0.001 0 0 0 0.001 0 0 +9.22852 0.001 0 0 0 0.001 0 0 +9.24805 0.001 0 0 0 0.001 0 0 +9.26758 0.001 0 0 0 0.001 0 0 +9.28711 0.001 0 0 0 0.001 0 0 +9.30664 0.001 0 0 0 0.001 0 0 +9.32617 0.001 0 0 0 0.001 0 0 +9.3457 0.001 0 0 0 0.001 0 0 +9.36523 0.001 0 0 0 0.001 0 0 +9.38477 0.001 0 0 0 0.001 0 0 +9.4043 0.001 0 0 0 0.001 0 0 +9.42383 0.001 0 0 0 0.001 0 0 +9.44336 0.001 0 0 0 0.001 0 0 +9.46289 0.001 0 0 0 0.001 0 0 +9.48242 0.001 0 0 0 0.001 0 0 +9.50195 0.001 0 0 0 0.001 0 0 +9.52148 0.001 0 0 0 0.001 0 0 +9.54102 0.001 0 0 0 0.001 0 0 +9.56055 0.001 0 0 0 0.001 0 0 +9.58008 0.001 0 0 0 0.001 0 0 +9.59961 0.001 0 0 0 0.001 0 0 +9.61914 0.001 0 0 0 0.001 0 0 +9.63867 0.001 0 0 0 0.001 0 0 +9.6582 0.001 0 0 0 0.001 0 0 +9.67773 0.001 0 0 0 0.001 0 0 +9.69727 0.001 0 0 0 0.001 0 0 +9.7168 0.001 0 0 0 0.001 0 0 +9.73633 0.001 0 0 0 0.001 0 0 +9.75586 0.001 0 0 0 0.001 0 0 +9.77539 0.001 0 0 0 0.001 0 0 +9.79492 0.001 0 0 0 0.001 0 0 +9.81445 0.001 0 0 0 0.001 0 0 +9.83398 0.001 0 0 0 0.001 0 0 +9.85352 0.001 0 0 0 0.001 0 0 +9.87305 0.001 0 0 0 0.001 0 0 +9.89258 0.001 0 0 0 0.001 0 0 +9.91211 0.001 0 0 0 0.001 0 0 +9.93164 0.001 0 0 0 0.001 0 0 +9.95117 0.001 0 0 0 0.001 0 0 +9.9707 0.001 0 0 0 0.001 0 0 +9.99023 0.001 0 0 0 0.001 0 0 From 163eb02a0b02b27df6721928e5263ae1ccbb9b27 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 27 Feb 2024 16:04:03 +0100 Subject: [PATCH 46/55] add hip_check() routine --- GPUSimulators/SHMEMSimulator.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py index cfee8f3..8417d7e 100644 --- a/GPUSimulators/SHMEMSimulator.py +++ b/GPUSimulators/SHMEMSimulator.py @@ -34,6 +34,20 @@ class SHMEMSimulator(Simulator.BaseSimulator): Class which handles communication and synchronization between simulators in different contexts (presumably on different GPUs) """ + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def __init__(self, sims, grid): self.logger = logging.getLogger(__name__) From ef32115e487a6bf246f25aa396de676fab93ca6e Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 27 Feb 2024 17:22:07 +0100 Subject: [PATCH 47/55] context with hip are deprecated --- GPUSimulators/IPythonMagic.py | 39 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/GPUSimulators/IPythonMagic.py b/GPUSimulators/IPythonMagic.py index fa452df..92baeb8 100644 --- a/GPUSimulators/IPythonMagic.py +++ b/GPUSimulators/IPythonMagic.py @@ -24,7 +24,8 @@ import gc from IPython.core import magic_arguments from IPython.core.magic import line_magic, Magics, magics_class -import pycuda.driver as cuda +#import pycuda.driver as cuda +from hip import hip, hiprtc from GPUSimulators import Common, CudaContext @@ -41,6 +42,20 @@ class MagicCudaContext(Magics): '--no_cache', '-nc', action="store_true", help='Disable caching of kernels') @magic_arguments.argument( '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels') + def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + def cuda_context_handler(self, line): args = magic_arguments.parse_argstring(self.cuda_context_handler, line) self.logger = logging.getLogger(__name__) @@ -49,7 +64,8 @@ class MagicCudaContext(Magics): context_flags = None if (args.blocking): - context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC + #context_flags = cuda.ctx_flags.SCHED_BLOCKING_SYNC + context_flags = hip_check(hip.hipSetDeviceFlags(hip.hipDeviceScheduleBlockingSync)) if args.name in self.shell.user_ns.keys(): self.logger.debug("Context already registered! Ignoring") @@ -63,11 +79,13 @@ class MagicCudaContext(Magics): # this function will be called on exceptions in any cell def custom_exc(shell, etype, evalue, tb, tb_offset=None): self.logger.exception("Exception caught: Resetting to CUDA context %s", args.name) - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() + #while (cuda.Context.get_current() != None): + while (hip.hipCtxGetCurrent() != None): + #context = cuda.Context.get_current() + context = hip_check(hip.hipCtxGetCurrent()) self.logger.info("Popping <%s>", str(context.handle)) - cuda.Context.pop() - + #cuda.Context.pop() + hip.hipCtxPopCurrent() if args.name in self.shell.user_ns.keys(): self.logger.info("Pushing <%s>", str(self.shell.user_ns[args.name].cuda_context.handle)) self.shell.user_ns[args.name].cuda_context.push() @@ -88,10 +106,13 @@ class MagicCudaContext(Magics): import atexit def exitfunc(): self.logger.info("Exitfunc: Resetting CUDA context stack") - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() + #while (cuda.Context.get_current() != None): + while (hip.hipCtxGetCurrent() != None): + #context = cuda.Context.get_current() + context = hip_check(hip.hipCtxGetCurrent()) self.logger.info("`-> Popping <%s>", str(context.handle)) - cuda.Context.pop() + #cuda.Context.pop() + hip.hipCtxPopCurrent() self.logger.debug("==================================================================") atexit.register(exitfunc) From 3fd27d5bb8591f61a3c7076638f0b1f667b29e30 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Tue, 27 Feb 2024 18:44:18 +0100 Subject: [PATCH 48/55] update cudaContext --- GPUSimulators/CudaContext.py | 100 ++++++++++++++++------------------- 1 file changed, 46 insertions(+), 54 deletions(-) diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py index e77ef06..d22bd5c 100644 --- a/GPUSimulators/CudaContext.py +++ b/GPUSimulators/CudaContext.py @@ -36,12 +36,16 @@ import gc #import pycuda.driver as cuda from hip import hip,hiprtc -from hip import rccl from GPUSimulators import Autotuner, Common -def hip_check(call_result): +""" +Class which keeps track of the CUDA context and some helper functions +""" +class CudaContext(object): + + def hip_check(call_result): err = call_result[0] result = call_result[1:] if len(result) == 1: @@ -55,11 +59,6 @@ def hip_check(call_result): raise RuntimeError(str(err)) return result -""" -Class which keeps track of the CUDA context and some helper functions -""" -class CudaContext(object): - def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): """ Create a new CUDA context @@ -92,18 +91,18 @@ class CudaContext(object): self.logger.debug(" => compute capability: %s", str(self.hip.hipDeviceComputeCapability(device))) # Create the CUDA context - #In HIP there is no need to specify a scheduling policy (it is abstracted). Here the HIP runtime system manages the workload to fit a specifc target architecture - #if context_flags is None: + if context_flags is None: # context_flags=cuda.ctx_flags.SCHED_AUTO - + context_flags=hip_check(hip.hipSetDeviceFlags(hip.hipDeviceScheduleAuto)) #self.cuda_context = self.cuda_device.make_context(flags=context_flags) - + self.cuda_context = self.hip_check(hip.hipCtxCreate(0, device)) + #free, total = cuda.mem_get_info() total = hip_check(hip.hipDeviceTotalMem(device)) #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) self.logger.debug(" => memory: %d / %d MB available", int(total/(1024*1024))) - #self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) #Create cache dir for cubin files self.cache_path = os.path.join(self.module_path, "cuda_cache") @@ -118,32 +117,37 @@ class CudaContext(object): self.autotuner = Autotuner.Autotuner() -# def __del__(self, *args): -# self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + def __del__(self, *args): + self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) # Loop over all contexts in stack, and remove "this" -# other_contexts = [] -# while (cuda.Context.get_current() != None): -# context = cuda.Context.get_current() -# if (context.handle != self.cuda_context.handle): -# self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) -# other_contexts = [context] + other_contexts -# cuda.Context.pop() -# else: -# self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) -# cuda.Context.pop() + other_contexts = [] + #while (cuda.Context.get_current() != None): + while (hip.hipCtxGetCurrent() != None): + #context = cuda.Context.get_current() + context = hip_check(hip.hipCtxGetCurrent()) + if (context.handle != self.cuda_context.handle): + self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) + other_contexts = [context] + other_contexts + #cuda.Context.pop() + hip.hipCtxPopCurrent() + else: + self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) + #cuda.Context.pop() + hip.hipCtxPopCurrent() # Add all the contexts we popped that were not our own -# for context in other_contexts: -# self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) -# cuda.Context.push(context) - -# self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) -# self.cuda_context.detach() + for context in other_contexts: + self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) + #cuda.Context.push(context) + hip_check(hip.hipCtxPushCurrent(context)) + + self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) + self.cuda_context.detach() -# def __str__(self): -# return "CudaContext id " + str(self.cuda_context.handle) + def __str__(self): + return "CudaContext id " + str(self.cuda_context.handle) def hash_kernel(kernel_filename, include_dirs): @@ -244,9 +248,9 @@ class CudaContext(object): with io.open(cached_kernel_filename, "rb") as file: file_str = file.read() - #No hip counterpart of module_from_buffer - module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) - + #module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) + module = hip_check(hip.hipModuleLoadDataEx(file_str, 0, None)) + print("HIP module loaded: to be checked!") self.modules[kernel_hash] = module return module @@ -272,21 +276,10 @@ class CudaContext(object): with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) - #cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) + cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) #module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) - #HIP version of compilation: but "name_of_fct" needs to be defined. e.g. - #source = b"""\ - #extern "C" __global__ void name_of_fct(float factor, int n, short unused1, int unused2, float unused3, float *x) { - #int tid = threadIdx.x + blockIdx.x * blockDim.x; - #if (tid < n) { - #x[tid] *= factor; - # } - #} - #""" - - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_string, b"name_of_fct", 0, [], [])) - + #cubin = hip_check(hiprtc.hiprtcCreateProgram(kernel_string.encode(), b"Kernel-Name", 0, [], [])) props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) arch = props.gcnArchName @@ -294,17 +287,16 @@ class CudaContext(object): print(f"Compiling kernel for {arch}") cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + err, = hiprtc.hiprtcCompileProgram(cubin, len(cflags), cflags) if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) + log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(cubin)) log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) + hip_check(hiprtc.hiprtcGetProgramLog(cubin, log)) raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(cubin)) code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(prog, code)) + hip_check(hiprtc.hiprtcGetCode(cubin, code)) module = hip_check(hip.hipModuleLoadData(code)) - #kernel = hip_check(hip.hipModuleGetFunction(module, b"name_of_fct")) if (self.use_cache): with io.open(cached_kernel_filename, "wb") as file: From d5601ec80848b36f223f6921f32d4d0e44c2c370 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Wed, 20 Mar 2024 16:54:19 +0100 Subject: [PATCH 49/55] update CudaContext --- GPUSimulators/CudaContext.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py index d22bd5c..4f2e0aa 100644 --- a/GPUSimulators/CudaContext.py +++ b/GPUSimulators/CudaContext.py @@ -22,7 +22,7 @@ along with this program. If not, see . import os - +import ctypes import numpy as np import time import re @@ -43,9 +43,8 @@ from GPUSimulators import Autotuner, Common """ Class which keeps track of the CUDA context and some helper functions """ -class CudaContext(object): - def hip_check(call_result): +def hip_check(call_result): err = call_result[0] result = call_result[1:] if len(result) == 1: @@ -59,6 +58,8 @@ class CudaContext(object): raise RuntimeError(str(err)) return result +class CudaContext(object): + def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): """ Create a new CUDA context @@ -79,30 +80,42 @@ class CudaContext(object): #Print some info about CUDA ##self.logger.info("CUDA version %s", str(cuda.get_version())) #self.logger.info("Driver version %s", str(cuda.get_driver_version())) + self.logger.info("HIP runtime version %s", str(hip_check(hip.hipRuntimeGetVersion()))) self.logger.info("Driver version %s", str(hip_check(hip.hipDriverGetVersion()))) if device is None: device = 0 - - self.cuda_device = hip.Device(device) + + hip_check(hip.hipSetDevice(device)) + props = hip.hipDeviceProp_t() + hip_check(hip.hipGetDeviceProperties(props,device)) + arch = props.gcnArchName + #self.cuda_device = cuda.Device(device) + #self.cuda_device = hip_check(hip.hipCtxGetDevice()) #self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) - self.logger.info("Using device %d/%d '%s' (%s) GPU", device, hip_check(hip.hipGetDeviceCount())) + + # Allocate memory to store the PCI BusID + pciBusId = ctypes.create_string_buffer(64) + # PCI Bus Id + hip_check(hip.hipDeviceGetPCIBusId(pciBusId, 64, device)) + + self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, hip_check(hip.hipGetDeviceCount()),arch,pciBusId.value.decode('utf-8')[5:7]) #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) - self.logger.debug(" => compute capability: %s", str(self.hip.hipDeviceComputeCapability(device))) + self.logger.debug(" => compute capability: %s", hip_check(hip.hipDeviceComputeCapability(device))) # Create the CUDA context if context_flags is None: # context_flags=cuda.ctx_flags.SCHED_AUTO context_flags=hip_check(hip.hipSetDeviceFlags(hip.hipDeviceScheduleAuto)) #self.cuda_context = self.cuda_device.make_context(flags=context_flags) - self.cuda_context = self.hip_check(hip.hipCtxCreate(0, device)) + self.cuda_context = hip_check(hip.hipCtxCreate(0, device)) #free, total = cuda.mem_get_info() total = hip_check(hip.hipDeviceTotalMem(device)) #self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) - self.logger.debug(" => memory: %d / %d MB available", int(total/(1024*1024))) + self.logger.debug(" => Total memory: %d MB available", int(total/(1024*1024))) - self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) + ##self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) #Create cache dir for cubin files self.cache_path = os.path.join(self.module_path, "cuda_cache") From 2a7a8c62583f857e4bd575a289d22906222c0767 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Sun, 9 Jun 2024 22:48:06 +0200 Subject: [PATCH 50/55] hip-python implementation --- GPUSimulators/Autotuner.py | 5 +- GPUSimulators/Common.py | 371 ++++++----- GPUSimulators/CudaContext.py | 82 ++- GPUSimulators/CudaContext_cu.py | 272 -------- GPUSimulators/EE2D_KP07_dimsplit.py | 614 ++++++++++-------- GPUSimulators/FORCE.py | 215 +++--- GPUSimulators/HLL.py | 224 ++++--- GPUSimulators/HLL2.py | 242 ++++--- GPUSimulators/IPythonMagic.py | 26 +- GPUSimulators/KP07.py | 247 ++++--- GPUSimulators/KP07_dimsplit.py | 242 ++++--- GPUSimulators/LxF.py | 223 ++++--- GPUSimulators/MPISimulator.py | 95 +-- GPUSimulators/SHMEMSimulator.py | 27 +- GPUSimulators/SHMEMSimulatorGroup.py | 27 +- GPUSimulators/Simulator.py | 33 +- GPUSimulators/WAF.py | 227 ++++--- .../__pycache__/MPISimulator.cpython-39.pyc | Bin 11058 -> 0 bytes .../__pycache__/Simulator.cpython-39.pyc | Bin 8433 -> 0 bytes .../__pycache__/__init__.cpython-39.pyc | Bin 182 -> 0 bytes GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip | 3 +- GPUSimulators/cuda/common.h | 13 +- .../InitialConditions.cpython-39.pyc | Bin 0 -> 7097 bytes 23 files changed, 1769 insertions(+), 1419 deletions(-) delete mode 100644 GPUSimulators/CudaContext_cu.py delete mode 100644 GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc delete mode 100644 GPUSimulators/__pycache__/Simulator.cpython-39.pyc delete mode 100644 GPUSimulators/__pycache__/__init__.cpython-39.pyc create mode 100644 GPUSimulators/helpers/__pycache__/InitialConditions.cpython-39.pyc diff --git a/GPUSimulators/Autotuner.py b/GPUSimulators/Autotuner.py index 84aedc2..89a071d 100644 --- a/GPUSimulators/Autotuner.py +++ b/GPUSimulators/Autotuner.py @@ -31,8 +31,7 @@ from hip import hip,hiprtc from GPUSimulators import Common, Simulator, CudaContext -class Autotuner: - def hip_check(call_result): +def hip_check(call_result): err = call_result[0] result = call_result[1:] if len(result) == 1: @@ -46,6 +45,8 @@ class Autotuner: raise RuntimeError(str(err)) return result + +class Autotuner: def __init__(self, nx=2048, ny=2048, block_widths=range(8, 32, 1), diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py index 6681450..f23d9a7 100644 --- a/GPUSimulators/Common.py +++ b/GPUSimulators/Common.py @@ -56,7 +56,7 @@ def hip_check(call_result): ): raise RuntimeError(str(err)) return result - + def safeCall(cmd): logger = logging.getLogger(__name__) try: @@ -158,7 +158,7 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names extent = sim.getExtent() ncvars['x'][:] = np.linspace(extent[0], extent[1], simulator_args['nx']) ncvars['y'][:] = np.linspace(extent[2], extent[3], simulator_args['ny']) - + #Choose which variables to download (prune None from list, but keep the index) download_vars = [] for i, var_name in enumerate(save_var_names): @@ -203,7 +203,7 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names #Download save_vars = sim.download(download_vars) - + #Save to file for i, var_name in enumerate(save_var_names): ncvars[var_name][k, :] = save_vars[i] @@ -216,12 +216,9 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names logger.debug(print_string) logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps())) - + return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi - - - - + #return outdata.filename class Timer(object): @@ -246,9 +243,6 @@ class Timer(object): def elapsed(self): return time.time() - self.start - - - class PopenFileBuffer(object): """ @@ -366,10 +360,6 @@ class IPEngine(object): gc.collect() - - - - class DataDumper(object): """ Simple class for holding a netCDF4 object @@ -443,8 +433,6 @@ class DataDumper(object): - - class ProgressPrinter(object): """ Small helper class for @@ -499,11 +487,6 @@ class ProgressPrinter(object): return progressbar - - - - - """ Class that holds 2D data """ @@ -520,24 +503,28 @@ class CudaArray2D: nx_halo = nx + 2*x_halo ny_halo = ny + 2*y_halo - + #self.logger.debug("Allocating [%dx%d] buffer", self.nx, self.ny) #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here #Initialize an array on GPU with zeros #self.data = pycuda.gpuarray.zeros((ny_halo, nx_halo), dtype) - self.data_h = np.zeros((ny_halo, nx_halo), dtype="float32") - num_bytes = self.data_h.size * self.data_h.itemsize - + #data.strides[0] == nx_halo*np.float32().itemsize + #data.strides[1] == np.float32().itemsize + num_bytes = ny_halo*nx_halo * np.float32().itemsize + + #data_h = np.zeros((ny_halo, nx_halo), dtype) # init device array and upload host data self.data = hip_check(hip.hipMalloc(num_bytes)).configure( typestr="float32",shape=(ny_halo, nx_halo)) + #num_bytes = ny*nx * np.float32().itemsize + #cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) # copy data from host to device - hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + #hip_check(hip.hipMemcpy(self.data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - #For returning to download (No counterpart in hip-python) + #https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemPoolCreate #self.memorypool = PageLockedMemoryPool() - + #If we don't have any data, just allocate and return if cpu_data is None: return @@ -547,16 +534,21 @@ class CudaArray2D: assert cpu_data.itemsize == 4, "Wrong size of data type" assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" + #Create copy object from host to device x = (nx_halo - cpu_data.shape[1]) // 2 y = (ny_halo - cpu_data.shape[0]) // 2 self.upload(stream, cpu_data, extent=[x, y, cpu_data.shape[1], cpu_data.shape[0]]) + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) - + def __del__(self, *args): #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) - self.data.gpudata.free() + #self.data.gpudata.free() + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data), self.nx, self.ny) + hip_check(hip.hipFree(self.data)) + #hip_check(hip.hipFreeAsync(self.data, self.stream)) self.data = None """ @@ -570,71 +562,84 @@ class CudaArray2D: ny = self.ny else: x, y, nx, ny = extent - + if (cpu_data is None): #self.logger.debug("Downloading [%dx%d] buffer", self.nx, self.ny) #Allocate host memory #The following fails, don't know why (crashes python) + #allocate a pinned (page-locked) memory array #cpu_data = cuda.pagelocked_empty((int(ny), int(nx)), dtype=np.float32, mem_flags=cuda.host_alloc_flags.PORTABLE) #see here type of memory: https://rocm.docs.amd.com/projects/hip-python/en/latest/python_api/hip.html#hip.hip.hipMemoryType - cpu_data = np.empty((ny, nx), dtype=np.float32) - num_bytes = cpu_data.size * cpu_data.itemsize - #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can #be accessed directly by the GPU device + cpu_data = np.zeros((ny, nx), dtype=np.float32) + #num_bytes = cpu_data.size * cpu_data.itemsize + #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can #be accessed directly by the GPU device #hipHostMallocDefault:Memory is mapped and portable (default allocation) #hipHostMallocPortable: memory is explicitely portable across different devices - cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) + #cpu_data = hip_check(hip.hipHostMalloc(num_bytes,hip.hipHostMallocPortable)) #Non-pagelocked: cpu_data = np.empty((ny, nx), dtype=np.float32) #cpu_data = self.memorypool.allocate((ny, nx), dtype=np.float32) - + assert nx == cpu_data.shape[1] assert ny == cpu_data.shape[0] assert x+nx <= self.nx + 2*self.x_halo assert y+ny <= self.ny + 2*self.y_halo - + + #Cuda + """ #Create copy object from device to host - #copy = cuda.Memcpy2D() - #copy.set_src_device(self.data.gpudata) - #copy.set_dst_host(cpu_data) + copy = cuda.Memcpy2D() + copy.set_src_device(self.data.gpudata) + copy.set_dst_host(cpu_data) #Set offsets and pitch of source - #copy.src_x_in_bytes = int(x)*self.data.strides[1] - #copy.src_y = int(y) - #copy.src_pitch = self.data.strides[0] + copy.src_x_in_bytes = int(x)*self.data.strides[1] + copy.src_y = int(y) + copy.src_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - #copy.width_in_bytes = int(nx)*cpu_data.itemsize - #copy.height = int(ny) - - #The equivalent of cuda.Memcpy2D in hip-python would be: but it fails with an error pointing to cpu_data - #and a message: "RuntimeError: hipError_t.hipErrorInvalidValue" - #shape = (nx,ny) - #num_bytes = cpu_data.size * cpu_data.itemsize - #dst_pitch_bytes = cpu_data.strides[0] - #src_pitch_bytes = num_bytes // shape[0] - #src_pitch_bytes = data.strides[0] - #width_bytes = int(nx)*cpu_data.itemsize - #height_Nrows = int(ny) - #hipMemcpy2D(dst, unsigned long dpitch, src, unsigned long spitch, unsigned long width, unsigned long height, kind) - #copy = hip_check(hip.hipMemcpy2D(cpu_data, #pointer to destination - # dst_pitch_bytes, #pitch of destination array - # data, #pointer to source - # src_pitch_bytes, #pitch of source array - # width_bytes, #number of bytes in each row - # height_Nrows, #number of rows to copy - # hip.hipMemcpyKind.hipMemcpyDeviceToHost)) #kind + copy.width_in_bytes = int(nx)*cpu_data.itemsize + copy.height = int(ny) + """ - #this is an alternative: #copy from device to host - cpu_data = np.empty((ny, nx), dtype=np.float32) - num_bytes = cpu_data.size * cpu_data.itemsize - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) - copy(stream) - if asynch==False: - stream.synchronize() + #host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault)) + #device_pointer = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault)) + + + copy_download = { + 'srcXInBytes': int(x)*np.float32().itemsize, + 'srcY': int(y), + 'srcMemoryType': hip.hipMemoryType.hipMemoryTypeDevice,#hipMemoryTypeManaged + 'srcDevice': self.data, + 'srcPitch': self.data.shape[0]*np.float32().itemsize, + + 'dstXInBytes': 0, + 'dstY': 0, + 'dstMemoryType': hip.hipMemoryType.hipMemoryTypeHost, + 'dstHost': cpu_data, #device_pointer, + 'dstPitch': cpu_data.strides[0], + + 'WidthInBytes': int(nx)*cpu_data.itemsize, + 'Height': int(ny) + } + + # Perform the copy back to host + Copy = hip.hip_Memcpy2D(**copy_download) + + #err = hip.hipMemcpyParam2D(Copy) + err = hip.hipMemcpyParam2DAsync(Copy, stream) + if err is None: + print("--download - DtoH: Failed to copy 2D data to Host") + print("--I stop:", err) + exit() + + #copy(stream) + if asynch==False: + #stream.synchronize() + hip_check(hip.hipStreamSynchronize(stream)) return cpu_data @@ -646,37 +651,67 @@ class CudaArray2D: ny = self.ny else: x, y, nx, ny = extent - + assert(nx == cpu_data.shape[1]) assert(ny == cpu_data.shape[0]) assert(x+nx <= self.nx + 2*self.x_halo) assert(y+ny <= self.ny + 2*self.y_halo) - + + #Cuda + """ #Create copy object from device to host #Well this copy from src:host to dst:device AND NOT from device to host - #copy = cuda.Memcpy2D() - #copy.set_dst_device(self.data.gpudata) - #copy.set_src_host(cpu_data) + copy = cuda.Memcpy2D() + copy.set_dst_device(self.data.gpudata) + copy.set_src_host(cpu_data) #Set offsets and pitch of source - #copy.dst_x_in_bytes = int(x)*self.data.strides[1] - #copy.dst_y = int(y) - #copy.dst_pitch = self.data.strides[0] + copy.dst_x_in_bytes = int(x)*self.data.strides[1] + copy.dst_y = int(y) + copy.dst_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - #copy.width_in_bytes = int(nx)*cpu_data.itemsize - #copy.height = int(ny) + copy.width_in_bytes = int(nx)*cpu_data.itemsize + copy.height = int(ny) + """ + + + #Copy from host to device + + #host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault)) + #device_pointer = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault)) + + copy_upload = { + 'srcXInBytes': 0, + 'srcY': 0, + 'srcMemoryType': hip.hipMemoryType.hipMemoryTypeHost, + 'srcHost': cpu_data, #device_pointer + 'srcPitch': cpu_data.strides[0], # assuming float32 (4 bytes) + + 'dstXInBytes': int(x)*np.float32().itemsize, + 'dstY': int(y), + 'dstMemoryType': hip.hipMemoryType.hipMemoryTypeDevice, #hipMemoryTypeManaged + 'dstDevice': self.data, + 'dstPitch': self.data.shape[0]*np.float32().itemsize, + + 'WidthInBytes': int(nx)*cpu_data.itemsize, + 'Height': int(ny) + } + + + # Perform the copy HtoD + Copy = hip.hip_Memcpy2D(**copy_upload) - #copy from host de device - num_bytes = cpu_data.size * cpu_data.itemsize - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=cpu_data.shape) - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - - copy(stream) + #err = hip.hipMemcpyParam2D(Copy) + err = hip.hipMemcpyParam2DAsync(Copy, stream) + + if err is None: + print("--Upload - HtoD: Failed to copy 2D data to Device") + print("--I stop:", err) + exit() + #copy(stream) @@ -704,15 +739,12 @@ class CudaArray3D: #Should perhaps use pycuda.driver.mem_alloc_data.pitch() here #self.data = pycuda.gpuarray.zeros((nz_halo, ny_halo, nx_halo), dtype) - self.data_h = np.zeros((nz_halo, ny_halo, nx_halo), dtype="float32") - num_bytes = self.data_h.size * self.data_h.itemsize - + """ + num_bytes = nz_halo*ny_halo*nx_halo * np.float32().itemsize # init device array and upload host data self.data = hip_check(hip.hipMalloc(num_bytes)).configure( typestr="float32",shape=(nz_halo, ny_halo, nx_halo)) - - # copy data from host to device - hip_check(hip.hipMemcpy(self.data,self.data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) + """ #For returning to download #self.memorypool = PageLockedMemoryPool() @@ -725,48 +757,85 @@ class CudaArray3D: assert cpu_data.shape == (nz_halo, ny_halo, nx_halo) or cpu_data.shape == (self.nz, self.ny, self.nx), "Wrong shape of data %s vs %s / %s" % (str(cpu_data.shape), str((self.nz, self.ny, self.nx)), str((nz_halo, ny_halo, nx_halo))) assert cpu_data.itemsize == 4, "Wrong size of data type" assert not np.isfortran(cpu_data), "Wrong datatype (Fortran, expected C)" - + + #Cuda + """ #Create copy object from host to device - #copy = cuda.Memcpy3D() - #copy.set_src_host(cpu_data) - #copy.set_dst_device(self.data.gpudata) + copy = cuda.Memcpy3D() + copy.set_src_host(cpu_data) + copy.set_dst_device(self.data.gpudata) #Set offsets of destination - #x_offset = (nx_halo - cpu_data.shape[2]) // 2 - #y_offset = (ny_halo - cpu_data.shape[1]) // 2 - #z_offset = (nz_halo - cpu_data.shape[0]) // 2 - #copy.dst_x_in_bytes = x_offset*self.data.strides[1] - #copy.dst_y = y_offset - #copy.dst_z = z_offset + x_offset = (nx_halo - cpu_data.shape[2]) // 2 + y_offset = (ny_halo - cpu_data.shape[1]) // 2 + z_offset = (nz_halo - cpu_data.shape[0]) // 2 + copy.dst_x_in_bytes = x_offset*self.data.strides[1] + copy.dst_y = y_offset + copy.dst_z = z_offset #Set pitch of destination - #copy.dst_pitch = self.data.strides[0] + copy.dst_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - #width = max(self.nx, cpu_data.shape[2]) - #height = max(self.ny, cpu_data.shape[1]) - #depth = max(self.nz, cpu-data.shape[0]) - #copy.width_in_bytes = width*cpu_data.itemsize - #copy.height = height - #copy.depth = depth + width = max(self.nx, cpu_data.shape[2]) + height = max(self.ny, cpu_data.shape[1]) + depth = max(self.nz, cpu-data.shape[0]) + copy.width_in_bytes = width*cpu_data.itemsize + copy.height = height + copy.depth = depth - #copy from host to device - num_bytes = cpu_data.size * cpu_data.itemsize - self.data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=cpu_data.shape) - #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice)) - #Perform the copy copy(stream) - #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data.gpudata), self.nx, self.ny) - + """ + + #copy from host to device + #src + host_array_pinned = hip_check(hip.hipHostMalloc(cpu_data.size * cpu_data.itemsize, hip.hipHostMallocDefault)) + src_ptr = hip_check(hip.hipHostGetDevicePointer(host_array_pinned,hip.hipHostMallocDefault)) + #src_ptr = hip.hipPitchedPtr() + + #dst + # Allocate 3D pitched memory on the device + self.data = hip.hipPitchedPtr() + c_extent = hip.hipExtent(nx_halo*np.float32().itemsize, ny_halo, nz_halo) + #hip.hipMalloc3D(pitchedDevPtr-OUT, extent-IN) + err, = hip.hipMalloc3D(self.data, c_extent) + dst_pitch = nx_halo * np.float32().itemsize + + #include offset: do we need make_hipPitchedPtr + x_offset = (nx_halo - cpu_data.shape[2]) // 2 + y_offset = (ny_halo - cpu_data.shape[1]) // 2 + z_offset = (nz_halo - cpu_data.shape[0]) // 2 + + if err != hip.hipError_t.hipSuccess: + raise RuntimeError(f"Error from hipMalloc3D: {hip.hipGetErrorString(err)}") + + copy_upload = { + 'srcPos': hip.hipPos(0, 0, 0), + 'srcPtr': src_ptr, + 'dstPos': hip.hipPos(0, 0, 0), + 'dstPtr': self.data, + 'extent': c_extent, + 'kind': hip.hipMemcpyKind.hipMemcpyHostToDevice + } + + # Perform the copy + copy = hip.hipMemcpy3DParms(**copy_upload) + err = hip.hipMemcpy3DAsync(copy, stream) + #copy = hip_check(hip.hipMemcpyAsync(self.data,cpu_data,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,stream)) + + + #self.logger.debug("Buffer <%s> [%dx%d]: Allocated ", int(self.data), self.nx, self.ny) + def __del__(self, *args): #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data.gpudata), self.nx, self.ny) - self.data.gpudata.free() + #self.logger.debug("Buffer <%s> [%dx%d]: Releasing ", int(self.data), self.nx, self.ny) + #self.data.gpudata.free() + hip_check(hip.hipFree(self.data)) + #hip_check(hip.hipFreeAsync(self.data, self.stream)) self.data = None """ @@ -778,33 +847,37 @@ class CudaArray3D: #cpu_data = cuda.pagelocked_empty((self.ny, self.nx), np.float32) cpu_data = np.empty((self.nz, self.ny, self.nx), dtype=np.float32) #cpu_data = self.memorypool.allocate((self.nz, self.ny, self.nx), dtype=np.float32) - + + #Cuda + """ #Create copy object from device to host - #copy = cuda.Memcpy2D() - #copy.set_src_device(self.data.gpudata) - #copy.set_dst_host(cpu_data) + copy = cuda.Memcpy2D() + copy.set_src_device(self.data.gpudata) + copy.set_dst_host(cpu_data) #Set offsets and pitch of source - #copy.src_x_in_bytes = self.x_halo*self.data.strides[1] - #copy.src_y = self.y_halo - #copy.src_z = self.z_halo - #copy.src_pitch = self.data.strides[0] + copy.src_x_in_bytes = self.x_halo*self.data.strides[1] + copy.src_y = self.y_halo + copy.src_z = self.z_halo + copy.src_pitch = self.data.strides[0] #Set width in bytes to copy for each row and #number of rows to copy - #copy.width_in_bytes = self.nx*cpu_data.itemsize - #copy.height = self.ny - #copy.depth = self.nz + copy.width_in_bytes = self.nx*cpu_data.itemsize + copy.height = self.ny + copy.depth = self.nz + copy(stream) + """ #copy from device to host num_bytes = cpu_data.size * cpu_data.itemsize #hip.hipMemcpy(dst, src, unsigned long sizeBytes, kind) - copy = hip_check(hip.hipMemcpy(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + copy = hip_check(hip.hipMemcpyAsync(cpu_data,self.data,num_bytes,hip.hipMemcpyKind.hipMemcpyDeviceToHost,stream)) - copy(stream) if asynch==False: - stream.synchronize() - + #stream.synchronize() + hip_check(hip.hipStreamSynchronize(stream)) + return cpu_data @@ -818,9 +891,11 @@ class ArakawaA2D: """ self.logger = logging.getLogger(__name__) self.gpu_variables = [] + for cpu_variable in cpu_variables: self.gpu_variables += [CudaArray2D(stream, nx, ny, halo_x, halo_y, cpu_variable)] - + + def __getitem__(self, key): assert type(key) == int, "Indexing is int based" if (key > len(self.gpu_variables) or key < 0): @@ -833,21 +908,23 @@ class ArakawaA2D: """ if variables is None: variables=range(len(self.gpu_variables)) - + cpu_variables = [] for i in variables: assert i < len(self.gpu_variables), "Variable {:d} is out of range".format(i) cpu_variables += [self.gpu_variables[i].download(stream, asynch=True)] + #print("--FIN: sum:", np.array(cpu_variables).sum()) + #stream.synchronize() + hip_check(hip.hipStreamSynchronize(stream)) return cpu_variables #hipblas def sum_hipblas(self, num_elements, data): num_bytes_r = np.dtype(np.float32).itemsize result_d = hip_check(hip.hipMalloc(num_bytes_r)) - result_h = np.zeros(1, dtype=np.float32) - print("--bytes:", num_bytes_r) + result_h0 = np.zeros(1, dtype=np.float32) # call hipblasSaxpy + initialization handle = hip_check(hipblas.hipblasCreate()) @@ -859,10 +936,12 @@ class ArakawaA2D: hip_check(hipblas.hipblasDestroy(handle)) # copy result (stored in result_d) back to host (store in result_h) - hip_check(hip.hipMemcpy(result_h,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + hip_check(hip.hipMemcpy(result_h0,result_d,num_bytes_r,hip.hipMemcpyKind.hipMemcpyDeviceToHost)) + + result_h = result_h0[0] # clean up - hip_check(hip.hipFree(data)) + #hip_check(hip.hipFree(data)) return result_h def check(self): @@ -872,8 +951,8 @@ class ArakawaA2D: for i, gpu_variable in enumerate(self.gpu_variables): #compute sum with hipblas #var_sum = pycuda.gpuarray.sum(gpu_variable.data).get() - var_sum = self.sum_hipblas(gpu_variable.ny,gpu_variable.data) + var_sum = self.sum_hipblas(gpu_variable.data.size,gpu_variable.data) + #print(f"GPU: Sum for column {i}: {var_sum}") self.logger.debug("Data %d with size [%d x %d] has average %f", i, gpu_variable.nx, gpu_variable.ny, var_sum / (gpu_variable.nx * gpu_variable.ny)) assert np.isnan(var_sum) == False, "Data contains NaN values!" - diff --git a/GPUSimulators/CudaContext.py b/GPUSimulators/CudaContext.py index 4f2e0aa..45b3b36 100644 --- a/GPUSimulators/CudaContext.py +++ b/GPUSimulators/CudaContext.py @@ -85,8 +85,9 @@ class CudaContext(object): if device is None: device = 0 - - hip_check(hip.hipSetDevice(device)) + + num_gpus = hip_check(hip.hipGetDeviceCount()) + hip.hipSetDevice(device) props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,device)) arch = props.gcnArchName @@ -97,9 +98,12 @@ class CudaContext(object): # Allocate memory to store the PCI BusID pciBusId = ctypes.create_string_buffer(64) # PCI Bus Id - hip_check(hip.hipDeviceGetPCIBusId(pciBusId, 64, device)) + #hip_check(hip.hipDeviceGetPCIBusId(pciBusId, 64, device)) + pciBusId = hip_check(hip.hipDeviceGetPCIBusId(64, device)) - self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, hip_check(hip.hipGetDeviceCount()),arch,pciBusId.value.decode('utf-8')[5:7]) + + #self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, num_gpus,arch,pciBusId.value.decode('utf-8')[5:7]) + self.logger.info("Using device %d/%d with --arch: '%s', --BusID: %s ", device, num_gpus,arch,pciBusId[5:7]) #self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) self.logger.debug(" => compute capability: %s", hip_check(hip.hipDeviceComputeCapability(device))) @@ -116,7 +120,8 @@ class CudaContext(object): self.logger.debug(" => Total memory: %d MB available", int(total/(1024*1024))) ##self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) - + self.logger.info("Created context handle <%s>", str(self.cuda_context)) + #Create cache dir for cubin files self.cache_path = os.path.join(self.module_path, "cuda_cache") if (self.use_cache): @@ -125,42 +130,51 @@ class CudaContext(object): self.logger.info("Using CUDA cache dir %s", self.cache_path) self.autotuner = None + """ if (autotuning): self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") self.autotuner = Autotuner.Autotuner() - + """ def __del__(self, *args): - self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) - + #self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) + #self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context)) + """ # Loop over all contexts in stack, and remove "this" other_contexts = [] #while (cuda.Context.get_current() != None): while (hip.hipCtxGetCurrent() != None): #context = cuda.Context.get_current() context = hip_check(hip.hipCtxGetCurrent()) - if (context.handle != self.cuda_context.handle): - self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) + #if (context.handle != self.cuda_context.handle): + if (context != self.cuda_context): + #self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) + #self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context), str(context)) other_contexts = [context] + other_contexts #cuda.Context.pop() hip.hipCtxPopCurrent() else: - self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) + #self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) + self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context), str(context)) #cuda.Context.pop() hip.hipCtxPopCurrent() # Add all the contexts we popped that were not our own for context in other_contexts: - self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) + #self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) + self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context), str(context)) #cuda.Context.push(context) hip_check(hip.hipCtxPushCurrent(context)) - self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) - self.cuda_context.detach() - + #self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) + self.logger.debug("<%s> Detaching", str(self.cuda_context)) + #self.cuda_context.detach() + hip_check(hip.hipCtxDestroy(self.cuda_context)) + """ def __str__(self): - return "CudaContext id " + str(self.cuda_context.handle) + #return "CudaContext id " + str(self.cuda_context.handle) + return "CudaContext id " + str(self.cuda_context) def hash_kernel(kernel_filename, include_dirs): @@ -227,7 +241,7 @@ class CudaContext(object): self.logger.debug("Info: %s", info_str) if error_str: self.logger.debug("Error: %s", error_str) - + kernel_filename = os.path.normpath(kernel_filename) kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) #self.logger.debug("Getting %s", kernel_filename) @@ -236,12 +250,12 @@ class CudaContext(object): options_hasher = hashlib.md5() options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); options_hash = options_hasher.hexdigest() - + # Create hash of kernel souce source_hash = CudaContext.hash_kernel( \ kernel_path, \ include_dirs=[self.module_path] + include_dirs) - + # Create final hash root, ext = os.path.splitext(kernel_filename) kernel_hash = root \ @@ -282,34 +296,16 @@ class CudaContext(object): os.mkdir(cached_kernel_dir) with io.open(cached_kernel_filename + ".txt", "w") as file: file.write(kernel_string) - + """cuda with Common.Timer("compiler") as timer: + import warnings with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) - #module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) - - #cubin = hip_check(hiprtc.hiprtcCreateProgram(kernel_string.encode(), b"Kernel-Name", 0, [], [])) - props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) - arch = props.gcnArchName - - print(f"Compiling kernel for {arch}") - - cflags = [b"--offload-arch="+arch] - err, = hiprtc.hiprtcCompileProgram(cubin, len(cflags), cflags) - if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: - log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(cubin)) - log = bytearray(log_size) - hip_check(hiprtc.hiprtcGetProgramLog(cubin, log)) - raise RuntimeError(log.decode()) - code_size = hip_check(hiprtc.hiprtcGetCodeSize(cubin)) - code = bytearray(code_size) - hip_check(hiprtc.hiprtcGetCode(cubin, code)) - module = hip_check(hip.hipModuleLoadData(code)) + module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) if (self.use_cache): with io.open(cached_kernel_filename, "wb") as file: @@ -317,7 +313,8 @@ class CudaContext(object): self.modules[kernel_hash] = module return module - + """ + """ Clears the kernel cache (useful for debugging & development) """ @@ -330,4 +327,5 @@ class CudaContext(object): Synchronizes all streams etc """ def synchronize(self): - self.cuda_context.synchronize() + #self.cuda_context.synchronize() + hip_check(hip.hipCtxSynchronize()) diff --git a/GPUSimulators/CudaContext_cu.py b/GPUSimulators/CudaContext_cu.py deleted file mode 100644 index 6c90636..0000000 --- a/GPUSimulators/CudaContext_cu.py +++ /dev/null @@ -1,272 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -This python module implements Cuda context handling - -Copyright (C) 2018 SINTEF ICT - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - - - -import os - -import numpy as np -import time -import re -import io -import hashlib -import logging -import gc - -import pycuda.compiler as cuda_compiler -import pycuda.gpuarray -import pycuda.driver as cuda - -from GPUSimulators import Autotuner, Common - - - -""" -Class which keeps track of the CUDA context and some helper functions -""" -class CudaContext(object): - - def __init__(self, device=None, context_flags=None, use_cache=True, autotuning=True): - """ - Create a new CUDA context - Set device to an id or pci_bus_id to select a specific GPU - Set context_flags to cuda.ctx_flags.SCHED_BLOCKING_SYNC for a blocking context - """ - self.use_cache = use_cache - self.logger = logging.getLogger(__name__) - self.modules = {} - - self.module_path = os.path.dirname(os.path.realpath(__file__)) - - #Initialize cuda (must be first call to PyCUDA) - cuda.init(flags=0) - - self.logger.info("PyCUDA version %s", str(pycuda.VERSION_TEXT)) - - #Print some info about CUDA - self.logger.info("CUDA version %s", str(cuda.get_version())) - self.logger.info("Driver version %s", str(cuda.get_driver_version())) - - if device is None: - device = 0 - - self.cuda_device = cuda.Device(device) - self.logger.info("Using device %d/%d '%s' (%s) GPU", device, cuda.Device.count(), self.cuda_device.name(), self.cuda_device.pci_bus_id()) - self.logger.debug(" => compute capability: %s", str(self.cuda_device.compute_capability())) - - # Create the CUDA context - if context_flags is None: - context_flags=cuda.ctx_flags.SCHED_AUTO - - self.cuda_context = self.cuda_device.make_context(flags=context_flags) - - free, total = cuda.mem_get_info() - self.logger.debug(" => memory: %d / %d MB available", int(free/(1024*1024)), int(total/(1024*1024))) - - self.logger.info("Created context handle <%s>", str(self.cuda_context.handle)) - - #Create cache dir for cubin files - self.cache_path = os.path.join(self.module_path, "cuda_cache") - if (self.use_cache): - if not os.path.isdir(self.cache_path): - os.mkdir(self.cache_path) - self.logger.info("Using CUDA cache dir %s", self.cache_path) - - self.autotuner = None - if (autotuning): - self.logger.info("Autotuning enabled. It may take several minutes to run the code the first time: have patience") - self.autotuner = Autotuner.Autotuner() - - - def __del__(self, *args): - self.logger.info("Cleaning up CUDA context handle <%s>", str(self.cuda_context.handle)) - - # Loop over all contexts in stack, and remove "this" - other_contexts = [] - while (cuda.Context.get_current() != None): - context = cuda.Context.get_current() - if (context.handle != self.cuda_context.handle): - self.logger.debug("<%s> Popping <%s> (*not* ours)", str(self.cuda_context.handle), str(context.handle)) - other_contexts = [context] + other_contexts - cuda.Context.pop() - else: - self.logger.debug("<%s> Popping <%s> (ours)", str(self.cuda_context.handle), str(context.handle)) - cuda.Context.pop() - - # Add all the contexts we popped that were not our own - for context in other_contexts: - self.logger.debug("<%s> Pushing <%s>", str(self.cuda_context.handle), str(context.handle)) - cuda.Context.push(context) - - self.logger.debug("<%s> Detaching", str(self.cuda_context.handle)) - self.cuda_context.detach() - - - def __str__(self): - return "CudaContext id " + str(self.cuda_context.handle) - - - def hash_kernel(kernel_filename, include_dirs): - # Generate a kernel ID for our caches - num_includes = 0 - max_includes = 100 - kernel_hasher = hashlib.md5() - logger = logging.getLogger(__name__) - - # Loop over file and includes, and check if something has changed - files = [kernel_filename] - while len(files): - - if (num_includes > max_includes): - raise("Maximum number of includes reached - circular include in {:}?".format(kernel_filename)) - - filename = files.pop() - - #logger.debug("Hashing %s", filename) - - modified = os.path.getmtime(filename) - - # Open the file - with io.open(filename, "r") as file: - - # Search for #inclue and also hash the file - file_str = file.read() - kernel_hasher.update(file_str.encode('utf-8')) - kernel_hasher.update(str(modified).encode('utf-8')) - - #Find all includes - includes = re.findall('^\W*#include\W+(.+?)\W*$', file_str, re.M) - - # Loop over everything that looks like an include - for include_file in includes: - - #Search through include directories for the file - file_path = os.path.dirname(filename) - for include_path in [file_path] + include_dirs: - - # If we find it, add it to list of files to check - temp_path = os.path.join(include_path, include_file) - if (os.path.isfile(temp_path)): - files = files + [temp_path] - num_includes = num_includes + 1 #For circular includes... - break - - return kernel_hasher.hexdigest() - - - """ - Reads a text file and creates an OpenCL kernel from that - """ - def get_module(self, kernel_filename, - include_dirs=[], \ - defines={}, \ - compile_args={'no_extern_c', True}, jit_compile_args={}): - """ - Helper function to print compilation output - """ - def cuda_compile_message_handler(compile_success_bool, info_str, error_str): - self.logger.debug("Compilation returned %s", str(compile_success_bool)) - if info_str: - self.logger.debug("Info: %s", info_str) - if error_str: - self.logger.debug("Error: %s", error_str) - - kernel_filename = os.path.normpath(kernel_filename) - kernel_path = os.path.abspath(os.path.join(self.module_path, kernel_filename)) - #self.logger.debug("Getting %s", kernel_filename) - - # Create a hash of the kernel options - options_hasher = hashlib.md5() - options_hasher.update(str(defines).encode('utf-8') + str(compile_args).encode('utf-8')); - options_hash = options_hasher.hexdigest() - - # Create hash of kernel souce - source_hash = CudaContext.hash_kernel( \ - kernel_path, \ - include_dirs=[self.module_path] + include_dirs) - - # Create final hash - root, ext = os.path.splitext(kernel_filename) - kernel_hash = root \ - + "_" + source_hash \ - + "_" + options_hash \ - + ext - cached_kernel_filename = os.path.join(self.cache_path, kernel_hash) - - # If we have the kernel in our hashmap, return it - if (kernel_hash in self.modules.keys()): - self.logger.debug("Found kernel %s cached in hashmap (%s)", kernel_filename, kernel_hash) - return self.modules[kernel_hash] - - # If we have it on disk, return it - elif (self.use_cache and os.path.isfile(cached_kernel_filename)): - self.logger.debug("Found kernel %s cached on disk (%s)", kernel_filename, kernel_hash) - - with io.open(cached_kernel_filename, "rb") as file: - file_str = file.read() - module = cuda.module_from_buffer(file_str, message_handler=cuda_compile_message_handler, **jit_compile_args) - - self.modules[kernel_hash] = module - return module - - # Otherwise, compile it from source - else: - self.logger.debug("Compiling %s (%s)", kernel_filename, kernel_hash) - - #Create kernel string - kernel_string = "" - for key, value in defines.items(): - kernel_string += "#define {:s} {:s}\n".format(str(key), str(value)) - kernel_string += '#include "{:s}"'.format(os.path.join(self.module_path, kernel_filename)) - if (self.use_cache): - cached_kernel_dir = os.path.dirname(cached_kernel_filename) - if not os.path.isdir(cached_kernel_dir): - os.mkdir(cached_kernel_dir) - with io.open(cached_kernel_filename + ".txt", "w") as file: - file.write(kernel_string) - - - with Common.Timer("compiler") as timer: - import warnings - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message="The CUDA compiler succeeded, but said the following:\nkernel.cu", category=UserWarning) - cubin = cuda_compiler.compile(kernel_string, include_dirs=include_dirs, cache_dir=False, **compile_args) - module = cuda.module_from_buffer(cubin, message_handler=cuda_compile_message_handler, **jit_compile_args) - if (self.use_cache): - with io.open(cached_kernel_filename, "wb") as file: - file.write(cubin) - - self.modules[kernel_hash] = module - return module - - """ - Clears the kernel cache (useful for debugging & development) - """ - def clear_kernel_cache(self): - self.logger.debug("Clearing cache") - self.modules = {} - gc.collect() - - """ - Synchronizes all streams etc - """ - def synchronize(self): - self.cuda_context.synchronize() \ No newline at end of file diff --git a/GPUSimulators/EE2D_KP07_dimsplit.py b/GPUSimulators/EE2D_KP07_dimsplit.py index 935eb90..4d46c3e 100644 --- a/GPUSimulators/EE2D_KP07_dimsplit.py +++ b/GPUSimulators/EE2D_KP07_dimsplit.py @@ -19,6 +19,9 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . """ +import os +import sys + #Import packages we need from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition @@ -27,13 +30,21 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc - - - - - - - +from hip import hipblas + +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the Forward-Backward linear scheme @@ -56,20 +67,6 @@ class EE2D_KP07_dimsplit (BaseSimulator): p: pressure """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - def __init__(self, context, rho, rho_u, rho_v, E, @@ -94,133 +91,210 @@ class EE2D_KP07_dimsplit (BaseSimulator): self.gamma = np.float32(gamma) self.theta = np.float32(theta) - #Get kernels - #module = context.get_module("cuda/EE2D_KP07_dimsplit.cu", - # defines={ - # 'BLOCK_WIDTH': self.block_size[0], - # 'BLOCK_HEIGHT': self.block_size[1] - # }, - # compile_args={ - # 'no_extern_c': True, - # 'options': ["--use_fast_math"], - # }, - # jit_compile_args={}) - #self.kernel = module.get_function("KP07DimsplitKernel") - #self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") - # - kernel_file_path = os.path.abspath(os.path.join('cuda', 'EE2D_KP07_dimsplit.cu.hip')) + + #Get cuda kernels + """ Cuda + module = context.get_module("cuda/EE2D_KP07_dimsplit.cu.hip", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + #compile and load to the device + self.kernel = module.get_function("KP07DimsplitKernel") + self.kernel.prepare("iiffffffiiPiPiPiPiPiPiPiPiPiiii") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + + #source code + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'EE2D_KP07_dimsplit.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + #EulerCommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'EulerCommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + #limiters.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h')) + with open(header_file_path, 'r') as file: + header_limiters = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"EulerCommon.h",b"limiters.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() + + #extract the arch of the device props = hip.hipDeviceProp_t() - hip_check(hip.hipGetDeviceProperties(props,0)) + hip_check(hip.hipGetDeviceProperties(props,0)) #only one device 0 arch = props.gcnArchName print(f"Compiling kernel for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named "KP07DimsplitKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07DimsplitKernel")) + print() + print("--Get the device kernel *KP07DimsplitKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, nx, ny, 2, 2, [rho, rho_u, rho_v, E]) + self.u1 = Common.ArakawaA2D(self.stream, nx, ny, 2, 2, [None, None, None, None]) + #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) # init device array cfl_data - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) - dt_x = np.min(self.dx / (np.abs(rho_u/rho) + np.sqrt(gamma*rho))) dt_y = np.min(self.dy / (np.abs(rho_v/rho) + np.sqrt(gamma*rho))) self.dt = min(dt_x, dt_y) - self.cfl_data.fill(self.dt, stream=self.stream) - - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + + def substep(self, dt, step_number, external=True, internal=True): self.substepDimsplit(0.5*dt, step_number, external, internal) def substepDimsplit(self, dt, substep, external, internal): + + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + u03_strides0 = self.u0[3].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize + u13_strides0 = self.u1[3].data.shape[0]*np.float32().itemsize + if external and internal: + #print("COMPLETE DOMAIN (dt=" + str(dt) + ")") -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# self.nx, self.ny) - - #launch kernel + """ Cuda + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + self.nx, self.ny) + """ + + #hip.hipModuleLaunchKernel(f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, stream, kernelParams, extra) + + #The argument grid/block requires 3 components x,y and z. in 2D z=1. hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, - 0, 0, - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) + ctypes.c_int(0), ctypes.c_int(0), + ctypes.c_int(self.nx), ctypes.c_int(self.ny), ) ) + ) - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--External & Internal: Launching Kernel is ok") + #print("--External & Internal: Launching Kernel is ok") return @@ -229,243 +303,250 @@ class EE2D_KP07_dimsplit (BaseSimulator): # XXX: Corners are treated twice! # ################################### - ns_grid_size = (self.grid_size[0], 1) - + ns_grid_size = (self.grid_size[0], 1, 1) # NORTH # (x0, y0) x (x1, y1) # (0, ny-y_halo) x (nx, ny) -# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, self.ny - int(self.u0[0].y_halo), -# self.nx, self.ny) - + """ Cuda + self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, self.ny - int(self.u0[0].y_halo), + self.nx, self.ny) + """ + hip_check( hip.hipModuleLaunchKernel( - kernel, - *ns_grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *ns_grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, - 0, ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo), - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) + ctypes.c_int(0), ctypes.c_int(self.ny - self.u0[0].y_halo), + ctypes.c_int(self.nx), ctypes.c_int(self.ny), ) ) + ) + hip_check(hip.hipStreamSynchronize(self.stream)) + #print() + #print("--I m at the NORTH:") + #print() # SOUTH # (x0, y0) x (x1, y1) # (0, 0) x (nx, y_halo) -# self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# self.nx, int(self.u0[0].y_halo)) - + """ Cuda + self.kernel.prepared_async_call(ns_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + self.nx, int(self.u0[0].y_halo)) + """ + hip_check( hip.hipModuleLaunchKernel( - kernel, - *ns_grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *ns_grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, - 0, 0, - ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo) - ) + ctypes.c_int(0), ctypes.c_int(0), + ctypes.c_int(self.nx), ctypes.c_int(self.u0[0].y_halo), ) ) - - - we_grid_size = (1, self.grid_size[1]) + ) + hip_check(hip.hipStreamSynchronize(self.stream)) + + we_grid_size = (1, self.grid_size[1], 1) # WEST # (x0, y0) x (x1, y1) # (0, 0) x (x_halo, ny) -# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# 0, 0, -# int(self.u0[0].x_halo), self.ny) + """ Cuda + self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + 0, 0, + int(self.u0[0].x_halo), self.ny) + """ hip_check( hip.hipModuleLaunchKernel( - kernel, - *we_grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *we_grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, - 0, 0, - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ) + ctypes.c_int(0), ctypes.c_int(0), + ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny), ) ) + ) + hip_check(hip.hipStreamSynchronize(self.stream)) # EAST # (x0, y0) x (x1, y1) # (nx-x_halo, 0) x (nx, ny) -# self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.gamma, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u0[3].data.gpudata, self.u0[3].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.u1[3].data.gpudata, self.u1[3].data.strides[0], -# self.cfl_data.gpudata, -# self.nx - int(self.u0[0].x_halo), 0, -# self.nx, self.ny) + """ Cuda + self.kernel.prepared_async_call(we_grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.gamma, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u0[3].data.gpudata, self.u0[3].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.u1[3].data.gpudata, self.u1[3].data.strides[0], + self.cfl_data.gpudata, + self.nx - int(self.u0[0].x_halo), 0, + self.nx, self.ny) + """ hip_check( hip.hipModuleLaunchKernel( - kernel, - *we_grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *we_grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, - ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), 0, - ctypes.c_int(self.nx), ctypes.c_int(self.ny) - ) + ctypes.c_int(self.nx - self.u0[0].x_halo), ctypes.c_int(0), + ctypes.c_int(self.nx), ctypes.c_int(self.ny), ) ) + ) - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--External and not Internal: Launching Kernel is ok") - +# print("--External and not Internal: Launching Kernel is ok") return if internal and not external: - + # INTERNAL DOMAIN # (x0, y0) x (x1, y1) # (x_halo, y_halo) x (nx - x_halo, ny - y_halo) + """ self.kernel.prepared_async_call(self.grid_size, self.block_size, self.internal_stream, self.nx, self.ny, self.dx, self.dy, dt, @@ -485,45 +566,40 @@ class EE2D_KP07_dimsplit (BaseSimulator): self.cfl_data.gpudata, int(self.u0[0].x_halo), int(self.u0[0].y_halo), self.nx - int(self.u0[0].x_halo), self.ny - int(self.u0[0].y_halo)) - + """ hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.internal_stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.gamma), ctypes.c_float(self.theta), ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u0[3].data), ctypes.c_float(self.u0[3].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - ctypes.c_float(self.u1[3].data), ctypes.c_float(self.u1[3].data.strides[0]), + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u0[3].data, ctypes.c_int(u03_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.u1[3].data, ctypes.c_int(u13_strides0), self.cfl_data, ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.u0[0].y_halo), - ctypes.c_int(self.nx) - ctypes.c_int(self.u0[0].x_halo), ctypes.c_int(self.ny) - ctypes.c_int(self.u0[0].y_halo) - ) + ctypes.c_int(self.nx - self.u0[0].x_halo), ctypes.c_int(self.ny - self.u0[0].y_halo), ) ) + ) - hip_check(hip.hipDeviceSynchronize()) - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Internal and not External: Launching Kernel is ok") + # print("--Internal and not External: Launching Kernel is ok") return def swapBuffers(self): diff --git a/GPUSimulators/FORCE.py b/GPUSimulators/FORCE.py index 092711a..fbdf73e 100644 --- a/GPUSimulators/FORCE.py +++ b/GPUSimulators/FORCE.py @@ -25,16 +25,24 @@ from GPUSimulators import Simulator, Common from GPUSimulators.Simulator import BaseSimulator, BoundaryCondition import numpy as np import ctypes + #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - - - - - - - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations @@ -53,19 +61,6 @@ class FORCE (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, context, @@ -87,25 +82,55 @@ class FORCE (Simulator.BaseSimulator): block_width, block_height) self.g = np.float32(g) - #Get kernels -# module = context.get_module("cuda/SWE2D_FORCE.cu.hip", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("FORCEKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_FORCE.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("FORCEKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_FORCE.cu')) + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_FORCE.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"FORCEKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -113,20 +138,38 @@ class FORCE (Simulator.BaseSimulator): print(f"Compiling kernel .FORCEKernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"FORCEKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + #Get the device kernel named named "FORCEKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"FORCEKernel")) + + print() + print("--Get the device kernel *FORCEKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -138,65 +181,79 @@ class FORCE (Simulator.BaseSimulator): 1, 1, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) -# self.u0, self.u1 = self.u1, self.u0 - + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize + #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) - - hip_check(hip.hipDeviceSynchronize()) + ) + self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .FORCEKernel. is ok") + #print("--Launching Kernel .FORCEKernel. is ok") def getOutput(self): return self.u0 diff --git a/GPUSimulators/HLL.py b/GPUSimulators/HLL.py index 792d3c6..3ef9737 100644 --- a/GPUSimulators/HLL.py +++ b/GPUSimulators/HLL.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the HLL flux +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -27,10 +28,21 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - - - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the Harten-Lax -van Leer approximate Riemann solver @@ -49,22 +61,9 @@ class HLL (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, - context, + context, h0, hu0, hv0, nx, ny, dx, dy, @@ -80,28 +79,58 @@ class HLL (Simulator.BaseSimulator): boundary_conditions, cfl_scale, 1, - block_width, block_height); + block_width, block_height) self.g = np.float32(g) - #Get kernels -# module = context.get_module("cuda/SWE2D_HLL.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("HLLKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL.cu.hip')) + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_HLL.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("HLLKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_HLL.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLLKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -109,19 +138,38 @@ class HLL (Simulator.BaseSimulator): print(f"Compiling kernel .HLLKernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"HLLKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "FORCEKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"HLLKernel")) + + print() + print("--Get the device kernel *HLLKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -133,71 +181,87 @@ class HLL (Simulator.BaseSimulator): 1, 1, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize + #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) + ) - hip_check(hip.hipDeviceSynchronize()) - self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .HLLKernel. is ok") + + #print("--Launching Kernel .HLLKernel. is ok") def getOutput(self): return self.u0 - + def check(self): self.u0.check() self.u1.check() - + # computing min with hipblas: the output is an index def min_hipblas(self, num_elements, cfl_data, stream): num_bytes = num_elements * np.dtype(np.float32).itemsize @@ -232,4 +296,4 @@ class HLL (Simulator.BaseSimulator): def computeDt(self): #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) - return max_dt*0.5 + return max_dt*0.5 diff --git a/GPUSimulators/HLL2.py b/GPUSimulators/HLL2.py index b5c0dc0..9f83417 100644 --- a/GPUSimulators/HLL2.py +++ b/GPUSimulators/HLL2.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the 2nd order HLL flux +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -27,15 +28,24 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc - - - - - +from hip import hipblas +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ -Class that solves the SW equations using the Forward-Backward linear scheme +Class that solves the SW equations """ class HLL2 (Simulator.BaseSimulator): @@ -51,27 +61,14 @@ class HLL2 (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - + def __init__(self, context, h0, hu0, hv0, nx, ny, dx, dy, g, - theta=1.8, + theta=1.8, cfl_scale=0.9, boundary_conditions=BoundaryCondition(), block_width=16, block_height=16): @@ -83,29 +80,63 @@ class HLL2 (Simulator.BaseSimulator): boundary_conditions, cfl_scale, 2, - block_width, block_height); + block_width, block_height) self.g = np.float32(g) self.theta = np.float32(theta) - - #Get kernels -# module = context.get_module("cuda/SWE2D_HLL2.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("HLL2Kernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_HLL2.cu.hip')) + + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_HLL2.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("HLL2Kernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_HLL2.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #limiters.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h')) + with open(header_file_path, 'r') as file: + header_limiters = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"HLL2Kernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -113,19 +144,38 @@ class HLL2 (Simulator.BaseSimulator): print(f"Compiling kernel .HLL2Kernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"HLL2Kernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "FORCEKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"HLL2Kernel")) + + print() + print("--Get the device kernel *HLL2Kernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -137,70 +187,87 @@ class HLL2 (Simulator.BaseSimulator): 2, 2, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): self.substepDimsplit(dt*0.5, step_number) - - def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + + def substepDimsplit(self, dt, substep): + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.theta), - ctypes.c_int(substep), + ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) + ) - hip_check(hip.hipDeviceSynchronize()) self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .HLL2Kernel. is ok") + + #print("--Launching Kernel .HLL2Kernel. is ok") def getOutput(self): return self.u0 @@ -208,7 +275,7 @@ class HLL2 (Simulator.BaseSimulator): def check(self): self.u0.check() self.u1.check() - + # computing min with hipblas: the output is an index def min_hipblas(self, num_elements, cfl_data, stream): num_bytes = num_elements * np.dtype(np.float32).itemsize @@ -244,4 +311,3 @@ class HLL2 (Simulator.BaseSimulator): #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) return max_dt*0.5 - diff --git a/GPUSimulators/IPythonMagic.py b/GPUSimulators/IPythonMagic.py index 92baeb8..ae10092 100644 --- a/GPUSimulators/IPythonMagic.py +++ b/GPUSimulators/IPythonMagic.py @@ -29,6 +29,19 @@ from hip import hip, hiprtc from GPUSimulators import Common, CudaContext +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result @magics_class class MagicCudaContext(Magics): @@ -42,19 +55,6 @@ class MagicCudaContext(Magics): '--no_cache', '-nc', action="store_true", help='Disable caching of kernels') @magic_arguments.argument( '--no_autotuning', '-na', action="store_true", help='Disable autotuning of kernels') - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def cuda_context_handler(self, line): args = magic_arguments.parse_argstring(self.cuda_context_handler, line) diff --git a/GPUSimulators/KP07.py b/GPUSimulators/KP07.py index 93ce5e9..2ce9d4e 100644 --- a/GPUSimulators/KP07.py +++ b/GPUSimulators/KP07.py @@ -1,12 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -32,8 +28,21 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the Forward-Backward linear scheme @@ -52,19 +61,6 @@ class KP07 (Simulator.BaseSimulator): dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, context, @@ -72,7 +68,7 @@ class KP07 (Simulator.BaseSimulator): nx, ny, dx, dy, g, - theta=1.3, + theta=1.3, cfl_scale=0.9, order=2, boundary_conditions=BoundaryCondition(), @@ -84,31 +80,65 @@ class KP07 (Simulator.BaseSimulator): dx, dy, boundary_conditions, cfl_scale, - order, - block_width, block_height); - self.g = np.float32(g) - self.theta = np.float32(theta) + order, + block_width, block_height) + self.g = np.float32(g) + self.theta = np.float32(theta) self.order = np.int32(order) - #Get kernels -# module = context.get_module("cuda/SWE2D_KP07.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("KP07Kernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07.cu.hip')) + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_KP07.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("KP07Kernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_KP07.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #limiters.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h')) + with open(header_file_path, 'r') as file: + header_limiters = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07Kernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -116,19 +146,38 @@ class KP07 (Simulator.BaseSimulator): print(f"Compiling kernel .KP07Kernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07Kernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "FORCEKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07Kernel")) + + print() + print("--Get the device kernel *KP07Kernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -140,73 +189,87 @@ class KP07 (Simulator.BaseSimulator): 2, 2, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - - - def substep(self, dt, step_number): - self.substepRK(dt, step_number) + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + + def substep(self, dt, step_number): + self.substepRK(dt, step_number) - def substepRK(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# Simulator.stepOrderToCodedInt(step=substep, order=self.order), -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + Simulator.stepOrderToCodedInt(step=substep, order=self.order), + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.theta), - Simulator.stepOrderToCodedInt(step=substep, order=self.order), + Simulator.stepOrderToCodedInt(step=substep, order=self.order), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) - - hip_check(hip.hipDeviceSynchronize()) + ) self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .KP07Kernel. is ok") + + #print("--Launching Kernel .KP07Kernel. is ok") def getOutput(self): return self.u0 @@ -214,7 +277,7 @@ class KP07 (Simulator.BaseSimulator): def check(self): self.u0.check() self.u1.check() - + # computing min with hipblas: the output is an index def min_hipblas(self, num_elements, cfl_data, stream): num_bytes = num_elements * np.dtype(np.float32).itemsize @@ -247,6 +310,6 @@ class KP07 (Simulator.BaseSimulator): return min_value def computeDt(self): - max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); + max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) return max_dt*0.5**(self.order-1) diff --git a/GPUSimulators/KP07_dimsplit.py b/GPUSimulators/KP07_dimsplit.py index 0a5cfc7..79dc0aa 100644 --- a/GPUSimulators/KP07_dimsplit.py +++ b/GPUSimulators/KP07_dimsplit.py @@ -1,12 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the Kurganov-Petrova numerical scheme -for the shallow water equations, described in -A. Kurganov & Guergana Petrova -A Second-Order Well-Balanced Positivity Preserving Central-Upwind -Scheme for the Saint-Venant System Communications in Mathematical -Sciences, 5 (2007), 133-160. +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -32,14 +28,26 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the dimentionally split KP07 scheme """ -class KP07_dimsplit(Simulator.BaseSimulator): +class KP07_dimsplit (Simulator.BaseSimulator): """ Initialization routine @@ -54,27 +62,13 @@ class KP07_dimsplit(Simulator.BaseSimulator): g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - def __init__(self, context, h0, hu0, hv0, nx, ny, dx, dy, g, - theta=1.3, + theta=1.3, cfl_scale=0.9, boundary_conditions=BoundaryCondition(), block_width=16, block_height=16): @@ -85,32 +79,66 @@ class KP07_dimsplit(Simulator.BaseSimulator): dx, dy, boundary_conditions, cfl_scale, - 2, + 2, block_width, block_height) self.gc_x = 2 self.gc_y = 2 - self.g = np.float32(g) + self.g = np.float32(g) self.theta = np.float32(theta) - #Get kernels -# module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("KP07DimsplitKernel") -# self.kernel.prepare("iifffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_KP07_dimsplit.cu.hip')) + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_KP07_dimsplit.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("KP07DimsplitKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_KP07_dimsplit.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #limiters.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'limiters.h')) + with open(header_file_path, 'r') as file: + header_limiters = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"KP07DimsplitKernel", 3, [header_common.encode(),header_EulerCommon.encode(),header_limiters.encode()], [b"common.h",b"SWECommon.h",b"limiters.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -118,19 +146,38 @@ class KP07_dimsplit(Simulator.BaseSimulator): print(f"Compiling kernel .KP07DimsplitKernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"KP07DimsplitKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "FORCEKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"KP07DimsplitKernel")) + + print() + print("--Get the device kernel *KP07DimsplitKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -139,77 +186,94 @@ class KP07_dimsplit(Simulator.BaseSimulator): [h0, hu0, hv0]) self.u1 = Common.ArakawaA2D(self.stream, nx, ny, - self.gc_x, self.gc_y, + self.gc_x, self.gc_y, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): self.substepDimsplit(dt*0.5, step_number) - + def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.theta, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.theta, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_float(self.theta), - ctypes.c_int(substep) + ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) - - hip_check(hip.hipDeviceSynchronize()) + ) self.u0, self.u1 = self.u1, self.u0 - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .KP07DimsplitKernel. is ok") + + #print("--Launching Kernel .KP07DimsplitKernel. is ok") def getOutput(self): return self.u0 - + def check(self): self.u0.check() self.u1.check() diff --git a/GPUSimulators/LxF.py b/GPUSimulators/LxF.py index 98e54c6..6edc6b6 100644 --- a/GPUSimulators/LxF.py +++ b/GPUSimulators/LxF.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the classical Lax-Friedrichs numerical -scheme for the shallow water equations +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -28,10 +28,21 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - - - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the Lax Friedrichs scheme @@ -51,20 +62,6 @@ class LxF (Simulator.BaseSimulator): g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - def __init__(self, context, h0, hu0, hv0, @@ -72,7 +69,7 @@ class LxF (Simulator.BaseSimulator): dx, dy, g, cfl_scale=0.9, - boundary_conditions=BoundaryCondition(), + boundary_conditions=BoundaryCondition(), block_width=16, block_height=16): # Call super constructor @@ -82,28 +79,58 @@ class LxF (Simulator.BaseSimulator): boundary_conditions, cfl_scale, 1, - block_width, block_height); + block_width, block_height) self.g = np.float32(g) - # Get kernels -# module = context.get_module("cuda/SWE2D_LxF.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("LxFKernel") -# self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_LxF.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("LxFKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_LxF.cu.hip')) + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_LxF.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"LxFKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -111,19 +138,38 @@ class LxF (Simulator.BaseSimulator): print(f"Compiling kernel .LxFKernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"LxFKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "LxFKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"LxFKernel")) + + print() + print("--Get the device kernel *LxFKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -135,72 +181,87 @@ class LxF (Simulator.BaseSimulator): 1, 1, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) - - hip_check(hip.hipDeviceSynchronize()) + ) self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .LxFKernel. is ok") + + #print("--Launching Kernel .LxFKernel. is ok") def getOutput(self): return self.u0 - + def check(self): self.u0.check() self.u1.check() - + # computing min with hipblas: the output is an index def min_hipblas(self, num_elements, cfl_data, stream): num_bytes = num_elements * np.dtype(np.float32).itemsize @@ -231,7 +292,7 @@ class LxF (Simulator.BaseSimulator): hip_check(hip.hipStreamDestroy(stream)) hip_check(hip.hipFree(cfl_data)) return min_value - + def computeDt(self): #max_dt = gpuarray.min(self.cfl_data, stream=self.stream).get(); max_dt = self.min_hipblas(self.cfl_data.size, self.cfl_data, self.stream) diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py index f13de52..c792fdd 100644 --- a/GPUSimulators/MPISimulator.py +++ b/GPUSimulators/MPISimulator.py @@ -30,6 +30,19 @@ import time #import nvtx from hip import hip, hiprtc +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result class MPIGrid(object): """ @@ -206,19 +219,6 @@ class MPISimulator(Simulator.BaseSimulator): """ Class which handles communication between simulators on different MPI nodes """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, sim, grid): self.profiling_data_mpi = { 'start': {}, 'end': {} } @@ -306,58 +306,73 @@ class MPISimulator(Simulator.BaseSimulator): #Note that east and west also transfer ghost cells #whilst north/south only transfer internal cells #Reuses the width/height defined in the read-extets above - ##self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) + """ + self.in_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty((self.nvars, self.read_e[3], self.read_e[2]), dtype=np.float32) + self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) + self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) + self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) + """ - ##self.in_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty((self.nvars, self.read_w[3], self.read_w[2]), dtype=np.float32) - ##self.in_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty((self.nvars, self.read_n[3], self.read_n[2]), dtype=np.float32) - ##self.in_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty((self.nvars, self.read_s[3], self.read_s[2]), dtype=np.float32) - - self.in_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + #HIP + self.in_e = np.zeros((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) num_bytes_e = self.in_e.size * self.in_e.itemsize #hipHostMalloc allocates pinned host memory which is mapped into the address space of all GPUs in the system, the memory can be accessed directly by the GPU device #hipHostMallocDefault:Memory is mapped and portable (default allocation) #hipHostMallocPortable: memory is explicitely portable across different devices - self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + #self.in_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.in_e, hip.hipHostMallocPortable)) - self.in_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + #print("--hip.hipGetDeviceFlags():", hip.hipGetDeviceFlags()) + + self.in_w = np.zeros((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) num_bytes_w = self.in_w.size * self.in_w.itemsize - self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + #self.in_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.in_w, hip.hipHostMallocPortable)) - self.in_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + self.in_n = np.zeros((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) num_bytes_n = self.in_n.size * self.in_n.itemsize - self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + #self.in_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.in_n, hip.hipHostMallocPortable)) - self.in_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + self.in_s = np.zeros((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) num_bytes_s = self.in_s.size * self.in_s.itemsize - self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + #self.in_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.in_s, hip.hipHostMallocPortable)) #Allocate data for sending - #self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) - #self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) - #self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) - #self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) - - self.out_e = np.empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) + """ + self.out_e = cuda.pagelocked_empty((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) #np.empty_like(self.in_e) + self.out_w = cuda.pagelocked_empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) #np.empty_like(self.in_w) + self.out_n = cuda.pagelocked_empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) #np.empty_like(self.in_n) + self.out_s = cuda.pagelocked_empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) #np.empty_like(self.in_s) + """ + + self.out_e = np.zeros((int(self.nvars), int(self.read_e[3]), int(self.read_e[2])), dtype=np.float32) num_bytes_e = self.out_e.size * self.out_e.itemsize - self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + #self.out_e = hip_check(hip.hipHostMalloc(num_bytes_e,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.out_e, hip.hipHostMallocPortable)) - self.out_w = np.empty((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) + self.out_w = np.zeros((int(self.nvars), int(self.read_w[3]), int(self.read_w[2])), dtype=np.float32) num_bytes_w = self.out_w.size * self.out_w.itemsize - self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + #self.out_w = hip_check(hip.hipHostMalloc(num_bytes_w,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.out_w, hip.hipHostMallocPortable)) - self.out_n = np.empty((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) + self.out_n = np.zeros((int(self.nvars), int(self.read_n[3]), int(self.read_n[2])), dtype=np.float32) num_bytes_n = self.out_n.size * self.out_n.itemsize - self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + #self.out_n = hip_check(hip.hipHostMalloc(num_bytes_n,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.out_n, hip.hipHostMallocPortable)) - self.out_s = np.empty((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) + self.out_s = np.zeros((int(self.nvars), int(self.read_s[3]), int(self.read_s[2])), dtype=np.float32) num_bytes_s = self.out_s.size * self.out_s.itemsize - self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + #self.out_s = hip_check(hip.hipHostMalloc(num_bytes_s,hip.hipHostMallocPortable)) + #hip_check(hip.hipHostGetDevicePointer(self.out_s, hip.hipHostMallocPortable)) self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) self.full_exchange() - sim.context.synchronize() + #hip_check(hip.hipDeviceSynchronize()) + #sim.context.synchronize() def substep(self, dt, step_number): diff --git a/GPUSimulators/SHMEMSimulator.py b/GPUSimulators/SHMEMSimulator.py index 8417d7e..73f28f0 100644 --- a/GPUSimulators/SHMEMSimulator.py +++ b/GPUSimulators/SHMEMSimulator.py @@ -29,24 +29,25 @@ from hip import hip, hiprtc import time +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + class SHMEMSimulator(Simulator.BaseSimulator): """ Class which handles communication and synchronization between simulators in different contexts (presumably on different GPUs) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, sims, grid): self.logger = logging.getLogger(__name__) diff --git a/GPUSimulators/SHMEMSimulatorGroup.py b/GPUSimulators/SHMEMSimulatorGroup.py index c9dc30f..b7cb68a 100644 --- a/GPUSimulators/SHMEMSimulatorGroup.py +++ b/GPUSimulators/SHMEMSimulatorGroup.py @@ -29,24 +29,25 @@ from hip import hip, hiprtc import time +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + class SHMEMGrid(object): """ Class which represents an SHMEM grid of GPUs. Facilitates easy communication between neighboring subdomains in the grid. Contains one CUDA context per subdomain. """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result def __init__(self, ngpus=None, ndims=2): self.logger = logging.getLogger(__name__) diff --git a/GPUSimulators/Simulator.py b/GPUSimulators/Simulator.py index b804d79..a6a41c2 100644 --- a/GPUSimulators/Simulator.py +++ b/GPUSimulators/Simulator.py @@ -22,6 +22,7 @@ along with this program. If not, see . #Import packages we need import numpy as np +import math import logging from enum import IntEnum @@ -34,6 +35,20 @@ from hip import hip, hiprtc from GPUSimulators import Common +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result + class BoundaryCondition(object): """ Class for holding boundary conditions for global boundaries @@ -102,15 +117,6 @@ class BoundaryCondition(object): class BaseSimulator(object): - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - return result - def __init__(self, context, nx, ny, @@ -155,14 +161,19 @@ class BaseSimulator(object): block_width = int(peak_configuration["block_width"]) block_height = int(peak_configuration["block_height"]) self.logger.debug("Used autotuning to get block size [%d x %d]", block_width, block_height) - + #Compute kernel launch parameters + """ self.block_size = (block_width, block_height, 1) self.grid_size = ( int(np.ceil(self.nx / float(self.block_size[0]))), int(np.ceil(self.ny / float(self.block_size[1]))) ) - + """ + self.block_size = hip.dim3(block_width, block_height) + #self.grid_size = hip.dim3(math.ceil(self.nx/block_width),math.ceil(self.ny/block_height)) + self.grid_size = hip.dim3(math.ceil((self.nx+block_width-1)/block_width),math.ceil((self.ny+block_height-1)/block_height)) + #Create a CUDA stream #self.stream = cuda.Stream() #self.internal_stream = cuda.Stream() diff --git a/GPUSimulators/WAF.py b/GPUSimulators/WAF.py index 7e2763c..b7fd664 100644 --- a/GPUSimulators/WAF.py +++ b/GPUSimulators/WAF.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -This python module implements the Weighted average flux (WAF) described in -E. Toro, Shock-Capturing methods for free-surface shallow flows, 2001 +This python module implements the FORCE flux +for the shallow water equations Copyright (C) 2016 SINTEF ICT @@ -28,8 +28,21 @@ import ctypes #from pycuda import gpuarray from hip import hip,hiprtc +from hip import hipblas - +def hip_check(call_result): + err = call_result[0] + result = call_result[1:] + if len(result) == 1: + result = result[0] + if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: + raise RuntimeError(str(err)) + elif ( + isinstance(err, hiprtc.hiprtcResult) + and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS + ): + raise RuntimeError(str(err)) + return result """ Class that solves the SW equations using the Forward-Backward linear scheme @@ -49,22 +62,8 @@ class WAF (Simulator.BaseSimulator): g: Gravitational accelleration (9.81 m/s^2) """ - def hip_check(call_result): - err = call_result[0] - result = call_result[1:] - if len(result) == 1: - result = result[0] - if isinstance(err, hip.hipError_t) and err != hip.hipError_t.hipSuccess: - raise RuntimeError(str(err)) - elif ( - isinstance(err, hiprtc.hiprtcResult) - and err != hiprtc.hiprtcResult.HIPRTC_SUCCESS - ): - raise RuntimeError(str(err)) - return result - def __init__(self, - context, + context, h0, hu0, hv0, nx, ny, dx, dy, @@ -80,28 +79,58 @@ class WAF (Simulator.BaseSimulator): boundary_conditions, cfl_scale, 2, - block_width, block_height); + block_width, block_height) self.g = np.float32(g) - #Get kernels -# module = context.get_module("cuda/SWE2D_WAF.cu", -# defines={ -# 'BLOCK_WIDTH': self.block_size[0], -# 'BLOCK_HEIGHT': self.block_size[1] -# }, -# compile_args={ -# 'no_extern_c': True, -# 'options': ["--use_fast_math"], -# }, -# jit_compile_args={}) -# self.kernel = module.get_function("WAFKernel") -# self.kernel.prepare("iiffffiiPiPiPiPiPiPiP") - - kernel_file_path = os.path.abspath(os.path.join('cuda', 'SWE2D_WAF.cu.hip')) + #Get cuda kernels + """ + module = context.get_module("cuda/SWE2D_WAF.cu", + defines={ + 'BLOCK_WIDTH': self.block_size[0], + 'BLOCK_HEIGHT': self.block_size[1] + }, + compile_args={ + 'no_extern_c': True, + 'options': ["--use_fast_math"], + }, + jit_compile_args={}) + self.kernel = module.get_function("WAFKernel") + self.kernel.prepare("iiffffiPiPiPiPiPiPiP") + """ + + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Specify the relative path to the "cuda" directory + cuda_dir = os.path.join(current_dir, 'cuda') + + #kernel source + kernel_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWE2D_WAF.cu.hip')) with open(kernel_file_path, 'r') as file: kernel_source = file.read() - prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 0, [], [])) + #headers + #common.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'common.h')) + with open(header_file_path, 'r') as file: + header_common = file.read() + + #SWECommon.h + header_file_path = os.path.abspath(os.path.join(cuda_dir, 'SWECommon.h')) + with open(header_file_path, 'r') as file: + header_EulerCommon = file.read() + + #hip.hiprtc.hiprtcCreateProgram(const char *src, const char *name, int numHeaders, headers, includeNames) + + prog = hip_check(hiprtc.hiprtcCreateProgram(kernel_source.encode(), b"WAFKernel", 2, [header_common.encode(),header_SWECommon.encode()], [b"common.h", b"SWECommon.h"])) + + # Check if the program is created successfully + if prog is not None: + print("--This is ") + print("--HIPRTC program created successfully") + print() + else: + print("--Failed to create HIPRTC program") + print("--I stop:", err) + exit() props = hip.hipDeviceProp_t() hip_check(hip.hipGetDeviceProperties(props,0)) @@ -109,19 +138,38 @@ class WAF (Simulator.BaseSimulator): print(f"Compiling kernel .WAFKernel. for {arch}") - cflags = [b"--offload-arch="+arch] + cflags = [b"--offload-arch="+arch, b"-O2", b"-D BLOCK_WIDTH="+ str(self.block_size[0]).encode(), b"-D BLOCK_HEIGHT=" + str(self.block_size[1]).encode()] + err, = hiprtc.hiprtcCompileProgram(prog, len(cflags), cflags) + # Check if the program is compiled successfully + if err is not None: + print("--Compilation:", err) + print("--The program is compiled successfully") + else: + print("--Compilation:", err) + print("--Failed to compile the program") + print("--I stop:", err) + if err != hiprtc.hiprtcResult.HIPRTC_SUCCESS: log_size = hip_check(hiprtc.hiprtcGetProgramLogSize(prog)) log = bytearray(log_size) hip_check(hiprtc.hiprtcGetProgramLog(prog, log)) raise RuntimeError(log.decode()) + code_size = hip_check(hiprtc.hiprtcGetCodeSize(prog)) code = bytearray(code_size) hip_check(hiprtc.hiprtcGetCode(prog, code)) - module = hip_check(hip.hipModuleLoadData(code)) - kernel = hip_check(hip.hipModuleGetFunction(module, b"WAFKernel")) + #Load the code as a module + self.module = hip_check(hip.hipModuleLoadData(code)) + + #Get the device kernel named named "LxFKernel" + self.kernel = hip_check(hip.hipModuleGetFunction(self.module, b"WAFKernel")) + + print() + print("--Get the device kernel *WAFKernel* is created successfully--") + print("--kernel", self.kernel) + print() #Create data by uploading to device self.u0 = Common.ArakawaA2D(self.stream, @@ -133,69 +181,84 @@ class WAF (Simulator.BaseSimulator): 2, 2, [None, None, None]) #self.cfl_data = gpuarray.GPUArray(self.grid_size, dtype=np.float32) - data_h = np.empty(self.grid_size, dtype=np.float32) - num_bytes = data_h.size * data_h.itemsize - self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( - typestr="float32",shape=self.grid_size) dt_x = np.min(self.dx / (np.abs(hu0/h0) + np.sqrt(g*h0))) dt_y = np.min(self.dy / (np.abs(hv0/h0) + np.sqrt(g*h0))) dt = min(dt_x, dt_y) - self.cfl_data.fill(dt, stream=self.stream) - + #in HIP, the "DeviceArray" object doesn't have a 'fill' attribute + #self.cfl_data.fill(self.dt, stream=self.stream) + grid_dim_x, grid_dim_y, grid_dim_z = self.grid_size + + data_h = np.zeros((grid_dim_x, grid_dim_y), dtype=np.float32) + num_bytes = data_h.size * data_h.itemsize + data_h.fill(self.dt) + + self.cfl_data = hip_check(hip.hipMalloc(num_bytes)).configure( + typestr="float32",shape=(grid_dim_x, grid_dim_y)) + + hip_check(hip.hipMemcpyAsync(self.cfl_data,data_h,num_bytes,hip.hipMemcpyKind.hipMemcpyHostToDevice,self.stream)) + #sets the memory region pointed to by x_d to zero asynchronously + #initiates the memset operation asynchronously + #hip_check(hip.hipMemsetAsync(self.cfl_data,0,num_bytes,self.stream)) + def substep(self, dt, step_number): self.substepDimsplit(dt*0.5, step_number) - - def substepDimsplit(self, dt, substep): -# self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, -# self.nx, self.ny, -# self.dx, self.dy, dt, -# self.g, -# substep, -# self.boundary_conditions, -# self.u0[0].data.gpudata, self.u0[0].data.strides[0], -# self.u0[1].data.gpudata, self.u0[1].data.strides[0], -# self.u0[2].data.gpudata, self.u0[2].data.strides[0], -# self.u1[0].data.gpudata, self.u1[0].data.strides[0], -# self.u1[1].data.gpudata, self.u1[1].data.strides[0], -# self.u1[2].data.gpudata, self.u1[2].data.strides[0], -# self.cfl_data.gpudata) + + def substepDimsplit(self, dt, substep): + #Cuda + """ + self.kernel.prepared_async_call(self.grid_size, self.block_size, self.stream, + self.nx, self.ny, + self.dx, self.dy, dt, + self.g, + substep, + self.boundary_conditions, + self.u0[0].data.gpudata, self.u0[0].data.strides[0], + self.u0[1].data.gpudata, self.u0[1].data.strides[0], + self.u0[2].data.gpudata, self.u0[2].data.strides[0], + self.u1[0].data.gpudata, self.u1[0].data.strides[0], + self.u1[1].data.gpudata, self.u1[1].data.strides[0], + self.u1[2].data.gpudata, self.u1[2].data.strides[0], + self.cfl_data.gpudata) + self.u0, self.u1 = self.u1, self.u0 + """ + u00_strides0 = self.u0[0].data.shape[0]*np.float32().itemsize + u01_strides0 = self.u0[1].data.shape[0]*np.float32().itemsize + u02_strides0 = self.u0[2].data.shape[0]*np.float32().itemsize + + u10_strides0 = self.u1[0].data.shape[0]*np.float32().itemsize + u11_strides0 = self.u1[1].data.shape[0]*np.float32().itemsize + u12_strides0 = self.u1[2].data.shape[0]*np.float32().itemsize #launch kernel hip_check( hip.hipModuleLaunchKernel( - kernel, - *self.grid_size, - *self.block_size, - sharedMemBytes=0, + self.kernel, + *self.grid_size, #grid + *self.block_size, #block + sharedMemBytes=0, #65536, stream=self.stream, kernelParams=None, extra=( # pass kernel's arguments ctypes.c_int(self.nx), ctypes.c_int(self.ny), - ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(self.dt), + ctypes.c_float(self.dx), ctypes.c_float(self.dy), ctypes.c_float(dt), ctypes.c_float(self.g), - ctypes.c_int(substep), + ctypes.c_int(substep), ctypes.c_int(self.boundary_conditions), - ctypes.c_float(self.u0[0].data), ctypes.c_float(self.u0[0].data.strides[0]), - ctypes.c_float(self.u0[1].data), ctypes.c_float(self.u0[1].data.strides[0]), - ctypes.c_float(self.u0[2].data), ctypes.c_float(self.u0[2].data.strides[0]), - ctypes.c_float(self.u1[0].data), ctypes.c_float(self.u1[0].data.strides[0]), - ctypes.c_float(self.u1[1].data), ctypes.c_float(self.u1[1].data.strides[0]), - ctypes.c_float(self.u1[2].data), ctypes.c_float(self.u1[2].data.strides[0]), - self.cfl_data - ) + self.u0[0].data, ctypes.c_int(u00_strides0), + self.u0[1].data, ctypes.c_int(u01_strides0), + self.u0[2].data, ctypes.c_int(u02_strides0), + self.u1[0].data, ctypes.c_int(u10_strides0), + self.u1[1].data, ctypes.c_int(u11_strides0), + self.u1[2].data, ctypes.c_int(u12_strides0), + self.cfl_data, ) ) - - hip_check(hip.hipDeviceSynchronize()) + ) self.u0, self.u1 = self.u1, self.u0 - - hip_check(hip.hipModuleUnload(module)) - - hip_check(hip.hipFree(cfl_data)) - - print("--Launching Kernel .WAFKernel. is ok") + + #print("--Launching Kernel .WAFKernel. is ok") def getOutput(self): return self.u0 @@ -203,7 +266,7 @@ class WAF (Simulator.BaseSimulator): def check(self): self.u0.check() self.u1.check() - + # computing min with hipblas: the output is an index def min_hipblas(self, num_elements, cfl_data, stream): num_bytes = num_elements * np.dtype(np.float32).itemsize diff --git a/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc b/GPUSimulators/__pycache__/MPISimulator.cpython-39.pyc deleted file mode 100644 index da4daacc64c69c69b772807cf18375b7c8cf127b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11058 zcmb_iOK=-UdY%`6!4RYS*uv09;PGHib$-8;*6=r=nJH)=_{+;GFt zvX)xyE$LTlk$rZlXuop)!W;JL^2*wcoA!0T>PK$FN@ta=s&tz+7N|(iv%^*;+H|FN z$=+&p?6Mo!(i46dNq@Z)VcE!b1F?v80Eb`MvQW_pgeUE&=Gl=ao1xvRaN*X$jf zMA^RBUm%_BoZXVvSvMkprQK@Npdz5$vVoAjaSI1Gytgl5`vC)8YqhbB8fL@BHvLA! zUia)y=v6w6a~3dQzrVcp_Ivl%?5isu*zaGxd-v+f+6QlTqmMeD9iq6>~AiG?PqV}c5#m&vlg=)}QXvyj#PMS_RT)eVi zeFa`oR*GtzU+q&U3OY!(OfADd9fh~>GJkmr_(~~hLM`dyw9tj|SS=Z1PGm(6xr{JH z{;^WZ3QLUPn-c{wj;|>u#4&vHVp1H(*Ai3W1ioY9j5sM~FtQ*{iCNT)iwyW{KF$G@ zThbTXp=IN@MBHO<*8Fk}IBQEUWEKM(5SLZakx&=3gcmN@H{G(|053&eXnStB1=Kd1 zoxm?M;ev9aP0tH>-oOJduD7HQobqU{{R+&jlxgGrALUKFOL)T?l1S;Pk+!RL6-Vy^ z9rv|x{vAbVkJOLUu39&Qem@f~pvIt@DAUVQZCBgXFe2Bbab{Pk>%F|l^ep6Zc+L7) zSLqdYm0f*Tg@96kitLKaV7=`(?*(C}-EPSU)Dm=>>tG>lS@;k(#DCcLyPYs1zKAwk zHaLa2x7i|D^p#38oZ)sRJG3@{naFMh9`Aem;*#`$FfqXACHohbb{6crZg3wtu>*Py zfykB=et?eDS{#pajaIb^1$`PT?rQ( z*gZ(&!Xh}L?zYdG)m)pU3YfuPqY&EZ+13Kk|OoYIK_3tvmOen%`c;cfbhY zB4}z*T4-<4j%bYI{1D59lSmY`piQb%c(a)KObfGZ@fz6%)EP~&_L;8x$VjLsyX@-%p4})@^+RspO2)H-EDnCVANV6#F z6rK@~sr_h>X`_dM%-{lxryw?F6X28MKL9*oBiX?-lsgK{_q{N>h7JQT=SRV$_M^c( z0<)mW6tjKW9KtM7BxCkSxJ(e%3WQee)-Yhe-+*EI9gc?V2m}%t>O_V>pgMv;wCtU} zvU`6{> z1u(Opf+VSQ8q7|-c3+b(p+>2qZWZK<1xZe!yaVvN8U%xa&HM^|v>64uP?MkYmbEz^ zKdU^_b~4=zZNIC5zI62de6LJ058H#ePNz$A3iWktaN~`x-ZcoS1fI)EKSz)ijqPRD z-()e+@2*$wz&L&2HMU?Cxw2e?MG)0wt5dDnE{w>ETZV1!Z?j>LSQ^Pl+fL}3j+n&; zi-lq~R)g4Rc$G+!e2a~Buv95e=~@Sdf2>Qd`BaNF(Thu0oC92qK*uR$UU10#wJ?|W?tef&@tQOmayvp z_{e=IdvHwqWIqBWY3G4VARS)?d>>AdweqQg6EcMI7utpa7}Oyq0r@YSj}5W`pQ<1) ztjPx99xue!WkNn|zxI`X#P`^(bNmZ z3)=SdVZZ_nbf`rYo>jt-6EcsA*m9gCJ^-V*;5Zu{w~?C2ImZ#Lvg628=$BX~Y>GTf z#pft_o{|?RAqJFhQbOD^M2T#&Fptt{QQweCrfL{g!OEGsnMt1|E5~+t8OXQkfW*e9T(QUPQ#6&HZF zfo;8gkLhmMxvL?J(SVdb)MeWXry!aU!1t+CJPH4XofZ~hJl}|vA*7LwOdAZRaXdaA zp?(RC4p!iKG9>e(G#klf2S@FIJ;U1-F}e5Ca9REO8XJBGJ_cf3nuCh#7^J4yC1f5%QM zyB5hvlz1M@ew<8sX9@sqZFXo3YIw{n z)nm*o@cn$3jj!khSe4(~RqEe|sirtTh+YKk9H$zBqW(k3V{>2KS=8l|y1IqDmFCBg zAB&2;m%0{c$Lx-Y@ot_>xGHdmxqrlM=AmuplD6(x*Xrie{x7F>`JuW6KsT62XhNgD zWXU^=Xh~jy@>JT&6vLq@!DV8p`Eh=oH&{N%uhZ`;SE z10%j|A8#HQ@ooFKd|<@4?c=97DWi8qO!j^cxvBbF^{dgf-V$O;a12zOa_{=Cf-j^% z?*`>K%5;3CsKFM3eFKXT7Gk2#mmxDtCnBfW_MI96kB;}S3;~Uh&O2y{=MQOtHrosu zEmy>Ihqb~;rnct}YePs+r18+$N;-s**NzK;6CujwP$6X7w`d>&L{23@Cf6-CD3}!+ zo3O(ZlbEEvyohA`MQEEO0?(EZhq5S`;;&3BZVI$-kptoi2M!2-H`6aWCa4eN z+@?fK4jt0WWqbq zyO(@6lTVO*LV1jEHc1H?j!8(3iRUVc@i^`m8Uw12$HJ|k4EHg>(G&|woa>Ki7#SGy zeI%t5>nRmDppujfN~Z#E(>bg`tNJCaS}z&Zvh$GBE%^b>RYC$kChs_WZv^{r&&DH5 z@n7+Vze0jQBuDZnnm7Dda49LuN1hp_8r)1no&4I!nOauO!cXJJFt`;x{>#i3w0XFq zrkbQLaz07nlIqjyG}bB@^Jqt2pHunCzM}G@r(f+^tjlr5ftfdmiH;a%gk=gkEa45m zL;`WeQAMF;l+$Dc!6AAbcH3oYAy*J?A>BBL%oBqI*+^ZsU)M8OywO}aDEdR@>SfX% zT0Kj~R63VA$ir|W2t+bIy@M}9@vA~JE9br5%&*+6%r?W zLL~a=gy>WdiiV4$@jj%BhNo)645P7AZdxM{Xck4?o*6d&DYB6Wm7^CWp6oB0Xe#+j zRP~o8{K!9}WZ%Nn@cz)kvm*-+aq-bhkM4JP**JqZQu+R&3x9-#OG67!jV#RLMwTQd zARWUfUPp%Sm6RUrB-{;wly}vt%CP{Yj%Zi}q8A!sAJm?$=js&w(4u_Z68aOJ33;qn z5Jqnt@dEfGU>KyS=I1<|Fx)6{%$C$6F8#Dtd;2-|bZrsQtxsj=oLz0S)^R`Fujo6o zB+_M>k|rfg$9Zz&aR+@pl0T+OmW6b1asFzffm`ZM*-PAY{oUmiR-t=Rx9^mjkmzp1 z0ZHeRrIFo#sE;w{hlE=?8bzH&jAaJ0XdhJ#pc%n4dEw+6GoDGf5>+A2L@A<`h-gqQ zjW(f7_(d?4Js2JFM+hI(X5d>uXv&ylMZD$Zpx!DQIWC{$f zF89JSBtPJ~rq`CqjV_xggTY)%FTm-xY|v25C9KD)f?|rSFXYcD$LJ@PhsdFLN1WO8 z1x%!DjjlSAxFz4q#M;C2C2i|Gz87L`>%u;Cv(mr8FnWY^(veDL5=fuutBn1q2SX=?5wf5d^#xEKp_&4_hskD;x6)B%Euy2q)#-KR!#DVF-~@LuhObF8AbX__=a*vK zA9JMpW356?HI1@_LAqdr9H23t?bqL7Hn`4*&NiOT*2QdCw>L4+F68bXqFvIxW%ui_ zDu@a%2k{6hIIJadzw5{9ll9~H=J6s*k92Ap8VHvf`6r0+Q`j5im-IV<@AODN3;o8@ ze$R1k0Ui$QmRsnI z3;Xea7Cz%IG`Vu*xtv0;e$n*BoGAcy{$bjII& zhZ&zeV#enNGk$~_|G)Sk*{dPfM`-m7R+t;`!NifYYLE0gHbSfYT^sT@BXd8?_~`Qi zjX9hTo*(H)FdWSX#>PLP_y2|ulKp4o&xTHBRNC$Ffqn2~u<8pb-DWoC8Df$v8TmKg zVZP@No$q^t`5J>g{a^SXv+?hPc@FRa+trh)UH$ve5No)NGRyEU>7u2S#Xoo8ZN=F{ zQ6>iUyXYogq69h#b|5Zk;Pc0N}+u0!mtx)F|Wc|7857sr#3%MIGdO-gQ2a*7giZP>X;a8E|Jq&>;c zDPb3CJoTK*9vt}8duAu$A$4H~>22ix1ur{DxGT{M74sR~mC$3F6O!7ClgYAVqsQYT z{}M_fEg3jp_(vxh#&H%AqgPSTQDEXKjkmWbXt5w9E}DFvsTQM!QWO%hgX7W#!bN>yAR3TRY diff --git a/GPUSimulators/__pycache__/Simulator.cpython-39.pyc b/GPUSimulators/__pycache__/Simulator.cpython-39.pyc deleted file mode 100644 index dc16706a34d26a4641268277e3f0d257f351d864..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8433 zcmbVR+ix3JdY>DI_|HP)7M^Ga1=ngC*SX>AHAB74nu)jRTkTver>A3f zJ6|hg^NKbI- zHCho1x^01)>pM}Sg$1k`O0^rMuY3^%3QW`L_HgJxTX1VRFrN8JtIYq0xXLj;jvE{H%qQA=>P z(Q|0+3Z1@|oz7N5?5YsC%7lk;$zBN=0WtR3@PKT$t1OhxoEsCsl7$_@z5*zF%vTi2 z*3BPB_y8}?yg&in3Ohifh24PIw&(k7L$GcnnqB{@0}8OGtLyh3K3Qi=)o1MK(&NWV z)%9n8g_&R}M#U?U?n7J!>%bH#-5~A(X6M13$IJIHf9b2$`>X5E05)4$U9a9*TVpE^ zAG0O)XzB6#>hhEOOOM&3CyyUJT)Q(5j)5$|-}w{dBuQw8QZO!J*YhKw`V7pA0FuvH z%Y7xl#D?%*0U(z(AQnRszI{{2^}}FOA%^2gYEwZ0M5|2}gz;4t0kWH|IPTn7SlHg) zp5F|*^P${a0H=w|(ZZMW&KoG5R;}nsel>{iK!#DZm&0~D3>q{v{>!0p3s3Yj6hf=% z&{k+^&4gwd+=OPC!h)vdL~hF-f4KE)tEnq()NA=v3u}c;|B8Hqm!O9ZpM)MxC8xep zq9uFFVZc2Smi`@w;kQhhr?jdS`rHHZ8`(^dESs8+Z~EZ|ggP7cM5NXq!$%+D!;av? zjhe-Q2e)cA&jDf^kmdVlk3lxS-s_0|Kd9{qL?U1+om@l>sH8}Z1~joLBy0$au|R3y z39yg0f!tPCt9Ng(2T#`4*@LC^<$G-X-qJdIxB^s(H}fsFT4l>mZZC}%Rdh)RRrEL6 z3-(NVW1^^53Q3_}58Sq>*Au5+Pc0kjrFtFa$j?Ud^*Rq5=q}#&$fo%sPV!aJZM#8` z6du94!K5@2=doz|q7g$}ztdzH2dLurKVRrHqXoYUTO>Ny7a)&YZl~j`3Z~b8`1y|( z@%!TQg%&nnSn&cc7C#Gp*v-3-p43}jX94duuib@R2xYV|s?K+MiAAKoio+suCk5TI zES>)344M@y$pxW|TS+d0nzs^5xKW%~+aih^M2F#z>hvt*iF_0To$D90fmWxQp=vs6 zrm7jJS*m6Zv@I>RwjkIqjOZJ*^g}J^aIpEL4J0{OMI||jT5`G?#;Zf0CiopG zLs>BrD-wP)$tn3fV5&StOCUVNta|-(w4(DUw4$S{KVwQSeQy}0KN@D~e+;XC@wko* z7dJ>Qhd-+4@hC>jp~Ce&E!II;;}+Ic;anRG}dt&FUT&!`JP~O;8^33g#ki`AQqcqOd5$tZ>Fgr+Poo6e^WqF zu}>mlVg#KehfO~Fv|=b~CdNkNC`BDNS`NA3@HSO!6_rqEQ01BQ&+cD3PTqqh$0%%J zwW9DhsCb}ftnGouT>ooD<$a~gH?^$S8tcuCdhM~^{HV7u)?2&@>i}9iRmpIUe4h%X zs~=EJDeQ+-Q(8-8Q>qIx5PYeHG9-duOsI6-=e+bG}V1rV_4{uXjHGA^s|0G{i6eHHaH z39IV`Vl<2pEEYTQ4F(@h9*PZ5S#+>{qv_Wp z+>dc}U4}Elt&8r=z03v9c<_MYOGfj|}X{#TgfScUG(@XMAKXltnHC@GHUmXyW`loT^8gsja1l6ptDFHk7*xIsfCCY*7S<6@(`nK&sG z2*Jd{Rnt#$>ik-96LM0|`n%k%*=va3-F9LlrlI#V#RW2O!MwCXj{$#NDWwt9bwcb@ zDn6s)M^sRln4BI48+Fhiy~suKFKPG-Dn6#-6BM->1vn-(y-kP(MJ{9BNjWBRU?HA} zE&$s3^Eq2DWq*!t>$X|Y%c@;AOU5Pr(jRRb(Wzfcdw^qQq+D3@GNNSQJK8 z--5B+qR`*kH(?MFd6p^UWMBW{2H)MG;&~+}-@yvGK!tL}2ef#X`VRc8!i2o=Z}1GG z$umdcQ;}q~K|Xc(Bfn0Mg74$ZSyA}63)uFIeQnnoz)_vkc5?_a?E!+vt}Yk3Np+a> zY}O$6Li?32KN{F_1M94Pgq&xQBYR<}c8(mvmOV>u!Hn~Jg}A7&&!uOa*IvT`?&b%$ z4ZqOkKVi(-o8b99a(38jAMohd|K&$$PY&|R`I&oD>FR-X9y4|egMwUCE2lYJ;-Elo ztxr3Y_cdL6s=ZvawD{Do^xlWMwp+y5{Ge!QTgcT7&q?>&H=6p7ag(dh{1U64A=uk3 z{$26qV)|ax7O|7c>pA^jvyp9BIUE>A=~a{N=yefr`fnIt#`pP6}wyYO*(ByMt8X?Ksue%LLvJQ6y&D3*;iz@<^MJ^;EKt zt0NZHreK5A*03LB9W5_thDi>&(8h~oYAK3@B-K{;&BSu$CR{^mLP)OVJruQkCxxw+ zk6~?G8P{@(HML^5gKVRy^SI_Tx{^{UR4=LdWSHngT^!Lig%xEJ#BZ^CG><~F9iyNZ z3`<9RtJ}tuIjfh=i!dN(b<1?j8T6KQ1IDDRvt#CCC^3g}3PjzJ`M8Njtc=HN{kHbJ zybA@~LmmOqqoHdr5z|4O%|R->P)xKep0+F{ux9_>NYIYRYi8S#KEiE{WE9!yk+X&O zRr12)(Q@U#s+C-lgR_M>C+bxoSLqwq)h=qqvi6}(_GHCniat5oa?<42_onEb&%85Sg#*nzAz{BkTPLazc zEXXdAwc;yRdhQ0~|0rb7FzerFe>zM%#;oCPhc1nDIiu+ct|?StjQ|x{AFi!w@+rD- zG69fGM7^NVl3@U+DNi!yHO!&}>JWenM~HH;{1^-bhl0UIgA1U?`X6J* z9q}ERI{!IG5&KWXY!w6lId;?$H##&H$Icq*!6hulhZ&Xc0D3yPmDHWiYMW^qCrwvc zY4D)p1{Fb2@Ng1LN>c9Z#8Ca;jX`$qFwJ3v(mBVZkqE6Q{sUA}uEWn8F_iH&e3$Z9 zI(ahqzZ}T|`TO^W0og&)ErOp=Hya8CkP0D#C`s+pK!6ZWui1A{%Y;}`{iyi<7=8bR zCn7!3V1N3T594xTJa~-cBP}aqmpDsxY{6H*M#>_UXEV)1sC&PH)nSLDs@5K3s0v8jDN6J=G;BkP3Y$$zlGy4xlrt1G-=`y>Oe(@lCbJX1Z{&H0a zxNo>cd=on2f3aHm-YPPE=K#qeqc=g@)ziz!Q7%2fv>{dB z8CCfl`$O2SDJh7wM4J(T8a${8b2K|DdJ<8c~Y zWBAO8Soxq6va4kK{@Vb3hyi?)9RYg!u=uLOjsf{r!pLvvoVQZtrx-;*I6CYIRYpNe z1zrXJGe`4c2RX1cI)}WPBnKkNh)Xi9l8m1GnhH`)`Ex3W{fZ-!u$2VGn1m3aRSr+i zq1MI|QGj1C3O^}W&eekBn1$h=g_@1WQgaWPCt}=)c?2kBFuzn^ByLfb*iiXPsI=0W zs?wT2mFpdbr1U8m;UazgMaUO2cNjHL060wb$`5d$_x<8q z={0ZzohM#;K5oy2VXZbv1Gq@01EZOnoi2U9?bYmbX)>8@fp7Boten{CTi&9RCJ%jG zQbBQIhZ|e?HkH~(;-La|48;+LlfupPv)Px#sfezL))_>SddVoIrCq*T{(J4}{{j@a BH8B7H diff --git a/GPUSimulators/__pycache__/__init__.cpython-39.pyc b/GPUSimulators/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index a966589566cf0036cb97a40facf080e9dc3106f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 182 zcmYe~<>g`kg7xe&sV+eJF^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5w0=Qav3^cz zaY<2XfuVjuQGQlpK|v0fk(yi*Z(?R@00fq1`WczY8TxLSd6^}tVfi_wxvA~}q461+ s1^PfbI5W32C$S{Is8~Nf9;75bUaz3?7Kcr4eoARhsvXGE&p^xo01!1T>Hq)$ diff --git a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip index 67b701b..590ae86 100644 --- a/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip +++ b/GPUSimulators/cuda/EE2D_KP07_dimsplit.cu.hip @@ -25,7 +25,6 @@ along with this program. If not, see . #include "EulerCommon.h" #include "limiters.h" - __device__ void computeFluxF(float Q[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], float Qx[4][BLOCK_HEIGHT+4][BLOCK_WIDTH+4], @@ -248,4 +247,4 @@ __global__ void KP07DimsplitKernel( } -} // extern "C" \ No newline at end of file +} // extern "C" diff --git a/GPUSimulators/cuda/common.h b/GPUSimulators/cuda/common.h index 5463294..9be378e 100644 --- a/GPUSimulators/cuda/common.h +++ b/GPUSimulators/cuda/common.h @@ -24,6 +24,8 @@ along with this program. If not, see . #pragma once +#include +#include /** * Float3 operators @@ -86,9 +88,6 @@ __device__ float desingularize(float x_, float eps_) { - - - /** * Returns the step stored in the leftmost 16 bits * of the 32 bit step-order integer @@ -497,14 +496,18 @@ __device__ void memset(float Q[vars][shmem_height][shmem_width], float value) { template -__device__ void reduce_max(float* data, unsigned int n) { +//__device__ void reduce_max(float* data, unsigned int n) { +__device__ float reduce_max(float* data, unsigned int n) { __shared__ float sdata[threads]; unsigned int tid = threadIdx.x; //Reduce to "threads" elements sdata[tid] = FLT_MIN; for (unsigned int i=tid; ii5@b)KFdU@!ndkc3F;HxpT93n?urt=1daS&P!r`r+=HQb;NrA&$8qdH@Uw z%wT&4B*FAlrOd5bU$)|FTuzmEh|9X9V&9UBeab0^ROL1&r*hcrEtNycwdF&;*8@@z zSL^J6^QQatn>YRUzV}|UF)`t2xE6l#gVs;Ju4(^CmGP^H${pO%2OwOlYn*Yt%X)fU zXH+)2M%`3Tt8VcsH@Wo@tLOL`p5u9x@_d#Tc?l((&+!RfMybH(d4*4*xH+7ShyZ1wAZNz>UTys4i?ZVylM1;C~ z^Y{ZJZP(mTI9D4nby&DzpI)k=mqWMPK!=XsTlnIDUyO^pexN$t3HumE2ih>OJ-^#^ zw>)#NSFqNVa z#6$Gi`QE*?<%f{J`}WFvD;rPIZTJ4l#_GNGb@%?GHTSOj`0m=q%JP%^^D~4&o_?_P~oh`K-!gIM7H~emdp+3dpMd(tOV?*qESj491@1jGrQTo)K|7VSy zMmG%FYG9D0-Ir|?^k}8!24Q^7jnK2VI&s|p_Tu8+-rhnx7%YUMy@-{jRgM;KFF2pU zNVFPDr`}czDu&Bpz>QzadxytGu? z8`6Bnq;jSx2vj+8`7z2b6QeDdjUMi zfLBz!36kUboHi~yuO(%Y9q}`H=m38*xP>>4zox#F=}$_T&IB4WPNpT}(wnj*%W^_a zN=FvkIoQ0pDi~(qGa{ej#`T+}nx2|%;q!E&?ZwMsD7YVBd)F$dwdeD=ljb@e*?Ou6 z`?!a~!fUDJ!#4NpdYkTH%KVgVr#goOAEx^L^;93?-oJt8jpv%E;+@aq#eOSV><(b_ zUjN1-)_uFt?{`(fYsQ;zy!p-RxW4t~V#jZG7VrCkAA5fmb_YH0!Q&^J9lyW${-d>b zzq_)wi2AzU8^8`j5iNGSZXY(hxPlk_M)zbVL<{|4TEMXG?Z?2g3+O}y`SM=aX`G@xu7P^p$nt*F!Jd#Ta&f;2C@tY$|3 z0eF58r)JM<1U0K(>0le6eKW$6kfGJ>{moXRi6yAp!%;~TNIf|^wdNT)gKFyVa4(1& zSZiPnU!^M2`$0|`Sd4$OV>Xwjdm;6W0GDQT_kt19{+<~ zP4=tSLrW}aNo~i9=GA=X6#oj=DJOH1fqz}`fd!DigLPzTHuV@f4yimT4U}`LT!9T) zWXDy)h>Fai3|miZIT^EUo$J!V)8NJ@1~-*u=V5Wgh8=K*Ysn1mYMgtPKhl^c3)0~xc2EKGHZ&FF8T83K4{7!N zGt%4v!T^J2V@J-?TmYvq^7GoEk<3BY?1`?TEPX{?^UyW_vMxD?c{mGw^C$WyWce%V zI|qH|UZL+i^qo7=2g|~GT#yF!KDj98e7xv1X|k(b&Ewh+X8hT!KI1OI!eR!pr{H*65d_0wDfp0(|zdBwWxIpMW7gZq1Uq6f^SZVj^9Agk?WwOv(H+W1HtkeBVJjPQ3CxVxJ&Q7?3W zo62lc)~Oc;CpEn@s26s}s=K00{U!T4ZMr6P{}CFZ?}BJK7%lMenA!?K2{A9ROAN-0 zR+HK0HDD^9#x)@8*pu*87vF)NCauu%brFRUZiTHsgDrEGm|RbQUr8?3wi*1R)`ES) z-po(*B##ny#7jK;6}-YLcn)}{^%V(4X)i*jacF%%mz0u;q?}A9m1GL?7Q87uElTu# z6ZT8IGLV6t@MciT$!XZ{3!AROdhKME7hrV-ScFROBHUgHjI{EVE>N@VA9c84Wr zAz{n2yeRWW2GFL;i#ucu#68DL8Fy5hkTzs6YSmVjMbwJCKsbdMgY*}ODnuv(>P<{`YhH$+P2Pzbgl!}odSMG?b z*a*TU@`oVz#n-5K1>{rQxc+TPS-1EG%Hj_|vRD7Qe5ZZ+!@v3a$-gu2jFeru^W03W zAQT8>tO${GCpEoB1h2N|Me*}uYKTslS|FQ)>;bd6H)g~qMv=01YPB1^UL&2^8pWKO z$c?g~GpbF9KSD=RGYq^)T%)3O;E6C&VMJ;pa{>l6J@Fd#c|o=BA^Y$IAyH~>0J2hN z&1(^;AfQg|$DZ(!3N>pN$GYxx7ADw4betKhV<|>OqAPRpjQ8F zwfG<7{fPRaSp*2CK|z-K85h)Z)=+$QnUxWE!SfqscE!+HF8lq?E>!3}rql29oGqS0 z*QcaV(|*15Qj|Xj7NG})G*Jua@v-(XYw46he`F^)U`U>uiOsE~z;g+q$hiIIfhVUi zfp|s8jbjY)N}N!}977&(;R7dN8NiO%f=@;%&ZA~jwL^BO<7XhgFLP%{PbQVMPI6N^ zuvUB(wmS)HSGEh80%R(RTcN1_M5Y4U&+NNM*@Z0f3D|dusIu<~*ta>d@2MCVAZ_Z; zkF+1L!yM3Tdi0|h!---0+odm3URKtB1{ni#27JYlCD18l{WHL=0<8ZGFrt{u%2`-{ z<)EufcP$*WJJ|Th=HNVg9;L@fe6L===H6Ji{+Eh(83FC<=$9a;R+VA346bwIBgDxuA~VeS z_>5RdF3D4Vr&fWmAL~X4irF{S{4vx@s96#*(vgk`t$>`4v|vqX6_N?G=CBHuoS4&g zOlj~O-Z5S&lUFj>6(bosg|QEAtAULpXrur#8}}%ij`u>h)gAa;oq@djS_ZlFk^;yk zLCEn%kU#*jjyw7}NDNns1Ou*;wql}62zsx^`nK^5>BJF?WFF!iQ`9wkR)XWESbvry zCnjkMjFBW#2$Dq7ktHf0@O)wgE^0RP1#oXxFpV0vxsbewn#oHD#C3#jm|i~d3AyrP z%{0|{OnLqgi3!K|Yvolz_M1UFY?#t}1Z>zj{Tp1k)N~aj%?BRNk&s1xgK9Y-E9!zE zpk9DO;Q~ITS+|;lEr2IW%^d`NwR~#y8v7uAkO4KYF5qFWJ{2|n{%|4tPN+_H{(?TW z3R1Hb^rkEj#6w`!uj|9U_yJW|E5lIAZO8JQNkYM17{;A7^?`p!bL4_(NN-fe37Df& z4#|(Yb)-Jv9ef9GzUy^&{otY3?RCO#e6a9VH*7Y#(d`Aa)yWiI;*5VMbDFpD9W{$> z`l6=&*HSI7#w)&u8l~;=&DWJnQUlDq5ruZO@|c&|B3YBRpzu(9l_yErW)>@+_>q(~ zp6i)!qNB{QDY1_`|F7U@iJw_D?2DHO^jXC$BLSrQ;L2B-fsqH$#=?YlOI`oFq|9L? zVBX2L6YV6mr$$rqn_p#d!xHf+pgnfome=dYL*={GY>D?sVw1=RMCklL&|GEnM@AX6AcMb@YM555pb#VC9MREBd7`_rw?(bTkQN zj<0hlkONlX`)@Jj2;VJ)@5=R_0%i)|$%&4;VB%4zewugMEt6a^HZrgVrkSq@)y-LajYR z8csnx-1%`}o^gPK87J6+#tEgdo5}O?X?bu?#zj_!L|}r2se*09qGulth8NN(FFiFrE+;M*+j!?+FQ}RY))oowiB^Y!!aY zsgQsYf;{kjp3fj8I4jQr#j6KzD3|s?ozTZ{-fN9vmyQr$+du*FKO*6{vE_GtoL*B> zpv1>5^tVeYs6QS2tNgzGSJi)b@E->sEoDYb4ncH@DA(bl^ckg<2a_woZlmk-<5POk zBcanapV|2}YNGR~F-*Kc)z?8%qp=lbX8*$C{}c_ibEhmm&5!<*@TCFzDJy>!u8x`Z z39^nw=*8h?Oowpr4m`tyQvje)KmouM0N{A)vzfmRyUZeTQ(6DW_N&GPM!#9ZK`cIv z*xUh~W?$FKdWBulzceB*h=}Hat Date: Sun, 9 Jun 2024 22:56:23 +0200 Subject: [PATCH 51/55] update slurm job --- Jobs/job_lumi.slrum | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Jobs/job_lumi.slrum b/Jobs/job_lumi.slrum index b946bbf..1d7019a 100644 --- a/Jobs/job_lumi.slrum +++ b/Jobs/job_lumi.slrum @@ -4,10 +4,11 @@ #SBATCH --time=00:10:00 #SBATCH --partition=dev-g #SBATCH --nodes=1 -#SBATCH --ntasks-per-node=2 -#SBATCH --gpus=2 -#SBATCH --gpus-per-node=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --gpus=8 +#SBATCH --gpus-per-node=8 #SBATCH -o %x-%j.out +#SBATCH --exclusive # N=$SLURM_JOB_NUM_NODES @@ -26,9 +27,13 @@ ml rocm/5.2.3 ml craype-accel-amd-gfx90a ml cray-mpich/8.1.27 -#Enable GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - export PATH="/project/project_4650000xx/FiniteVolumeGPU_hip/MyCondaEnv/bin:$PATH" -srun python ${Myapplication} -nx 1024 -ny 1024 --profile +#missing library +export LD_LIBRARY_PATH=/opt/cray/pe/mpich/8.1.27/ofi/cray/14.0/lib-abi-mpich:$LD_LIBRARY_PATH + +#Binding mask +bind_mask="0x${fe}000000000000,0x${fe}00000000000000,0x${fe}0000,0x${fe}000000,0x${fe},0x${fe}00,0x${fe}00000000,0x${fe}0000000000" + +srun --cpu-bind=mask_cpu:$bind_mask \ + python ${Myapplication} -nx 1024 -ny 1024 --profile From 4ddf19bef7268dc23476c5265665c3ef8ceb8e72 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Sun, 9 Jun 2024 23:13:01 +0200 Subject: [PATCH 52/55] update README --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f68f36..36b03e9 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Here is a step-by-step guide on installing packages on LUMI-G ### Step 0: load modules ``` -ml LUMI/23.03 +ml LUMI/23.03 partition/G ml lumi-container-wrapper ml cray-python/3.9.13.1 ``` @@ -43,3 +43,20 @@ export the bin path: export PATH="$PWD/MyCondaEnv/bin:$PATH" ``` cotainr build my_container.sif --system=lumi-g --conda-env=conda_environment_lumi.yml ``` + +### Error when running MPI. +``` +`MPI startup(): PMI server not found. Please set I_MPI_PMI_LIBRARY variable if it is not a singleton case. +``` +This can be resolved by exporting this: +``` +export I_MPI_PMI_LIBRARY=/opt/cray/pe/mpich/8.1.27/ofi/cray/14.0/lib/libmpi.so +``` +### Install hip-python +``` +python -m pip install -i https://test.pypi.org/simple/ hip-python==5.4.3.470.16 +``` + +The testing was done with this specific version `hip-python==5.4.3.470.16` + + From b1dc2938adc21293f2374ac480f4f67e5e808b1f Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Sun, 9 Jun 2024 23:15:06 +0200 Subject: [PATCH 53/55] update conda_environment_lumi.yml --- conda_environment_lumi.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conda_environment_lumi.yml b/conda_environment_lumi.yml index 661ebc5..4a14fd5 100644 --- a/conda_environment_lumi.yml +++ b/conda_environment_lumi.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: -- python=3.9 +- python=3.9.13 - numpy - mpi4py - six @@ -13,8 +13,7 @@ dependencies: - netcdf4 - scipy - pip: - - hip-python - - hip-python-as-cuda + - hip-python==5.4.3.470.16 - -i https://test.pypi.org/simple/ From b8603e939e48247da0186d93e0f33456139093a0 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Sun, 9 Jun 2024 23:18:17 +0200 Subject: [PATCH 54/55] update README --- README.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/README.md b/README.md index 36b03e9..8e2250f 100644 --- a/README.md +++ b/README.md @@ -2,22 +2,6 @@ This is a HIP version of the [FiniteVolume code](https://github.com/babrodtk/FiniteVolumeGPU) (work in progress). It is a Python software package that implements several finite volume discretizations on Cartesian grids for the shallow water equations and the Euler equations. -## Setup -A good place to start exploring this codebase is the notebooks. Complete the following steps to run the notebooks: - -1. Install conda (see e.g. Miniconda or Anaconda) -2. Change directory to the repository root and run the following commands -3. conda env create -f conda_environment.yml -4. conda activate ShallowWaterGPU -5. jupyter notebook - -Make sure you are running the correct kernel ("conda:ShallowWaterGPU"). If not, change kernel using the "Kernel"-menu in the notebook. - -If you do not need to run notebooks you may use the conda environment found in conda_environment_hpc.yml - -## Troubleshooting -Have a look at the conda documentation and https://towardsdatascience.com/how-to-set-up-anaconda-and-jupyter-notebook-the-right-way-de3b7623ea4a - ## Setup on LUMI-G Here is a step-by-step guide on installing packages on LUMI-G @@ -28,7 +12,7 @@ ml lumi-container-wrapper ml cray-python/3.9.13.1 ``` -### Step 1: run conda-container +### Step 1: run conda-container Installation via conda can be done as: ``` conda-containerize new --prefix MyCondaEnv conda_environment_lumi.yml From dc78082f749fda4c8c2985f70e5283781f21fc85 Mon Sep 17 00:00:00 2001 From: Hicham Agueny Date: Sun, 9 Jun 2024 23:21:39 +0200 Subject: [PATCH 55/55] update mpiTesting.py --- mpiTesting.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mpiTesting.py b/mpiTesting.py index 5869fdd..6e9a4b5 100644 --- a/mpiTesting.py +++ b/mpiTesting.py @@ -28,17 +28,19 @@ import logging import os #GPU-aware MPI +""" from os import environ if environ.get("MPICH_GPU_SUPPORT_ENABLED", False): from ctypes import CDLL, RTLD_GLOBAL CDLL(f"{environ.get('CRAY_MPICH_ROOTDIR')}/gtl/lib/libmpi_gtl_hsa.so", mode=RTLD_GLOBAL) +""" # MPI from mpi4py import MPI # CUDA #import pycuda.driver as cuda -from hip import hip +from hip import hip,hiprtc # Simulator engine etc from GPUSimulators import MPISimulator, Common, CudaContext @@ -110,6 +112,15 @@ logger.info("File logger using level %s to %s", logger.info("Creating MPI grid") grid = MPISimulator.MPIGrid(MPI.COMM_WORLD) +""" +job_id = int(os.environ["SLURM_JOB_ID"]) +allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"]) +allocated_gpus = int(os.environ["ROCR_VISIBLE_DEVICES"].count(",") + 1) + +print("job_id:", job_id) +print("allocated_nodes", allocated_nodes) +print("allocated_gpus", allocated_gpus) +""" #### # Initialize CUDA @@ -123,7 +134,6 @@ cuda_device = local_rank % num_cuda_devices logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device)) cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False) - #### # Set initial conditions #### @@ -183,7 +193,7 @@ if(args.profile and MPI.COMM_WORLD.rank == 0): if "SLURM_JOB_ID" in os.environ: job_id = int(os.environ["SLURM_JOB_ID"]) allocated_nodes = int(os.environ["SLURM_JOB_NUM_NODES"]) - allocated_gpus = int(os.environ["HIP_VISIBLE_DEVICES"].count(",") + 1) + allocated_gpus = int(os.environ["ROCR_VISIBLE_DEVICES"].count(",") + 1) # allocated_gpus = int(os.environ["CUDA_VISIBLE_DEVICES"].count(",") + 1) profiling_file = "MPI_jobid_" + \ str(job_id) + "_" + str(allocated_nodes) + "_nodes_and_" + str(allocated_gpus) + "_GPUs_profiling.json"