From 9f88d611dd79f5b5ece56813c761297d03bba359 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Lilleeng=20S=C3=A6tra?= <martinls@met.no>
Date: Fri, 6 Aug 2021 15:05:19 +0000
Subject: [PATCH] Single-node testing

---
 mpiTesting.py       |   3 +-
 run_script_ppi.sh   |   8 +++
 singleGPUTesting.py | 125 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 135 insertions(+), 1 deletion(-)
 create mode 100755 run_script_ppi.sh
 create mode 100644 singleGPUTesting.py

diff --git a/mpiTesting.py b/mpiTesting.py
index d7192d9..4d30d81 100644
--- a/mpiTesting.py
+++ b/mpiTesting.py
@@ -97,6 +97,7 @@ logger.info("Initializing CUDA")
 local_rank = grid.getLocalRank()
 num_cuda_devices = cuda.Device.count()
 cuda_device = local_rank % num_cuda_devices
+logger.info("Process %s using CUDA device %s", str(local_rank), str(cuda_device))
 cuda_context = CudaContext.CudaContext(device=cuda_device, autotuning=False)
 
 
@@ -108,7 +109,7 @@ nx = args.nx
 ny = args.ny
 
 gamma = 1.4
-save_times = np.linspace(0, 10.0, 2)
+save_times = np.linspace(0, 0.02, 2)
 outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc"
 save_var_names = ['rho', 'rho_u', 'rho_v', 'E']
 
diff --git a/run_script_ppi.sh b/run_script_ppi.sh
new file mode 100755
index 0000000..4f1803b
--- /dev/null
+++ b/run_script_ppi.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+module purge
+module load git/2.21.0 hdf5/1.10.5-gcc cuda/10.1 conda/production
+
+activate ShallowWaterGPU_HPC
+
+/modules/centos7/conda/Feb2021/bin/python3 mpiTesting.py
+
diff --git a/singleGPUTesting.py b/singleGPUTesting.py
new file mode 100644
index 0000000..fee48ac
--- /dev/null
+++ b/singleGPUTesting.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+"""
+This python module implements simulations for benchmarking
+
+Copyright (C) 2018  SINTEF ICT
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+
+import numpy as np
+import gc
+import logging
+import os
+
+# CUDA
+import pycuda.driver as cuda
+
+# Simulator engine etc
+from GPUSimulators import Common, CudaContext
+from GPUSimulators import EE2D_KP07_dimsplit
+from GPUSimulators.helpers import InitialConditions as IC
+from GPUSimulators.Simulator import BoundaryCondition as BC
+
+import argparse
+parser = argparse.ArgumentParser(description='Single GPU testing.')
+parser.add_argument('-nx', type=int, default=128)
+parser.add_argument('-ny', type=int, default=128)
+
+
+args = parser.parse_args()
+
+####
+# Initialize logging
+####
+log_level_console = 20
+log_level_file = 10
+log_filename = 'single_gpu.log'
+logger = logging.getLogger('GPUSimulators')
+logger.setLevel(min(log_level_console, log_level_file))
+
+ch = logging.StreamHandler()
+ch.setLevel(log_level_console)
+logger.addHandler(ch)
+logger.info("Console logger using level %s",
+            logging.getLevelName(log_level_console))
+
+fh = logging.FileHandler(log_filename)
+formatter = logging.Formatter(
+    '%(asctime)s:%(name)s:%(levelname)s: %(message)s')
+fh.setFormatter(formatter)
+fh.setLevel(log_level_file)
+logger.addHandler(fh)
+logger.info("File logger using level %s to %s",
+            logging.getLevelName(log_level_file), log_filename)
+
+
+####
+# Initialize CUDA
+####
+cuda.init(flags=0)
+logger.info("Initializing CUDA")
+cuda_context = CudaContext.CudaContext(autotuning=False)
+
+
+####
+# Set initial conditions
+####
+logger.info("Generating initial conditions")
+nx = args.nx
+ny = args.ny
+
+gamma = 1.4
+save_times = np.linspace(0, 10.0, 2)
+outfile = "single_gpu_out.nc"
+save_var_names = ['rho', 'rho_u', 'rho_v', 'E']
+
+arguments = IC.genKelvinHelmholtz(nx, ny, gamma)
+arguments['context'] = cuda_context
+arguments['theta'] = 1.2
+
+
+####
+# Run simulation
+####
+logger.info("Running simulation")
+# Helper function to create MPI simulator
+
+
+def genSim(**kwargs):
+    local_sim = EE2D_KP07_dimsplit.EE2D_KP07_dimsplit(**kwargs)
+    return local_sim
+
+
+outfile = Common.runSimulation(
+    genSim, arguments, outfile, save_times, save_var_names)
+
+####
+# Clean shutdown
+####
+local_sim = None
+cuda_context = None
+arguments = None
+logging.shutdown()
+gc.collect()
+
+
+
+####
+# Print completion and exit
+####
+print("Completed!")
+exit(0)