smyalygames/FiniteVolumeGPU
1
0
Fork 0
mirror of https://github.com/smyalygames/FiniteVolumeGPU.git synced 2025-05-18 06:24:13 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added more profiling timers

Browse Source
This commit is contained in:
Martin Lilleeng Sætra 2022-04-07 09:07:20 +00:00
parent 124e033ff5
commit 8fd9d3d850
3 changed files with 52 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
GPUSimulators/Common.py
View File

@ -95,6 +95,16 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
save_times, and saves all of the variables in list save_var_names. Elements in
save_var_names can be set to None if you do not want to save them
"""
profiling_data_sim_runner = { 'start': {}, 'end': {} }
profiling_data_sim_runner["start"]["t_sim_init"] = 0
profiling_data_sim_runner["end"]["t_sim_init"] = 0
profiling_data_sim_runner["start"]["t_nc_write"] = 0
profiling_data_sim_runner["end"]["t_nc_write"] = 0
profiling_data_sim_runner["start"]["t_step"] = 0
profiling_data_sim_runner["end"]["t_step"] = 0
profiling_data_sim_runner["start"]["t_sim_init"] = time.time()
logger = logging.getLogger(__name__)
assert len(save_times) > 0, "Need to specify which times to save"
@ -146,6 +156,8 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
t_steps[0] = save_times[0]
t_steps[1:] = save_times[1:] - save_times[0:-1]
profiling_data_sim_runner["end"]["t_sim_init"] = time.time()
#Start simulation loop
progress_printer = ProgressPrinter(save_times[-1], print_every=10)
for k in range(len(save_times)):
@ -160,18 +172,24 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e)))
return outdata.filename
profiling_data_sim_runner["start"]["t_step"] += time.time()
#Simulate
if (t_step > 0.0):
sim.simulate(t_step)
profiling_data_sim_runner["end"]["t_step"] += time.time()
profiling_data_sim_runner["start"]["t_nc_write"] += time.time()
#Download
"""
save_vars = sim.download(download_vars)
#Save to file
for i, var_name in enumerate(save_var_names):
ncvars[var_name][k, :] = save_vars[i]
"""
profiling_data_sim_runner["end"]["t_nc_write"] += time.time()
#Write progress to screen
print_string = progress_printer.getPrintString(t_end)
@ -180,7 +198,7 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names
logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps()))
return outdata.filename, sim.profiling_data
return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi

38
GPUSimulators/MPISimulator.py
View File

@ -201,13 +201,13 @@ class MPISimulator(Simulator.BaseSimulator):
Class which handles communication between simulators on different MPI nodes
"""
def __init__(self, sim, grid):
self.profiling_data = { 'start': {}, 'end': {} }
self.profiling_data["start"]["t_halo_exchange"] = 0
self.profiling_data["end"]["t_halo_exchange"] = 0
self.profiling_data["start"]["t_step"] = 0
self.profiling_data["end"]["t_step"] = 0
self.profiling_data["n_time_steps"] = 0
self.profiling_data["start"]["t_mpi_sim_init"] = time.time()
self.profiling_data_mpi = { 'start': {}, 'end': {} }
self.profiling_data_mpi["start"]["t_step_mpi_halo_exchange"] = 0
self.profiling_data_mpi["end"]["t_step_mpi_halo_exchange"] = 0
self.profiling_data_mpi["start"]["t_step_mpi"] = 0
self.profiling_data_mpi["end"]["t_step_mpi"] = 0
self.profiling_data_mpi["n_time_steps"] = 0
self.profiling_data_mpi["start"]["t_sim_mpi_init"] = time.time()
self.logger = logging.getLogger(__name__)
autotuner = sim.context.autotuner
@ -292,17 +292,25 @@ class MPISimulator(Simulator.BaseSimulator):
self.out_s = np.empty_like(self.in_s)
self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name()))
self.profiling_data["end"]["t_mpi_sim_init"] = time.time()
self.profiling_data_mpi["end"]["t_sim_mpi_init"] = time.time()
def substep(self, dt, step_number):
self.profiling_data["start"]["t_halo_exchange"] += time.time()
self.exchange()
self.profiling_data["end"]["t_halo_exchange"] += time.time()
if self.profiling_data_mpi["n_time_steps"] > 0:
self.profiling_data_mpi["start"]["t_step_mpi_halo_exchange"] += time.time()
self.exchange()
self.sim.stream.synchronize() # only necessary for profiling!
if self.profiling_data_mpi["n_time_steps"] > 0:
self.profiling_data_mpi["end"]["t_step_mpi_halo_exchange"] += time.time()
self.profiling_data_mpi["start"]["t_step_mpi"] += time.time()
self.profiling_data["start"]["t_step"] += time.time()
self.sim.substep(dt, step_number)
self.profiling_data["end"]["t_step"] += time.time()
self.profiling_data["n_time_steps"] += 1
self.sim.stream.synchronize() # only necessary for profiling!
if self.profiling_data_mpi["n_time_steps"] > 0:
self.profiling_data_mpi["end"]["t_step_mpi"] += time.time()
self.profiling_data_mpi["n_time_steps"] += 1
def getOutput(self):
return self.sim.getOutput()
@ -423,5 +431,3 @@ class MPISimulator(Simulator.BaseSimulator):
for comm in comm_send:
comm.wait()

7
mpiTesting.py
View File

@ -111,7 +111,7 @@ nx = args.nx
ny = args.ny
gamma = 1.4
save_times = np.linspace(0, 0.1, 2)
save_times = np.linspace(0, 0.5, 2)
outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc"
save_var_names = ['rho', 'rho_u', 'rho_v', 'E']
@ -138,7 +138,7 @@ def genSim(grid, **kwargs):
return sim
outfile, sim_profiling_data = Common.runSimulation(
outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation(
genSim, arguments, outfile, save_times, save_var_names)
if(args.profile):
@ -159,6 +159,9 @@ if(args.profile and MPI.COMM_WORLD.rank == 0):
else:
profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json"
for stage in sim_runner_profiling_data["start"].keys():
profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage]
for stage in sim_profiling_data["start"].keys():
profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage]