From 8fd9d3d850a0efe2955abb2dbfabfcde6abc3130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Lilleeng=20S=C3=A6tra?= Date: Thu, 7 Apr 2022 09:07:20 +0000 Subject: [PATCH] Added more profiling timers --- GPUSimulators/Common.py | 30 ++++++++++++++++++++------ GPUSimulators/MPISimulator.py | 40 ++++++++++++++++++++--------------- mpiTesting.py | 7 ++++-- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/GPUSimulators/Common.py b/GPUSimulators/Common.py index 36b82be..d052db0 100644 --- a/GPUSimulators/Common.py +++ b/GPUSimulators/Common.py @@ -95,10 +95,20 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names save_times, and saves all of the variables in list save_var_names. Elements in save_var_names can be set to None if you do not want to save them """ + profiling_data_sim_runner = { 'start': {}, 'end': {} } + profiling_data_sim_runner["start"]["t_sim_init"] = 0 + profiling_data_sim_runner["end"]["t_sim_init"] = 0 + profiling_data_sim_runner["start"]["t_nc_write"] = 0 + profiling_data_sim_runner["end"]["t_nc_write"] = 0 + profiling_data_sim_runner["start"]["t_step"] = 0 + profiling_data_sim_runner["end"]["t_step"] = 0 + + profiling_data_sim_runner["start"]["t_sim_init"] = time.time() + logger = logging.getLogger(__name__) - + assert len(save_times) > 0, "Need to specify which times to save" - + with Timer("construct") as t: sim = simulator(**simulator_args) logger.info("Constructed in " + str(t.secs) + " seconds") @@ -140,12 +150,14 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names #Create variables for var_name in save_var_names: ncvars[var_name] = outdata.ncfile.createVariable(var_name, np.dtype('float32').char, ('time', 'y', 'x'), zlib=True, least_significant_digit=3) - + #Create step sizes between each save t_steps = np.empty_like(save_times) t_steps[0] = save_times[0] t_steps[1:] = save_times[1:] - save_times[0:-1] + profiling_data_sim_runner["end"]["t_sim_init"] = time.time() + #Start simulation loop progress_printer = ProgressPrinter(save_times[-1], print_every=10) for k in range(len(save_times)): @@ -160,18 +172,24 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names logger.error("Error after {:d} steps (t={:f}: {:s}".format(sim.simSteps(), sim.simTime(), str(e))) return outdata.filename + profiling_data_sim_runner["start"]["t_step"] += time.time() + #Simulate if (t_step > 0.0): sim.simulate(t_step) + profiling_data_sim_runner["end"]["t_step"] += time.time() + + profiling_data_sim_runner["start"]["t_nc_write"] += time.time() + #Download - """ save_vars = sim.download(download_vars) #Save to file for i, var_name in enumerate(save_var_names): ncvars[var_name][k, :] = save_vars[i] - """ + + profiling_data_sim_runner["end"]["t_nc_write"] += time.time() #Write progress to screen print_string = progress_printer.getPrintString(t_end) @@ -180,7 +198,7 @@ def runSimulation(simulator, simulator_args, outfile, save_times, save_var_names logger.debug("Simulated to t={:f} in {:d} timesteps (average dt={:f})".format(t_end, sim.simSteps(), sim.simTime() / sim.simSteps())) - return outdata.filename, sim.profiling_data + return outdata.filename, profiling_data_sim_runner, sim.profiling_data_mpi diff --git a/GPUSimulators/MPISimulator.py b/GPUSimulators/MPISimulator.py index 4b1a98e..823960b 100644 --- a/GPUSimulators/MPISimulator.py +++ b/GPUSimulators/MPISimulator.py @@ -201,13 +201,13 @@ class MPISimulator(Simulator.BaseSimulator): Class which handles communication between simulators on different MPI nodes """ def __init__(self, sim, grid): - self.profiling_data = { 'start': {}, 'end': {} } - self.profiling_data["start"]["t_halo_exchange"] = 0 - self.profiling_data["end"]["t_halo_exchange"] = 0 - self.profiling_data["start"]["t_step"] = 0 - self.profiling_data["end"]["t_step"] = 0 - self.profiling_data["n_time_steps"] = 0 - self.profiling_data["start"]["t_mpi_sim_init"] = time.time() + self.profiling_data_mpi = { 'start': {}, 'end': {} } + self.profiling_data_mpi["start"]["t_step_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["end"]["t_step_mpi_halo_exchange"] = 0 + self.profiling_data_mpi["start"]["t_step_mpi"] = 0 + self.profiling_data_mpi["end"]["t_step_mpi"] = 0 + self.profiling_data_mpi["n_time_steps"] = 0 + self.profiling_data_mpi["start"]["t_sim_mpi_init"] = time.time() self.logger = logging.getLogger(__name__) autotuner = sim.context.autotuner @@ -292,18 +292,26 @@ class MPISimulator(Simulator.BaseSimulator): self.out_s = np.empty_like(self.in_s) self.logger.debug("Simlator rank {:d} initialized on {:s}".format(self.grid.comm.rank, MPI.Get_processor_name())) - self.profiling_data["end"]["t_mpi_sim_init"] = time.time() + self.profiling_data_mpi["end"]["t_sim_mpi_init"] = time.time() def substep(self, dt, step_number): - self.profiling_data["start"]["t_halo_exchange"] += time.time() - self.exchange() - self.profiling_data["end"]["t_halo_exchange"] += time.time() + if self.profiling_data_mpi["n_time_steps"] > 0: + self.profiling_data_mpi["start"]["t_step_mpi_halo_exchange"] += time.time() + + self.exchange() + + self.sim.stream.synchronize() # only necessary for profiling! + if self.profiling_data_mpi["n_time_steps"] > 0: + self.profiling_data_mpi["end"]["t_step_mpi_halo_exchange"] += time.time() + self.profiling_data_mpi["start"]["t_step_mpi"] += time.time() - self.profiling_data["start"]["t_step"] += time.time() self.sim.substep(dt, step_number) - self.profiling_data["end"]["t_step"] += time.time() - self.profiling_data["n_time_steps"] += 1 - + + self.sim.stream.synchronize() # only necessary for profiling! + if self.profiling_data_mpi["n_time_steps"] > 0: + self.profiling_data_mpi["end"]["t_step_mpi"] += time.time() + self.profiling_data_mpi["n_time_steps"] += 1 + def getOutput(self): return self.sim.getOutput() @@ -422,6 +430,4 @@ class MPISimulator(Simulator.BaseSimulator): #Wait for sending to complete for comm in comm_send: comm.wait() - - \ No newline at end of file diff --git a/mpiTesting.py b/mpiTesting.py index 4d6f859..9750b37 100644 --- a/mpiTesting.py +++ b/mpiTesting.py @@ -111,7 +111,7 @@ nx = args.nx ny = args.ny gamma = 1.4 -save_times = np.linspace(0, 0.1, 2) +save_times = np.linspace(0, 0.5, 2) outfile = "mpi_out_" + str(MPI.COMM_WORLD.rank) + ".nc" save_var_names = ['rho', 'rho_u', 'rho_v', 'E'] @@ -138,7 +138,7 @@ def genSim(grid, **kwargs): return sim -outfile, sim_profiling_data = Common.runSimulation( +outfile, sim_runner_profiling_data, sim_profiling_data = Common.runSimulation( genSim, arguments, outfile, save_times, save_var_names) if(args.profile): @@ -159,6 +159,9 @@ if(args.profile and MPI.COMM_WORLD.rank == 0): else: profiling_file = "MPI_" + str(MPI.COMM_WORLD.size) + "_procs_and_" + str(num_cuda_devices) + "_GPUs_profiling.json" + for stage in sim_runner_profiling_data["start"].keys(): + profiling_data[stage] = sim_runner_profiling_data["end"][stage] - sim_runner_profiling_data["start"][stage] + for stage in sim_profiling_data["start"].keys(): profiling_data[stage] = sim_profiling_data["end"][stage] - sim_profiling_data["start"][stage]