{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Starting cluster\n", "\n", "## Prerequisites\n", "First, you need to install MPI, on windows use MS-MPI:\n", "https://msdn.microsoft.com/en-us/library/bb524831(v=vs.85).aspx\n", "\n", "\n", "## With a profile (not working)\n", "In theory, you should be able to create a profile using\n", "```\n", "ipython profile create --parallel --profile=myprofile\n", "```\n", "and then set\n", "```\n", "c.IPClusterEngines.engine_launcher_class = 'MPIEngineSetLauncher'\n", "```\n", "in ```/profile_myprofile/ipcluster_config.py```. This should then enable you to start a cluster using\n", "```\n", "ipcluster start --profile=myprofile\n", "```\n", "or alternatively through the Clusters tab in Jupyter\n", "\n", "\n", "## Without a profile (not working)\n", "An alternative is to run\n", "```\n", "ipcluster start --engines=MPI\n", "```\n", "\n", "\n", "## Manual start (working)\n", "This, however, does *not* work for me on Windows. What does work is the following:\n", "\n", "Start a controller using\n", "```\n", "ipcontroller --ip='*'\n", "```\n", "and then start several engines using mpiexec:\n", "```\n", "mpiexec -n 4 ipengine --mpi\n", "```" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Console logger using level INFO\n", "File logger using level DEBUG to mpi.log\n", "Python version 3.6.9 |Anaconda, Inc.| (default, Jul 30 2019, 14:00:49) [MSC v.1915 64 bit (AMD64)]\n" ] } ], "source": [ "%setup_logging --out mpi.log my_logger" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Starting IPController\n", "Starting IPEngines\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Waiting for cluster...\n", "Done\n" ] } ], "source": [ "%setup_mpi mpi_context --num_engines 3" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "%matplotlib inline\n", "\n", "import numpy as np\n", "from matplotlib import pyplot as plt\n", "from mpi4py import MPI\n", "import json\n", "\n", "import GPUSimulators.mpi as MPISimulator\n", "from GPUSimulators.common import run_simulation, DataDumper" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "%setup_logging --out \"'mpi_' + str(MPI.COMM_WORLD.rank) + '.log'\" my_logger" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "%cuda_context_handler my_context" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "def plot_extent(extent, *args, **kwargs):\n", " x0, x1, y0, y1 = extent\n", " plt.plot([x0, x1, x1, x0, x0], [y0, y0, y1, y1, y0], *args, **kwargs)" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "import importlib\n", "\n", "importlib.reload(MPISimulator)\n", "\n", "grid = MPISimulator.MPIGrid(MPI.COMM_WORLD)\n", "print(grid.get_local_rank())" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "from GPUSimulators.model import EE2DKP07Dimsplit\n", "\n", "my_context.autotuner = None\n", "\n", "nx = 128\n", "ny = 128\n", "gamma = 1.4\n", "save_times = np.linspace(0, 5.0, 10)\n", "outfile = \"mpi_out_\" + str(MPI.COMM_WORLD.rank) + \".nc\"\n", "save_var_names = ['rho', 'rho_u', 'rho_v', 'E']\n", "\n", "grid = MPISimulator.MPIGrid(MPI.COMM_WORLD)\n", "print(grid.get_local_rank())\n", "\n", "#arguments = InitialConditions.genShockBubble(nx, ny, gamma, grid=grid)\n", "arguments = InitialConditions.gen_kelvin_helmholtz(nx, ny, gamma, grid=grid)\n", "#arguments = InitialConditions.genRayleighTaylor(nx, ny, gamma, grid=grid)\n", "\n", "arguments['context'] = my_context\n", "arguments['theta'] = 1.2\n", "arguments['grid'] = grid\n", "\n", "\n", "def gen_sim(grid, **kwargs):\n", " local_sim = EE2DKP07Dimsplit(**kwargs)\n", " sim = MPISimulator.MPISimulator(local_sim, grid)\n", " return sim\n", "\n", "\n", "outfile = run_simulation(gen_sim, arguments, outfile, save_times, save_var_names)" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "with DataDumper(outfile, 'r') as indata:\n", " t = indata.nc.variables['time'][:]\n", " x = indata.nc.variables['x'][:]\n", " y = indata.nc.variables['y'][:]\n", "\n", " x_data = grid.gather(x)\n", " y_data = grid.gather(y)\n", "\n", " created = indata.nc.created\n", " sim_args = json.loads(indata.nc.sim_args)\n", " for key in sim_args:\n", " if isinstance(sim_args[key], list):\n", " sim_args[key] = \"[...]\"\n", " num_steps = len(t)\n", " print(f\"{outfile} created {created} contains {num_steps} timesteps\")\n", " print(\"Simulator arguments: \\n\", sim_args)\n", "\n", " for i in range(num_steps):\n", " rho = indata.nc.variables['rho'][i]\n", " rho_data = grid.gather(rho)\n", "\n", " #Plot on rank 0\n", " if grid.comm.rank == 0:\n", " plt.figure(figsize=(16, 12))\n", " for k in range(grid.comm.size):\n", " extent = [x_data[k].min(), x_data[k].max(), y_data[k].min(), y_data[k].max()]\n", " plt.imshow(rho_data[k], extent=extent, origin='lower', vmin=1.0, vmax=2.0)\n", " plot_extent(extent, 'k:', alpha=0.1)\n", " plt.title(\"t=\" + str(t[i]))\n", " plt.colorbar(shrink=0.33)" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "from GPUSimulators.model import HLL2\n", "from GPUSimulators.Simulator import BoundaryCondition\n", "\n", "nx = 128\n", "ny = 128\n", "g = 9.81\n", "dt = 0.05\n", "width = 50\n", "height = 100\n", "save_times = np.linspace(0, 80, 10)\n", "outfile = \"mpi_out_\" + str(MPI.COMM_WORLD.rank) + \".nc\"\n", "save_var_names = ['h', 'hu', 'hv']\n", "\n", "if MPI.COMM_WORLD.rank == 0:\n", " h0, hu0, hv0, dx, dy = InitialConditions.bump(nx, ny, width, height, g, x_center=0.75, y_center=0.55)\n", "else:\n", " h0, hu0, hv0, dx, dy = InitialConditions.bump(nx, ny, width, height, g, h_ref=0.5, h_amp=0.0, u_amp=0.0, v_amp=0.0)\n", "\n", "bc = BoundaryCondition({\n", " 'north': BoundaryCondition.Type.Reflective,\n", " 'south': BoundaryCondition.Type.Reflective,\n", " 'east': BoundaryCondition.Type.Reflective,\n", " 'west': BoundaryCondition.Type.Reflective\n", "})\n", "grid = MPISimulator.MPIGrid(MPI.COMM_WORLD)\n", "\n", "arguments = {\n", " 'context': my_context,\n", " 'h0': h0, 'hu0': hu0, 'hv0': hv0,\n", " 'nx': nx, 'ny': ny,\n", " 'dx': dx, 'dy': dy,\n", " 'g': g,\n", " 'boundary_conditions': bc,\n", " 'grid': grid\n", "}\n", "\n", "\n", "def gen_sim(grid, **kwargs):\n", " local_sim = HLL2(**kwargs)\n", " sim = MPISimulator.MPISimulator(local_sim, grid)\n", " return sim\n", "\n", "\n", "outfile = run_simulation(gen_sim, arguments, outfile, save_times, save_var_names)" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": [ "%%px\n", "\n", "with DataDumper(outfile, 'r') as indata:\n", " t = indata.nc.variables['time'][:]\n", " x = indata.nc.variables['x'][:]\n", " y = indata.nc.variables['y'][:]\n", "\n", " x_data = grid.gather(x)\n", " y_data = grid.gather(y)\n", "\n", " created = indata.nc.created\n", " sim_args = json.loads(indata.nc.sim_args)\n", " for key in sim_args:\n", " if isinstance(sim_args[key], list):\n", " sim_args[key] = \"[...]\"\n", " num_steps = len(t)\n", " print(f\"{outfile} created {created} contains {num_steps} timesteps\")\n", " print(\"Simulator arguments: \\n\", sim_args)\n", "\n", " for i in range(num_steps):\n", " h = indata.nc.variables['h'][i]\n", " #hu = indata.ncfile.variables['hu'][i]\n", " #hv = indata.ncfile.variables['hv'][i]\n", " h_data = grid.gather(h)\n", "\n", " #Plot on rank 0\n", " if grid.comm.rank == 0:\n", " plt.figure(figsize=(16, 12))\n", " for k in range(grid.comm.size):\n", " extent = [x_data[k].min(), x_data[k].max(), y_data[k].min(), y_data[k].max()]\n", " plt.imshow(h_data[k], extent=extent, origin='lower', vmin=0.49, vmax=0.52)\n", " plot_extent(extent, 'k:', alpha=0.1)\n", " plt.title(\"t=\" + str(t[i]))\n", " plt.colorbar(shrink=0.33)" ] }, { "metadata": {}, "cell_type": "code", "outputs": [], "execution_count": null, "source": "" } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:root] *", "language": "python", "name": "conda-root-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 4 }