From 7c9adb05bec02975f26a9dbe11206ae097ca69a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Lilleeng=20S=C3=A6tra?= <martinls@met.no>
Date: Fri, 16 Oct 2020 17:28:51 +0200
Subject: [PATCH] Added conda env and job script for DGX-2.

---
 conda_environment_hpc.yml | 22 ++++++++++++++++++++++
 dgx-2-test.job            | 22 ++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 conda_environment_hpc.yml
 create mode 100644 dgx-2-test.job

diff --git a/conda_environment_hpc.yml b/conda_environment_hpc.yml
new file mode 100644
index 0000000..677a828
--- /dev/null
+++ b/conda_environment_hpc.yml
@@ -0,0 +1,22 @@
+# Assumes that conda, pip, build-essentials and cuda are installed
+---
+name: ShallowWaterGPU_HPC
+channels:
+- conda-forge
+
+dependencies:
+- python=3.7
+- numpy
+- mpi4py
+- six
+- pytools
+- netcdf4
+- scipy
+
+# Install conda environment (one-time operation):
+# $ conda env create -f conda_environment_hpc.yml
+# Activate environment and install the following packages using pip:
+# $ conda activate ShallowWaterGPU
+#  - pycuda: $ pip3 install --no-deps -U pycuda
+# on Windows: make sure your visual studio c++ compiler is available in PATH
+# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\
diff --git a/dgx-2-test.job b/dgx-2-test.job
new file mode 100644
index 0000000..e458586
--- /dev/null
+++ b/dgx-2-test.job
@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH -p dgx2q                   # partition (GPU queue)
+#SBATCH -N 1                       # number of nodes
+#SBATCH -n 4                       # number of cores
+#SBATCH -w g001                    # DGX-2 node
+#SBATCH --gres=gpu:4               # number of V100's
+#SBATCH --mem 10G                  # memory pool for all cores
+#SBATCH -t 0-00:10                 # time (D-HH:MM)
+#SBATCH -o slurm.%N.%j.out  # STDOUT
+#SBATCH -e slurm.%N.%j.err  # STDERR
+
+ulimit -s 10240
+module load slurm 
+module load openmpi/4.0.1
+
+# Check how many gpu's your job got
+#nvidia-smi
+
+# Run job
+cd /home/martinls/src/ShallowWaterGPU
+mpirun --mca btl_openib_if_include mlx5_0 --mca btl_openib_warn_no_device_params_found 0 /home/martinls/miniconda3/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py
+