#!/bin/bash
# Job name:
#SBATCH --job-name=ShallowWaterGPUScaling
#
# Project:
#SBATCH --account=nn9882k
#
# Wall clock limit:
#SBATCH --time=00:10:00
#
# NOTE: See https://documentation.sigma2.no/jobs/projects_accounting.html when adjusting the values below
#
# Note: The environment variable CUDA_VISIBLE_DEVICES will show which GPU 
# device(s) to use. It will have values '0', '1' or '0,1' corresponding to 
# /dev/nvidia0, /dev/nvidia1 or both, respectively.
#SBATCH --partition=accel
#
# Max memory usage per task (core) - increasing this will cost more core hours:
##SBATCH --mem-per-cpu=3800M
#SBATCH --mem-per-cpu=24G
#
#SBATCH --qos=devel

## Set up job environment: (this is done automatically behind the scenes)
## (make sure to comment '#' or remove the following line 'source ...')
# source /cluster/bin/jobsetup

module restore system   # instead of 'module purge' rather set module environment to the system default
module load CUDA/11.4.1
#module load CUDA/11.1.1-GCC-10.2.0
#module load OpenMPI/4.0.5-gcccuda-2020b

# It is also recommended to to list loaded modules, for easier debugging:
module list

set -o errexit # exit on errors
set -o nounset # Treat unset variables as errors (added for more easily discovering issues in your batch script)

## Copy input files to the work directory:
mkdir $SCRATCH/ShallowWaterGPU
cp -r . $SCRATCH/ShallowWaterGPU

## Make sure the results are copied back to the submit directory (see Work Directory below):
# chkfile MyResultFile
# chkfile is replaced by 'savefile' on Saga
#savefile "$SCRATCH/ShallowWaterGPU/*.log"
#savefile "$SCRATCH/ShallowWaterGPU/*.nc"
#savefile "$SCRATCH/ShallowWaterGPU/*.json"
#savefile "$SCRATCH/ShallowWaterGPU/*.qdrep"

cleanup "rm -rf $SCRATCH/ShallowWaterGPU"

export OMPI_MCA_opal_cuda_support=true

## Do some work:
cd $SCRATCH/ShallowWaterGPU
srun /cluster/projects/nn9882k/martinls/.conda/envs/ShallowWaterGPU_HPC/bin/python3 --version
srun /cluster/projects/nn9882k/martinls/.conda/envs/ShallowWaterGPU_HPC/bin/python3 mpiTesting.py -nx $NX -ny $NY --profile

cd $HOME/src/ShallowWaterGPU
mkdir -p output_saga/$NOW/$SLURM_JOB_ID
mv $SCRATCH/ShallowWaterGPU/*.log ./output_saga/$NOW/$SLURM_JOB_ID
mv $SCRATCH/ShallowWaterGPU/*.nc ./output_saga/$NOW/$SLURM_JOB_ID
mv $SCRATCH/ShallowWaterGPU/*.json ./output_saga/$NOW
mv $SCRATCH/ShallowWaterGPU/*.qdrep ./output_saga/$NOW