feat(build): add LUMI Apptainer container

.gitignore

@@ -5,11 +5,15 @@
 
 /data
 
+# Apptainer/Singularity Containers
+*.sif
+
 # Numpy Zipped
 *.npz
 
 # NetCDF
 *.nc
+*.nc4
 
 # Python Related files
 # Taken from: https://github.com/github/gitignore/blob/main/Python.gitignore

conda_environment.yml

@@ -1,6 +1,6 @@
 # Assumes that conda, pip, build-essentials and cuda are installed
 ---
-name: FiniteVolumeGPU_HPC
+name: FiniteVolumeGPU
 channels:
   - conda-forge
 
@@ -9,10 +9,10 @@ dependencies:
     - pip
     - numpy
     - mpi4py
-    - six
+    - mpich
     - pytools
     - scipy
-    - netcdf4=*=mpi_openmpi_*
+    - netcdf4=*=mpi_mpich_*
     - matplotlib
     - ipyparallel
     - jupyter

conda_environment_lumi.yml

@@ -0,0 +1,30 @@
+# Assumes that conda, pip, build-essentials and cuda are installed
+---
+name: FiniteVolumeGPU_LUMI
+channels:
+  - conda-forge
+
+dependencies:
+  - python=3.10
+  - pip
+  - mpi=*=mpich
+  - mpich=3.4.3=external_*
+  - netcdf4=*=mpi_mpich_*
+  - pytools
+  - numpy
+  - scipy
+  - tqdm
+  - pip:
+      - mpi4py==3.1.1
+      - hip-python==6.1.2.495.17
+      - --extra-index-url https://test.pypi.org/simple/
+# Install conda environment (one-time operation):
+# $ conda env create -f conda_environment.yml
+# Activate environment
+# $ conda activate FiniteVolumeGPU_HPC
+
+# OPTIONAL: If you want to compile pycuda yourself, uncomment pycuda under
+# "dependencies" above and do the following (one-time operation):
+#  - pycuda: $ pip3 install --no-deps -U pycuda
+# on Windows: make sure your visual studio c++ compiler is available in PATH
+# PATH should have something like C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\

containers/lumi.def

@@ -0,0 +1,62 @@
+BootStrap: docker
+From: rocm/dev-ubuntu-22.04:6.1.2
+
+%files
+    conda_environment_lumi.yml /opt
+
+%environment
+    export MAMBA_ROOT_PREFIX="/opt/conda"
+    export MAMBA_EXE="/usr/bin/micromamba"
+    export PATH="${MAMBA_ROOT_PREFIX}/bin:$PATH"
+
+%post
+    # Version values
+    # MPI on LUMI is 3.4a2
+    export MPICH_VERSION=3.4.3
+    export MPICH_URL="http://www.mpich.org/static/downloads/$MPICH_VERSION/mpich-$MPICH_VERSION.tar.gz"
+
+    # Base OS
+    echo -e "\e[32m + \e[39mInstalling \e[1mrequired packages\e[0m..."
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+    apt-get install -y file g++ gcc gfortran make gdb strace wget curl ca-certificates build-essential
+    rm -rf /var/lib/apt /var/lib/dpkg /var/lib/cache /var/lib/log
+
+    # Install MPICH
+    echo -e "\e[32m + \e[39mInstalling \e[1mMPICH ${MPICH_VERSION}\e[0m..."
+    mkdir -p /tmp/mpich
+    mkdir -p /opt
+
+    cd /tmp/mpich
+    wget -q $MPICH_URL -O mpich-$MPICH_VERSION.tar.gz
+    tar xzf mpich-$MPICH_VERSION.tar.gz
+    cd mpich-$MPICH_VERSION
+
+    export FFLAGS=-fallow-argument-mismatch
+    ./configure --enable-fast=all,O3 --prefix=/usr --with-device=ch3
+    make -j$(nproc)
+    make install
+    ldconfig
+
+    # Install Miniconda
+    echo -e "\e[32m + \e[39mInstalling \e[1mMiniconda\e[0m..."
+    mkdir -p /tmp/micromamba
+    cd /tmp/micromamba
+    curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xj -C /usr bin/micromamba
+    mkdir -p /opt/conda/base
+
+    export MAMBA_ROOT_PREFIX="/opt/conda"
+    export MAMBA_EXE="/usr/bin/micromamba"
+
+    mkdir -p "$MAMBA_ROOT_PREFIX/conda-meta"
+    chmod -R a+rwx "$MAMBA_ROOT_PREFIX"
+
+    # Create Conda environment
+    echo -e "\e[32m + \e[39mCreating \e[1mConda environment\e[0m..."
+    eval ${MAMBA_EXE} shell hook --shell=bash
+    micromamba install -q -y -n base -f /opt/conda_environment_lumi.yml && \
+    micromamba clean --all --yes
+
+%help
+    This is a container built for running FiniteVolumeGPU on LUMI-G.
+    It includes MPICH and the required Conda packages.

jobs/job_lumi.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -l
+#SBATCH --job-name=FVM-Test
+#SBATCH --account=project_465001926
+#SBATCH --time=00:10:00
+#SBATCH --partition=dev-g
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --gpus-per-node=8
+#SBATCH --output=%x-%j.out
+#SBATCH --exclusive
+
+
+module load LUMI/24.03 partition/G
+module load singularity-bindings
+
+##### Helper Functions #####
+curr_time() {
+  date "+%Y-%m-%d %H:%M:%S"
+}
+
+##### Variables ######
+project_dir=/project/project_465001926/anthony
+application=${project_dir}/FiniteVolumeGPU/mpi_testing_hip.py
+container=${project_dir}/FiniteVolumeGPU/fvm.sif
+
+CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"
+
+##### Required modules #####
+
+#export MPICH_GPU_SUPPORT_ENABLED=1
+
+##### Running the job ######
+start_time=$(curr_time)
+echo "Starting job at: ${start_time}"
+
+srun --cpu-bind=${CPU_BIND} --mpi=cray_shasta \
+	  singularity exec \
+	  -B ${project_dir}/FiniteVolumeGPU \
+	  ${container} \
+	  micromamba run -n base python ${application} -nx 1028 -ny 1028 --profile
+
+
+end_time=$(curr_time)
+echo "Finished job at: ${end_time}"