polishing and example submission script

.gitignore

 venv
+slurm.*
+overlay.*

config.sh

@@ -9,5 +9,4 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
 export ENV_NAME=`basename $ABSOLUTE_PATH`    # Default Name of the venv is the directory that contains this file
 export ENV_DIR=${ABSOLUTE_PATH}/venv         # Default location of this VENV is "./venv"
 
-export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_tensorflow_21.02-tf1-py3.sif
-export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_pytorch_bert_20.06-py3-m5.1.sif
+export SINGULARITY_IMAGE=/p/project/hai_mlperf/oc20_singularity_v4_2022_09_06.sif

example_submission_script.sh

+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=2
+#SBATCH --time=00:01:00
+#SBATCH --gres=gpu
+#SBATCH --partition=develbooster
+#SBATCH --account=atmlaml
+
+set -e
+
+######## How to run with container #########
+module purge
+
+source config.sh
+
+echo "Running with image $SINGULARITY_IMAGE"
+echo "Check that your virtual environment is in the PYTHONPATH:"
+srun singularity run $SINGULARITY_IMAGE bash -c ' 
+    source activate.sh;
+    echo hello world;
+    python -c \"import sys; print(sys.path)\"
+'

setup.sh

@@ -3,25 +3,33 @@ export RELATIVE_PATH=`dirname ${BASH_SOURCE}`
 export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
 
 source ${ABSOLUTE_PATH}/config.sh
-echo path ${ABSOLUTE_PATH}
+echo Environment in ${ABSOLUTE_PATH}
 
+#### Optional: Set the Variable Overlay if you want to use the Singularity Overlay feature
+#### This might be necessary if you want to modify the image. Documentation: https://apptainer.org/docs/user/main/persistent_overlays.html
+if [ -z $OVERLAY ]; then
     rm -f overlay.img
     rm -rf overlay
-rm -rf venv
     
     mkdir -p overlay/upper
     mkdir -p overlay/work
     dd if=/dev/zero of=overlay.img bs=1M count=500 && \
          mkfs.ext3 -d overlay overlay.img
+    export OVERLAY_ARGS=" --overlay ${ABSOLUTE_PATH}/overlay.img "
+else
+    export OVERLAY_ARGS=""
+fi
+#### /Overlay
 
-singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE  bash -c '
+rm -rf venv
+singularity run $OVERLAY_ARGS $SINGULARITY_IMAGE  bash -c '
     python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR};
     
     source ${ABSOLUTE_PATH}/activate.sh;
     
     pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip
     #conda install -y ipykernel==5.1.3
-conda uninstall -y ipykernel
+    conda uninstall -y ipykernel # For Jupyter-JSC compatibility, it might be necessary to install a certain ipykernel version.
     
     pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt;
 '