diff --git a/.gitignore b/.gitignore index 5ceb3864c2911029f0a6010fadab352e4b8e2d07..8d773cb3feb1a556dc904e085570bf54ca471687 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ venv +slurm.* +overlay.* diff --git a/config.sh b/config.sh index cd70857f04533e46f929e57ce9f54dbd4da815e0..e64f33ead18ccffa5b01a9e2c1325037c3243d99 100644 --- a/config.sh +++ b/config.sh @@ -6,8 +6,7 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` #################################### ### User Configuration -export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file +export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file export ENV_DIR=${ABSOLUTE_PATH}/venv # Default location of this VENV is "./venv" -export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_tensorflow_21.02-tf1-py3.sif -export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_pytorch_bert_20.06-py3-m5.1.sif +export SINGULARITY_IMAGE=/p/project/hai_mlperf/oc20_singularity_v4_2022_09_06.sif diff --git a/example_submission_script.sh b/example_submission_script.sh new file mode 100644 index 0000000000000000000000000000000000000000..9629d7d42d112381bc4f80deb545334f035410fe --- /dev/null +++ b/example_submission_script.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=2 +#SBATCH --time=00:01:00 +#SBATCH --gres=gpu +#SBATCH --partition=develbooster +#SBATCH --account=atmlaml + +set -e + +######## How to run with container ######### +module purge + +source config.sh + +echo "Running with image $SINGULARITY_IMAGE" +echo "Check that your virtual environment is in the PYTHONPATH:" +srun singularity run $SINGULARITY_IMAGE bash -c ' + source activate.sh; + echo hello world; + python -c \"import sys; print(sys.path)\" +' diff --git a/setup.sh b/setup.sh index 9d07baa790252b899a5bb9230ef541a8e84bdeb1..1c5402a2de31aef4755253c296e5df3f2cb27ac1 100755 --- a/setup.sh +++ b/setup.sh @@ -3,27 +3,35 @@ export RELATIVE_PATH=`dirname ${BASH_SOURCE}` export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` source ${ABSOLUTE_PATH}/config.sh -echo path ${ABSOLUTE_PATH} +echo Environment in ${ABSOLUTE_PATH} + +#### Optional: Set the Variable Overlay if you want to use the Singularity Overlay feature +#### This might be necessary if you want to modify the image. Documentation: https://apptainer.org/docs/user/main/persistent_overlays.html +if [ -z $OVERLAY ]; then + rm -f overlay.img + rm -rf overlay + + mkdir -p overlay/upper + mkdir -p overlay/work + dd if=/dev/zero of=overlay.img bs=1M count=500 && \ + mkfs.ext3 -d overlay overlay.img + export OVERLAY_ARGS=" --overlay ${ABSOLUTE_PATH}/overlay.img " +else + export OVERLAY_ARGS="" +fi +#### /Overlay -rm -f overlay.img -rm -rf overlay rm -rf venv - -mkdir -p overlay/upper -mkdir -p overlay/work -dd if=/dev/zero of=overlay.img bs=1M count=500 && \ - mkfs.ext3 -d overlay overlay.img - -singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE bash -c ' -python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR}; - -source ${ABSOLUTE_PATH}/activate.sh; - -pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip -#conda install -y ipykernel==5.1.3 -conda uninstall -y ipykernel - -pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt; +singularity run $OVERLAY_ARGS $SINGULARITY_IMAGE bash -c ' + python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR}; + + source ${ABSOLUTE_PATH}/activate.sh; + + pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip + #conda install -y ipykernel==5.1.3 + conda uninstall -y ipykernel # For Jupyter-JSC compatibility, it might be necessary to install a certain ipykernel version. + + pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt; '