From 421c44ccd45c43bd5f93df43ea2519ca7281bc9a Mon Sep 17 00:00:00 2001
From: Stefan Kesselheim <s.kesselheim@fz-juelich.de>
Date: Thu, 10 Nov 2022 11:06:22 +0100
Subject: [PATCH] polishing and example submission script
---
.gitignore | 2 ++
config.sh | 5 ++--
example_submission_script.sh | 22 +++++++++++++++++
setup.sh | 46 +++++++++++++++++++++---------------
4 files changed, 53 insertions(+), 22 deletions(-)
create mode 100644 example_submission_script.sh
diff --git a/.gitignore b/.gitignore
index 5ceb386..8d773cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
venv
+slurm.*
+overlay.*
diff --git a/config.sh b/config.sh
index cd70857..e64f33e 100644
--- a/config.sh
+++ b/config.sh
@@ -6,8 +6,7 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
####################################
### User Configuration
-export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file
+export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file
export ENV_DIR=${ABSOLUTE_PATH}/venv # Default location of this VENV is "./venv"
-export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_tensorflow_21.02-tf1-py3.sif
-export SINGULARITY_IMAGE=/p/project/jb_benchmark/nvidia_singularity_images/nvidia_pytorch_bert_20.06-py3-m5.1.sif
+export SINGULARITY_IMAGE=/p/project/hai_mlperf/oc20_singularity_v4_2022_09_06.sif
diff --git a/example_submission_script.sh b/example_submission_script.sh
new file mode 100644
index 0000000..9629d7d
--- /dev/null
+++ b/example_submission_script.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=2
+#SBATCH --time=00:01:00
+#SBATCH --gres=gpu
+#SBATCH --partition=develbooster
+#SBATCH --account=atmlaml
+
+set -e
+
+######## How to run with container #########
+module purge
+
+source config.sh
+
+echo "Running with image $SINGULARITY_IMAGE"
+echo "Check that your virtual environment is in the PYTHONPATH:"
+srun singularity run $SINGULARITY_IMAGE bash -c '
+ source activate.sh;
+ echo hello world;
+ python -c \"import sys; print(sys.path)\"
+'
diff --git a/setup.sh b/setup.sh
index 9d07baa..1c5402a 100755
--- a/setup.sh
+++ b/setup.sh
@@ -3,27 +3,35 @@ export RELATIVE_PATH=`dirname ${BASH_SOURCE}`
export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
source ${ABSOLUTE_PATH}/config.sh
-echo path ${ABSOLUTE_PATH}
+echo Environment in ${ABSOLUTE_PATH}
+
+#### Optional: Set the Variable Overlay if you want to use the Singularity Overlay feature
+#### This might be necessary if you want to modify the image. Documentation: https://apptainer.org/docs/user/main/persistent_overlays.html
+if [ -z $OVERLAY ]; then
+ rm -f overlay.img
+ rm -rf overlay
+
+ mkdir -p overlay/upper
+ mkdir -p overlay/work
+ dd if=/dev/zero of=overlay.img bs=1M count=500 && \
+ mkfs.ext3 -d overlay overlay.img
+ export OVERLAY_ARGS=" --overlay ${ABSOLUTE_PATH}/overlay.img "
+else
+ export OVERLAY_ARGS=""
+fi
+#### /Overlay
-rm -f overlay.img
-rm -rf overlay
rm -rf venv
-
-mkdir -p overlay/upper
-mkdir -p overlay/work
-dd if=/dev/zero of=overlay.img bs=1M count=500 && \
- mkfs.ext3 -d overlay overlay.img
-
-singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE bash -c '
-python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR};
-
-source ${ABSOLUTE_PATH}/activate.sh;
-
-pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip
-#conda install -y ipykernel==5.1.3
-conda uninstall -y ipykernel
-
-pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt;
+singularity run $OVERLAY_ARGS $SINGULARITY_IMAGE bash -c '
+ python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR};
+
+ source ${ABSOLUTE_PATH}/activate.sh;
+
+ pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip
+ #conda install -y ipykernel==5.1.3
+ conda uninstall -y ipykernel # For Jupyter-JSC compatibility, it might be necessary to install a certain ipykernel version.
+
+ pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt;
'
--
GitLab