modified for no venv

activate.sh

@@ -5,11 +5,13 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
 [[ $0 != $BASH_SOURCE ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ;
 
 source ${ABSOLUTE_PATH}/config.sh
-source ${ABSOLUTE_PATH}/modules.sh
 
 export PYTHONPATH=`echo ${ENV_DIR}/lib/python*/site-packages`:${PYTHONPATH}
 
-source ${ENV_DIR}/bin/activate
+
+export PYTHONUSERBASE=${PWD}/pip
+
+
 
 
 

config.sh

+
 ## Check if this script is sourced
 [[ $0 != $BASH_SOURCE ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ;
 ## Determine location of this file
@@ -7,5 +8,9 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
 
 ### User Configuration
 export ENV_NAME=`basename $ABSOLUTE_PATH`             # Default Name of the venv is the directory that contains this file
-export ENV_DIR=${ABSOLUTE_PATH}/venv         # Default location of this VENV is "./venv"
+export ENV_DIR=${ABSOLUTE_PATH}/pip         # Default location of this VENV is "./venv"
+
+module purge
 
+#export SINGULARITY_IMAGE=/p/project/hai_hhhack/singularity/tensorflow_21.08-tf2-py3.sif
+export SINGULARITY_IMAGE=/p/project/hai_hhhack/singularity/tensorflow_2.6.0-gpu.sif

create_kernel.sh

 #!/bin/bash
 
-RELATIVE_PATH=`dirname ${BASH_SOURCE}`
-ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
+export RELATIVE_PATH=`dirname ${BASH_SOURCE}`
+export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
 source ${ABSOLUTE_PATH}/config.sh
 
-KERNELFILE=${ENV_DIR}/kernel.sh
+export KERNELFILE=${ENV_DIR}/kernel.sh
+export KERNELWRAPPER=${ENV_DIR}/kernel_wrapper.sh
 
 echo the name is $ENV_NAME
 
@@ -18,13 +19,25 @@ exec python -m ipykernel $@' > ${KERNELFILE}
 
 chmod a+x ${KERNELFILE}
 
-mkdir -p ~/.local/share/jupyter/kernels/${ENV_NAME}
+# Setup wrapper for kernel
+
+echo "singularity run $SINGULARITY_IMAGE ${KERNELFILE}"
+echo "#!/bin/bash
+module purge
+singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE  bash -c \"
+${KERNELFILE} \\\"\$@\\\"
+\"
+" > ${KERNELWRAPPER}
+
+chmod a+x ${KERNELWRAPPER}
+
+mkdir -p $OLDHOME/.local/share/jupyter/kernels/${ENV_NAME}
 echo '{
  "argv": [
-  "'"${KERNELFILE}"'", 
+  "'"${KERNELWRAPPER}"'", 
   "-f",
   "{connection_file}"
  ],
  "display_name": "'${ENV_NAME}'",
  "language": "python"
-}' > ~/.local/share/jupyter/kernels/${ENV_NAME}/kernel.json
+}' > $OLDHOME/.local/share/jupyter/kernels/${ENV_NAME}/kernel.json

readme.md

 Supercomputing Environment Template using Python Virtual Environments
 =================
 
-# Idea
+# TLDR
+This repo contains an example on how to easily create a consistent working environment that can be used 
+in the "normal" supercomputer workflow as well as in Jupyter-JSC. 
+
+There are two flavors of this type of repo: 
+1. The "normal" version based on environment modules and a python virtual environment
+2. The singularity version where a Singularity container replaces the module environment
+
+The user creates a working environment as a fork of this repo, developes it along with the requirement of 
+their project and finally archives or publishes it jointly with the results of the research project.
+
+## HowTo Environment Modules
+1. Clone the repo into a folder with the name of the environment module `git clone /path/to/repo ./my_favourite_environment`.
+2. Edit `modules.sh` to select the modules to load prior to creating a virtual environment
+3. Edit `requirements.txt` to select the packages to be installed via pip 
+4. Create the environment
+   * execute `setup.sh` to create the virtual environment
+   * execute `create_kernel.sh` to create a kernel for Jupyer-JSC
+5. `source activate.sh` to enter the environment 
+
+## HowTo Singularity
+1. Clone the repo into a folder with the name of the environment module `git clone /path/to/repo ./my_favourite_environment`.
+2. Edit `config.sh` to select the singularity container.  
+3. If required, edit `setup.sh` to make changes to the container. This is possible as we support a persistent overlay file system
+3. Edit `requirements.txt` to select the packages to be installed via pip 
+4. Create the environment
+   * execute `setup.sh` to create the virtual environment
+   * execute `create_kernel.sh` to create a kernel for Jupyer-JSC
+5. Make sure you execute `source activate.sh` each time you execute something within the container.
+
+# Description
 This project contains a lightweight set of scripts to easily create Python working environments on
 typical supercomputer setups, including creating Jupyter Kernels.
 
+## Environment Modules
 On Supercomputers, typically a basic environment based on **Environment Modules**. This setup is carefully
 curated and optimized, including compilers, MPI version etc. Extra Python packages can be installed
 with pip into user space. This, however, does not create a reproducible environment that can be used
 by other users as well. 
 
-Conceptuall, with Virtual Environments, it is easily possible to create project-based virtual environments.
+Conceptuall, with virtual environments, it is easily possible to create project-based virtual environments.
 These scripts streamline the creation und usage of such environments and make it easy for a users to share a setup
 and to put it under version control with the main code.
 
@@ -19,6 +50,9 @@ developement. In the context of these setups, it is intended to include them as
 them into the workflow. This can e.g. mean that a compilation step is added in the setup step and 
 setting appropriate environment variables is included in the activation step.
 
+## Singularity
+A useful alternative to environment modules are container-based environments. ToDo: More Details
+
 # Details
 The setup is configured in the bash script `config.sh`. The user can define a name for the venv and directory
 where the venv files are stored. This defaults to the directory name of the containing folder and the "." folder 
@@ -45,9 +79,3 @@ by jupyter and a helper script in the virtual environment folder.
 
 
 
-# Intended Workflow
-1. Edit `config.sh` to change name an location of the venv if required.
-2. Edit `modules.sh` to change the modules loaded prior to the creation of the venv.
-3. Edit `requirements.txt` to change the packages to be installed during setup.
-4. Edit `setup.sh` and `activate.sh` to add extra steps for custom modules.
-5. Create a kernel with `create_kernel.sh`

requirements.txt

-opencv-python
-ipykernel
+ipykernel==5.1.3

setup.sh

 #!/bin/bash
-RELATIVE_PATH=`dirname ${BASH_SOURCE}`
-ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
+export RELATIVE_PATH=`dirname ${BASH_SOURCE}`
+export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}`
+export PYTHONUSERBASE=${PWD}/pip
 
 source ${ABSOLUTE_PATH}/config.sh
-source ${ABSOLUTE_PATH}/modules.sh
+echo path ${ABSOLUTE_PATH}
 
-python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR}
+rm -f overlay.img
+rm -rf overlay
+rm -rf venv
 
-source ${ABSOLUTE_PATH}/activate.sh
+mkdir -p overlay/upper
+mkdir -p overlay/work
+dd if=/dev/zero of=overlay.img bs=1M count=500 && \
+     mkfs.ext3 -d overlay overlay.img
 
-pip install -r ${ABSOLUTE_PATH}/requirements.txt
+singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE  bash -c '
+
+pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip
+#conda install -y ipykernel==5.1.3
+conda uninstall -y ipykernel
+
+pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt;
+'