diff --git a/activate.sh b/activate.sh index bde09a1ed936b98474636e4fb178461f635dd5e7..fd353995fe52241f49cbb87e5387a903dd8aaaa3 100644 --- a/activate.sh +++ b/activate.sh @@ -5,11 +5,13 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` [[ $0 != $BASH_SOURCE ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ; source ${ABSOLUTE_PATH}/config.sh -source ${ABSOLUTE_PATH}/modules.sh export PYTHONPATH=`echo ${ENV_DIR}/lib/python*/site-packages`:${PYTHONPATH} -source ${ENV_DIR}/bin/activate + +export PYTHONUSERBASE=${PWD}/pip + + diff --git a/config.sh b/config.sh index 17d0bd7baf8f7115f8276928f969635461c8b1f0..20a677cf572801ac16d53aecb4ef000d895b600e 100644 --- a/config.sh +++ b/config.sh @@ -1,3 +1,4 @@ + ## Check if this script is sourced [[ $0 != $BASH_SOURCE ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ; ## Determine location of this file @@ -7,5 +8,9 @@ ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` ### User Configuration export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file -export ENV_DIR=${ABSOLUTE_PATH}/venv # Default location of this VENV is "./venv" +export ENV_DIR=${ABSOLUTE_PATH}/pip # Default location of this VENV is "./venv" + +module purge +#export SINGULARITY_IMAGE=/p/project/hai_hhhack/singularity/tensorflow_21.08-tf2-py3.sif +export SINGULARITY_IMAGE=/p/project/hai_hhhack/singularity/tensorflow_2.6.0-gpu.sif diff --git a/create_kernel.sh b/create_kernel.sh index 3e79328ac9fc98fbbb3df13c04cbeaf3d620a0e7..a33fab50fdb875887e2d269c34663d13693b8228 100755 --- a/create_kernel.sh +++ b/create_kernel.sh @@ -1,10 +1,11 @@ #!/bin/bash -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` +export RELATIVE_PATH=`dirname ${BASH_SOURCE}` +export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` source ${ABSOLUTE_PATH}/config.sh -KERNELFILE=${ENV_DIR}/kernel.sh +export KERNELFILE=${ENV_DIR}/kernel.sh +export KERNELWRAPPER=${ENV_DIR}/kernel_wrapper.sh echo the name is $ENV_NAME @@ -18,13 +19,25 @@ exec python -m ipykernel $@' > ${KERNELFILE} chmod a+x ${KERNELFILE} -mkdir -p ~/.local/share/jupyter/kernels/${ENV_NAME} +# Setup wrapper for kernel + +echo "singularity run $SINGULARITY_IMAGE ${KERNELFILE}" +echo "#!/bin/bash +module purge +singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE bash -c \" +${KERNELFILE} \\\"\$@\\\" +\" +" > ${KERNELWRAPPER} + +chmod a+x ${KERNELWRAPPER} + +mkdir -p $OLDHOME/.local/share/jupyter/kernels/${ENV_NAME} echo '{ "argv": [ - "'"${KERNELFILE}"'", + "'"${KERNELWRAPPER}"'", "-f", "{connection_file}" ], "display_name": "'${ENV_NAME}'", "language": "python" -}' > ~/.local/share/jupyter/kernels/${ENV_NAME}/kernel.json +}' > $OLDHOME/.local/share/jupyter/kernels/${ENV_NAME}/kernel.json diff --git a/readme.md b/readme.md index 138ef2979fd6d319c01c7d18f83a48b9a0e74b2a..389256f5b8c2a9bf6b142608805dd969e223b8e3 100644 --- a/readme.md +++ b/readme.md @@ -1,16 +1,47 @@ Supercomputing Environment Template using Python Virtual Environments ================= -# Idea +# TLDR +This repo contains an example on how to easily create a consistent working environment that can be used +in the "normal" supercomputer workflow as well as in Jupyter-JSC. + +There are two flavors of this type of repo: +1. The "normal" version based on environment modules and a python virtual environment +2. The singularity version where a Singularity container replaces the module environment + +The user creates a working environment as a fork of this repo, developes it along with the requirement of +their project and finally archives or publishes it jointly with the results of the research project. + +## HowTo Environment Modules +1. Clone the repo into a folder with the name of the environment module `git clone /path/to/repo ./my_favourite_environment`. +2. Edit `modules.sh` to select the modules to load prior to creating a virtual environment +3. Edit `requirements.txt` to select the packages to be installed via pip +4. Create the environment + * execute `setup.sh` to create the virtual environment + * execute `create_kernel.sh` to create a kernel for Jupyer-JSC +5. `source activate.sh` to enter the environment + +## HowTo Singularity +1. Clone the repo into a folder with the name of the environment module `git clone /path/to/repo ./my_favourite_environment`. +2. Edit `config.sh` to select the singularity container. +3. If required, edit `setup.sh` to make changes to the container. This is possible as we support a persistent overlay file system +3. Edit `requirements.txt` to select the packages to be installed via pip +4. Create the environment + * execute `setup.sh` to create the virtual environment + * execute `create_kernel.sh` to create a kernel for Jupyer-JSC +5. Make sure you execute `source activate.sh` each time you execute something within the container. + +# Description This project contains a lightweight set of scripts to easily create Python working environments on typical supercomputer setups, including creating Jupyter Kernels. +## Environment Modules On Supercomputers, typically a basic environment based on **Environment Modules**. This setup is carefully curated and optimized, including compilers, MPI version etc. Extra Python packages can be installed with pip into user space. This, however, does not create a reproducible environment that can be used by other users as well. -Conceptuall, with Virtual Environments, it is easily possible to create project-based virtual environments. +Conceptuall, with virtual environments, it is easily possible to create project-based virtual environments. These scripts streamline the creation und usage of such environments and make it easy for a users to share a setup and to put it under version control with the main code. @@ -19,6 +50,9 @@ developement. In the context of these setups, it is intended to include them as them into the workflow. This can e.g. mean that a compilation step is added in the setup step and setting appropriate environment variables is included in the activation step. +## Singularity +A useful alternative to environment modules are container-based environments. ToDo: More Details + # Details The setup is configured in the bash script `config.sh`. The user can define a name for the venv and directory where the venv files are stored. This defaults to the directory name of the containing folder and the "." folder @@ -45,9 +79,3 @@ by jupyter and a helper script in the virtual environment folder. -# Intended Workflow -1. Edit `config.sh` to change name an location of the venv if required. -2. Edit `modules.sh` to change the modules loaded prior to the creation of the venv. -3. Edit `requirements.txt` to change the packages to be installed during setup. -4. Edit `setup.sh` and `activate.sh` to add extra steps for custom modules. -5. Create a kernel with `create_kernel.sh` diff --git a/requirements.txt b/requirements.txt index ac9356d438ff60b7ae219738e889c33c002191c9..98d27be1c398c1717533fb87532592067b4b099d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -opencv-python -ipykernel +ipykernel==5.1.3 diff --git a/setup.sh b/setup.sh index dc4ed986b531e27586b580b0d75b5236554a118d..4c8fb6b63c96454f7afb440eb4f5f0d5afeee8d4 100755 --- a/setup.sh +++ b/setup.sh @@ -1,15 +1,28 @@ #!/bin/bash -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` +export RELATIVE_PATH=`dirname ${BASH_SOURCE}` +export ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` +export PYTHONUSERBASE=${PWD}/pip source ${ABSOLUTE_PATH}/config.sh -source ${ABSOLUTE_PATH}/modules.sh +echo path ${ABSOLUTE_PATH} -python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR} +rm -f overlay.img +rm -rf overlay +rm -rf venv -source ${ABSOLUTE_PATH}/activate.sh +mkdir -p overlay/upper +mkdir -p overlay/work +dd if=/dev/zero of=overlay.img bs=1M count=500 && \ + mkfs.ext3 -d overlay overlay.img -pip install -r ${ABSOLUTE_PATH}/requirements.txt +singularity run --overlay ${ABSOLUTE_PATH}/overlay.img $SINGULARITY_IMAGE bash -c ' + +pip install --cache-dir ${ABSOLUTE_PATH}/cache --upgrade pip +#conda install -y ipykernel==5.1.3 +conda uninstall -y ipykernel + +pip install --cache-dir ${ABSOLUTE_PATH}/cache -r ${ABSOLUTE_PATH}/requirements.txt; +'