diff --git a/activate.sh b/activate.sh index bde09a1ed936b98474636e4fb178461f635dd5e7..1ebb7bc0ad75fe7d6c838a84b120b9d1d83eda30 100644 --- a/activate.sh +++ b/activate.sh @@ -1,16 +1,16 @@ #!/bin/bash -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` -[[ $0 != $BASH_SOURCE ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ; +# See https://stackoverflow.com/a/28336473 +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" -source ${ABSOLUTE_PATH}/config.sh -source ${ABSOLUTE_PATH}/modules.sh - -export PYTHONPATH=`echo ${ENV_DIR}/lib/python*/site-packages`:${PYTHONPATH} - -source ${ENV_DIR}/bin/activate +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" +[[ "$0" != "${SOURCE_PATH}" ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ; +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh +export PYTHONPATH="$(echo "${ENV_DIR}"/lib/python*/site-packages):${PYTHONPATH}" +source "${ENV_DIR}"/bin/activate diff --git a/config.sh b/config.sh index 17d0bd7baf8f7115f8276928f969635461c8b1f0..b7de5f78f5b6b124d549cca916c9ea7d3ec56c6c 100644 --- a/config.sh +++ b/config.sh @@ -1,11 +1,12 @@ +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + ## Check if this script is sourced -[[ $0 != $BASH_SOURCE ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ; +[[ "$0" != "${SOURCE_PATH}" ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ; ## Determine location of this file -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" #################################### ### User Configuration -export ENV_NAME=`basename $ABSOLUTE_PATH` # Default Name of the venv is the directory that contains this file -export ENV_DIR=${ABSOLUTE_PATH}/venv # Default location of this VENV is "./venv" - +export ENV_NAME="$(basename "$ABSOLUTE_PATH")" # Default Name of the venv is the directory that contains this file +export ENV_DIR="${ABSOLUTE_PATH}"/venv # Default location of this VENV is "./venv" diff --git a/create_kernel.sh b/create_kernel.sh index 3e79328ac9fc98fbbb3df13c04cbeaf3d620a0e7..e09007b1b2ed78fd9d10c69eb493c13f2670370f 100755 --- a/create_kernel.sh +++ b/create_kernel.sh @@ -1,30 +1,32 @@ #!/bin/bash -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` -source ${ABSOLUTE_PATH}/config.sh +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" -KERNELFILE=${ENV_DIR}/kernel.sh +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" +source "${ABSOLUTE_PATH}"/config.sh -echo the name is $ENV_NAME +KERNELFILE="${ENV_DIR}"/kernel.sh -echo "Setting up the kernel script in the following dir: " ${KERNELFILE} +echo the name is "$ENV_NAME" + +echo "Setting up the kernel script in the following dir: " "${KERNELFILE}" echo '#!/bin/bash -source '"${ABSOLUTE_PATH}"'/activate.sh - -exec python -m ipykernel $@' > ${KERNELFILE} +source "'"${ABSOLUTE_PATH}"'"/activate.sh + +exec python -m ipykernel "$@"' > "${KERNELFILE}" -chmod a+x ${KERNELFILE} +chmod a+x "${KERNELFILE}" -mkdir -p ~/.local/share/jupyter/kernels/${ENV_NAME} +mkdir -p ~/.local/share/jupyter/kernels/"${ENV_NAME}" echo '{ "argv": [ - "'"${KERNELFILE}"'", + "'"${KERNELFILE}"'", "-f", "{connection_file}" ], - "display_name": "'${ENV_NAME}'", + "display_name": "'"${ENV_NAME}"'", "language": "python" -}' > ~/.local/share/jupyter/kernels/${ENV_NAME}/kernel.json +}' > ~/.local/share/jupyter/kernels/"${ENV_NAME}"/kernel.json diff --git a/modules.sh b/modules.sh index d6d8a9d91e04f251b7d12b8cacf143d0e7706282..6512ef50610da295a239302ce077022223222cb1 100644 --- a/modules.sh +++ b/modules.sh @@ -1 +1,2 @@ -ml GCC/9.3.0 ParaStationMPI/5.4.7-1 Python/3.8.5 +module purge +module load GCC/10.3.0 ParaStationMPI/5.4.10-1 Python/3.8.5 diff --git a/readme.md b/readme.md index 138ef2979fd6d319c01c7d18f83a48b9a0e74b2a..0e114403da4c92a1f858a87102bc432678662122 100644 --- a/readme.md +++ b/readme.md @@ -8,23 +8,23 @@ typical supercomputer setups, including creating Jupyter Kernels. On Supercomputers, typically a basic environment based on **Environment Modules**. This setup is carefully curated and optimized, including compilers, MPI version etc. Extra Python packages can be installed with pip into user space. This, however, does not create a reproducible environment that can be used -by other users as well. +by other users as well. -Conceptuall, with Virtual Environments, it is easily possible to create project-based virtual environments. -These scripts streamline the creation und usage of such environments and make it easy for a users to share a setup +Conceptually, with Virtual Environments, it is easily possible to create project-based virtual environments. +These scripts streamline the creation and usage of such environments and make it easy for a users to share a setup and to put it under version control with the main code. Furthermore, in typical compute setup of scientific projects, one or more packages possibly are in active -developement. In the context of these setups, it is intended to include them as submodules and add integrate -them into the workflow. This can e.g. mean that a compilation step is added in the setup step and +development. In the context of these setups, it is intended to include them as submodules and add integrate +them into the workflow. This can e.g. mean that a compilation step is added in the setup step and setting appropriate environment variables is included in the activation step. # Details The setup is configured in the bash script `config.sh`. The user can define a name for the venv and directory -where the venv files are stored. This defaults to the directory name of the containing folder and the "." folder +where the venv files are stored. This defaults to the directory name of the containing folder and the "." folder of the scripts. Please **edit** this file if you want a custom name and location for the venv. -The modules ontop of which the the venv should be built are defined in `modules.sh`. Please **edit** the file +The modules on top of which the the venv should be built are defined in `modules.sh`. Please **edit** the file to your needs. The file `requirements.txt` contains a list of packages to be installed during the setup process. Add required @@ -32,16 +32,16 @@ packages to this file to reproducibly add them to the venv. The script `setup.sh` creates the venv according to the config given in `config.sh`. Please **edit** this file to add a setup step for submodules (e.g. compilation of libraries). If only plain venvs are used, this file -can remain unchanged. +can remain unchanged. Note that the script *must* be ran at least once after the above configurations to actually create the environment. The script `activate.sh` sets the environment variables such that the venv can be used. Please **edit** this file -to add environment variables for submodules. Note that it the script must be *sourced* to take effect. Example: +to add environment variables for submodules. Note that the script must be *sourced* to take effect. Example: ```bash source /activate.sh ``` The script `create_kernel.sh` will create a kernel json file in the user's home directory that can be found -by jupyter and a helper script in the virtual environment folder. +by Jupyter and a helper script in the virtual environment folder. @@ -50,4 +50,5 @@ by jupyter and a helper script in the virtual environment folder. 2. Edit `modules.sh` to change the modules loaded prior to the creation of the venv. 3. Edit `requirements.txt` to change the packages to be installed during setup. 4. Edit `setup.sh` and `activate.sh` to add extra steps for custom modules. -5. Create a kernel with `create_kernel.sh` +5. Create the environment with `bash setup.sh`. +6. Create a kernel with `bash create_kernel.sh`. diff --git a/setup.sh b/setup.sh index dc4ed986b531e27586b580b0d75b5236554a118d..326a7c457733f97ea9809d3ed3c89a72a1776c25 100755 --- a/setup.sh +++ b/setup.sh @@ -1,15 +1,15 @@ #!/bin/bash -RELATIVE_PATH=`dirname ${BASH_SOURCE}` -ABSOLUTE_PATH=`realpath ${RELATIVE_PATH}` -source ${ABSOLUTE_PATH}/config.sh -source ${ABSOLUTE_PATH}/modules.sh +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" -python -m venv --prompt $ENV_NAME --system-site-packages ${ENV_DIR} +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" -source ${ABSOLUTE_PATH}/activate.sh - -pip install -r ${ABSOLUTE_PATH}/requirements.txt +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh +python -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}" +source "${ABSOLUTE_PATH}"/activate.sh +python -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt