From 6230249d6ac20e707ec59e72ee1c49e9eb7bef8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jens=20Henrik=20G=C3=B6bbert?= <goebbert1@jwlogin10.juwels>
Date: Fri, 31 Mar 2023 13:25:53 +0200
Subject: [PATCH] update build instruction for kernels
---
03-HowTos/Create_JupyterKernel_general.ipynb | 153 ++++++++-----------
1 file changed, 62 insertions(+), 91 deletions(-)
diff --git a/03-HowTos/Create_JupyterKernel_general.ipynb b/03-HowTos/Create_JupyterKernel_general.ipynb
index 79dea0b..b714b34 100644
--- a/03-HowTos/Create_JupyterKernel_general.ipynb
+++ b/03-HowTos/Create_JupyterKernel_general.ipynb
@@ -28,6 +28,11 @@
"\n",
"This notebook shows you how you can build your own kernel for a **python environment**.\n",
"\n",
+ "<div class=\"alert alert-block alert-info\">\n",
+ "<b>Attention:</b>\n",
+ "This notebook is meant to run out of a JupyterLab on JSC's HPC systems.</br>\n",
+ "</div>\n",
+ "\n",
"-------------------------"
]
},
@@ -55,7 +60,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* Set kernel name\n",
+ "#### Set the kernel name\n",
" - must be lower case\n",
" - change if you like"
]
@@ -77,7 +82,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* List directories where JupyterLab will search for kernels"
+ "#### Set the kernel directory\n",
+ " - check that the kernel name is unique\n",
+ " - print the location of the new kernel"
]
},
{
@@ -86,55 +93,10 @@
"metadata": {},
"outputs": [],
"source": [
- "# JUPYTER SEARCH PATH (for kernels-directory)\n",
- "echo \"jupyter search paths for kernels-directories\"\n",
- "if [ -z $JUPYTER_PATH ]; then\n",
- " echo \"$HOME/.local/share/jupyter\"\n",
- "else\n",
- " tr ':' '\\n' <<< \"$JUPYTER_PATH\"\n",
- "fi"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "<div class=\"alert alert-block alert-info\">\n",
- "<b>Attention:</b>\n",
- "Please choose 'private kernel' if you are unsure.</br>\n",
- "Using 'project kernel's need to be enabled for your project first by our Jupyter-JSC admins.\n",
- "</div>\n",
- "\n",
- "* Set kernel type\n",
- " - private kernel = \"\\${HOME}/.local/\" \n",
- " - project kernel = \"\\${PROJECT}/.local/\" \n",
- " - other kernel = \"\\<your-path\\>\" (ensure it is part of $JUPYTER_PATH or your kernel will not be found by JuypterLab)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# INPUT NEEDED:\n",
- "export KERNEL_TYPE=private # private, project or other\n",
+ "# define KERNEL_SPECS_DIR\n",
"export KERNEL_SPECS_PREFIX=${HOME}/.local\n",
- "\n",
- "###################\n",
- "# project kernel\n",
- "if [ \"${KERNEL_TYPE}\" == \"project\" ]; then\n",
- " export KERNEL_SPECS_PREFIX=${PROJECT}/.local\n",
- " echo \"project kernel\"\n",
- "# private kernel\n",
- "elif [ \"${KERNEL_TYPE}\" == \"private\" ]; then\n",
- " export KERNEL_SPECS_PREFIX=${HOME}/.local\n",
- " echo \"private kernel\"\n",
- "else\n",
- " if [ ! -d \"$KERNEL_SPECS_PREFIX\" ]; then\n",
- " echo \"ERROR: please create directory $KERNEL_SPECS_PREFIX\"\n",
- " fi\n",
- " echo \"other kernel\"\n",
+ "if [ ! -d \"$KERNEL_SPECS_PREFIX\" ]; then\n",
+ " echo \"ERROR: please create directory $KERNEL_SPECS_PREFIX\"\n",
"fi\n",
"export KERNEL_SPECS_DIR=${KERNEL_SPECS_PREFIX}/share/jupyter/kernels\n",
"\n",
@@ -144,15 +106,17 @@
" echo \" Rename kernel name or remove directory.\"\n",
"fi\n",
"\n",
- "echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} # double check"
+ "# print the location of the new kernel\n",
+ "echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "* Set directory for kernels virtual environment\n",
- " - change if you like"
+ "#### Set the kernel's virtual environment\n",
+ " - by default it is located at $PROJECT\n",
+ " - print the location of the new kernels virtual environment"
]
},
{
@@ -161,18 +125,12 @@
"metadata": {},
"outputs": [],
"source": [
- "# INPUT NEEDED:\n",
+ "# define KERNEL_VENVS_DIR\n",
"export KERNEL_VENVS_DIR=${PROJECT}/${USER}/jupyter/kernels\n",
- "\n",
- "###################\n",
"mkdir -p ${KERNEL_VENVS_DIR}\n",
- "if [ \"${KERNEL_TYPE}\" != \"private\" ] && [ \"${KERNEL_TYPE}\" != \"other\" ]; then\n",
- " echo \"Please check the permissions and ensure your project partners have read/execute permissions:\"\n",
- " namei -l ${KERNEL_VENVS_DIR}\n",
- "fi\n",
"\n",
- "echo ${KERNEL_VENVS_DIR} # double check\n",
- "ls -lt ${KERNEL_VENVS_DIR}"
+ "# print the location of the new kernels virtual environment\n",
+ "echo ${KERNEL_VENVS_DIR}"
]
},
{
@@ -193,7 +151,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.1 - Load basic Python module"
+ "#### 1.1 - Load basic Python module"
]
},
{
@@ -203,18 +161,27 @@
"outputs": [],
"source": [
"module -q purge\n",
- "module -q use $OTHERSTAGES \n",
- "module -q load Stages/2020 2> /dev/null # any stage can be used\n",
- "module -q load GCCcore/.9.3.0 2> /dev/null\n",
- "module -q load Python/3.8.5 # only Python is required\n",
- "module list # double check"
+ "module -q load Stages/2023 # any stage can be used\n",
+ "module -q load GCC\n",
+ "module -q load Python # only Python is required"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# get Python version\n",
+ "export PYV=$(python -c 'import sys; print(\".\".join(map(str, sys.version_info[:2])))')\n",
+ "echo $PYV"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.2 - Load extra modules you need for your kernel"
+ "#### 1.2 - Load extra modules you need for your kernel"
]
},
{
@@ -230,7 +197,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.3 - Create and activate a virtual environment for the kernel \n",
+ "#### 1.3 - Create and activate a virtual environment for the kernel \n",
"and ensure python packages installed in the virtual environment are always prefered"
]
},
@@ -240,13 +207,14 @@
"metadata": {},
"outputs": [],
"source": [
- "if [ -d \"${KERNEL_VENVS_DIR}/${KERNEL_NAME}\" ]; then\n",
- " echo \"ERROR: Directory for virtual environment already ${KERNEL_VENVS_DIR}/${KERNEL_NAME}\"\n",
+ "export VIRTUAL_ENV=${KERNEL_VENVS_DIR}/${KERNEL_NAME}\n",
+ "if [ -d \"${VIRTUAL_ENV}\" ]; then\n",
+ " echo \"ERROR: Directory for virtual environment already ${VIRTUAL_ENV}\"\n",
" echo \" Rename kernel name or remove directory.\"\n",
"else\n",
- " python -m venv --system-site-packages ${KERNEL_VENVS_DIR}/${KERNEL_NAME}\n",
- " source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate\n",
- " export PYTHONPATH=${VIRTUAL_ENV}/lib/python3.8/site-packages:${PYTHONPATH}\n",
+ " python -m venv --system-site-packages ${VIRTUAL_ENV}\n",
+ " source ${VIRTUAL_ENV}/bin/activate\n",
+ " export PYTHONPATH=${VIRTUAL_ENV}/lib/python${PYV}/site-packages:${PYTHONPATH}\n",
" echo ${VIRTUAL_ENV} # double check\n",
"fi"
]
@@ -255,7 +223,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.4 - Install Python libraries required for communication with Jupyter"
+ "#### 1.4 - Install Python libraries required for communication with Jupyter"
]
},
{
@@ -268,8 +236,7 @@
"if [ -z \"${VIRTUAL_ENV}\" ]; then\n",
" echo \"ERROR: Virtual environment not successfully initialized.\"\n",
"else\n",
- " pip install --ignore-installed ipykernel\n",
- " ls ${VIRTUAL_ENV}/lib/python3.8/site-packages/ # double check\n",
+ " pip install ipykernel\n",
"fi"
]
},
@@ -277,7 +244,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.5 - Install whatever else you need in your Python virtual environment (using pip)"
+ "#### 1.5 - Install whatever else you need in your Python virtual environment (using pip)"
]
},
{
@@ -307,7 +274,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 2.1 - Create launch script, which loads your Python virtual environment and starts the ipykernel process inside:\n",
+ "#### 2.1 - Create launch script, which loads your Python virtual environment and starts the ipykernel process inside:\n",
"\n",
"<div class=\"alert alert-block alert-info\">\n",
"<b>Attention:</b>\n",
@@ -325,19 +292,18 @@
"\n",
"# Load basic Python module\n",
"module purge\n",
- "module use \"'$OTHERSTAGES'\"\n",
- "module load Stages/2020\n",
- "module load GCCcore/.9.3.0\n",
- "module load Python/3.8.5\n",
+ "module load Stages/2023\n",
+ "module load GCC\n",
+ "module load Python\n",
"\n",
"# Load extra modules you need for your kernel (as you did in step 1.2)\n",
"#module load <module you need>\n",
" \n",
"# Activate your Python virtual environment\n",
- "source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate\n",
+ "source ${VIRTUAL_ENV}/bin/activate\n",
" \n",
"# Ensure python packages installed in the virtual environment are always prefered\n",
- "export PYTHONPATH=${VIRTUAL_ENV}/lib/python3.8/site-packages:\"'${PYTHONPATH}'\"\n",
+ "export PYTHONPATH=${VIRTUAL_ENV}/lib/python${PYV}/site-packages:\"'${PYTHONPATH}'\"\n",
" \n",
"exec python -m ipykernel \"'$@' > ${VIRTUAL_ENV}/kernel.sh\n",
"chmod +x ${VIRTUAL_ENV}/kernel.sh\n",
@@ -363,7 +329,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.1 - Create Jupyter kernel configuration directory and files"
+ "#### 3.1 - Create Jupyter kernel configuration directory and files"
]
},
{
@@ -380,7 +346,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.2 - Adjust kernel.json file"
+ "#### 3.2 - Adjust kernel.json file"
]
},
{
@@ -400,7 +366,10 @@
" \"{connection_file}\"\n",
" ],\n",
" \"display_name\": \"'${KERNEL_NAME}'\",\n",
- " \"language\": \"python\"\n",
+ " \"language\": \"python\",\n",
+ " \"metadata\": {\n",
+ " \"debugger\": true\n",
+ " }\n",
"}' > ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json\n",
"\n",
"cat ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json # double check"
@@ -410,7 +379,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.3 - Create link to kernel specs"
+ "#### 3.3 - Create link to kernel specs"
]
},
{
@@ -431,7 +400,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.4 - You can select the new kernel in the top right corner of your notebook. The kernel icon will be added to your launcher, once you've restarted the JupyterLab"
+ "#### 3.4 - Use the kernel\n",
+ "- You can select the new kernel in the top right corner of your notebook or from JupyterLab's Launchpad\n",
+ "- The kernel icon will be added to your launcher, after a while by JupyterLab automatically or once you've restarted the JupyterLab"
]
},
{
--
GitLab