diff --git a/day1_introduction/5_create-kernels/2-create_JupyterKernel_general.ipynb b/day1_introduction/5_create-kernels/2-create_JupyterKernel_general.ipynb
index ea7a6ab75d8ffb252208ddf2980a0f7661c62be0..00124d198187b823c240b4064eeb252a6ff28d75 100644
--- a/day1_introduction/5_create-kernels/2-create_JupyterKernel_general.ipynb
+++ b/day1_introduction/5_create-kernels/2-create_JupyterKernel_general.ipynb
@@ -29,6 +29,11 @@
"\n",
"This notebook shows you how you can build your own kernel for a **python environment**.\n",
"\n",
+ "<div class=\"alert alert-block alert-info\">\n",
+ "<b>Attention:</b>\n",
+ "This notebook is meant to run out of a JupyterLab on JSC's HPC systems.</br>\n",
+ "</div>\n",
+ "\n",
"-------------------------"
]
},
@@ -56,27 +61,21 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* Set kernel name\n",
+ "#### Set kernel name\n",
" - must be lower case\n",
" - change if you like"
]
},
{
"cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "jovyan_kernel_ecar2023\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"# INPUT NEEDED:\n",
- "KERNEL_NAME=$(whoami)_kernel_ecar2023\n",
+ "KERNEL_NAME=$(whoami)_kernel\n",
"\n",
"export KERNEL_NAME=$(echo \"${KERNEL_NAME}\" | awk '{print tolower($0)}')\n",
"echo ${KERNEL_NAME} # double check"
@@ -86,37 +85,22 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* List directories where JupyterLab will search for kernels"
+ "#### List directories where JupyterLab will search for kernels"
]
},
{
"cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "jupyter search paths for kernels-directories\n",
- "/home/jovyan/.local/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-Ruby/3.0.1-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-R/4.1.2-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-PyVisualization/1.0-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-PyQuantum/3.0-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-PyDeepLearning/1.1-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-Octave/6.4.0-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-Julia/1.7.1-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-JavaScript/5.2.1-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-Cling/0.9-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/JupyterKernel-Bash/0.7.2-gcccoremkl-11.2.0-2021.4.0-2022.3.4/share/jupyter\n",
- "/p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/share/jupyter\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
+ "tags": []
+ },
+ "outputs": [],
"source": [
"# JUPYTER SEARCH PATH (for kernels-directory)\n",
- "echo \"jupyter search paths for kernels-directories\"\n",
+ "echo \"jupyter search paths for kernel-directories\"\n",
"if [ -z $JUPYTER_PATH ]; then\n",
" echo \"$HOME/.local/share/jupyter\"\n",
"else\n",
@@ -126,53 +110,27 @@
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"source": [
- "<div class=\"alert alert-block alert-info\">\n",
- "<b>Attention:</b>\n",
- "Please choose 'private kernel' if you are unsure.</br>\n",
- "Using 'project kernel's need to be enabled for your project first by our Jupyter-JSC admins.\n",
- "</div>\n",
- "\n",
- "* Set kernel type\n",
- " - private kernel = \"\\${HOME}/.local/\" \n",
- " - project kernel = \"\\${PROJECT}/.local/\" \n",
- " - other kernel = \"\\<your-path\\>\" (ensure it is part of $JUPYTER_PATH or your kernel will not be found by JuypterLab)"
+ "#### Set the kernel directory\n",
+ " - check that the kernel name is unique\n",
+ " - print the location of the new kernel"
]
},
{
"cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "private kernel\n",
- "/home/jovyan/.local/share/jupyter/kernels/jovyan_kernel_ecar2023\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
- "# INPUT NEEDED:\n",
- "export KERNEL_TYPE=private # private, project or other\n",
+ "# define KERNEL_SPECS_DIR\n",
"export KERNEL_SPECS_PREFIX=${HOME}/.local\n",
- "\n",
- "###################\n",
- "# project kernel\n",
- "if [ \"${KERNEL_TYPE}\" == \"project\" ]; then\n",
- " export KERNEL_SPECS_PREFIX=${PROJECT}/.local\n",
- " echo \"project kernel\"\n",
- "# private kernel\n",
- "elif [ \"${KERNEL_TYPE}\" == \"private\" ]; then\n",
- " export KERNEL_SPECS_PREFIX=${HOME}/.local\n",
- " echo \"private kernel\"\n",
- "else\n",
- " if [ ! -d \"$KERNEL_SPECS_PREFIX\" ]; then\n",
- " echo \"ERROR: please create directory $KERNEL_SPECS_PREFIX\"\n",
- " fi\n",
- " echo \"other kernel\"\n",
+ "if [ ! -d \"$KERNEL_SPECS_PREFIX\" ]; then\n",
+ " echo \"ERROR: please create directory $KERNEL_SPECS_PREFIX\"\n",
"fi\n",
"export KERNEL_SPECS_DIR=${KERNEL_SPECS_PREFIX}/share/jupyter/kernels\n",
"\n",
@@ -182,49 +140,40 @@
" echo \" Rename kernel name or remove directory.\"\n",
"fi\n",
"\n",
- "echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} # double check"
+ "# print the location of the new kernel\n",
+ "echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "* Set directory for kernels virtual environment\n",
- " - change if you like"
+ "#### Set the kernel's virtual environment\n",
+ " - by default it is located at $PROJECT\n",
+ " - print the location of the new kernels virtual environment"
]
},
{
"cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/home/jovyan/jupyter_kernel_venvs\n",
- "total 0\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
- "# INPUT NEEDED:\n",
- "export KERNEL_VENVS_DIR=${HOME}/jupyter_kernel_venvs\n",
- "\n",
- "###################\n",
+ "# define KERNEL_VENVS_DIR\n",
+ "export KERNEL_VENVS_DIR=${PROJECT}/${USER}/jupyter/kernels\n",
"mkdir -p ${KERNEL_VENVS_DIR}\n",
- "if [ \"${KERNEL_TYPE}\" != \"private\" ] && [ \"${KERNEL_TYPE}\" != \"other\" ]; then\n",
- " echo \"Please check the permissions and ensure your project partners have read/execute permissions:\"\n",
- " namei -l ${KERNEL_VENVS_DIR}\n",
- "fi\n",
"\n",
- "echo ${KERNEL_VENVS_DIR} # double check\n",
- "ls -lt ${KERNEL_VENVS_DIR}"
+ "# print the location of the new kernels virtual environment\n",
+ "echo ${KERNEL_VENVS_DIR}"
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"source": [
"---"
]
@@ -240,204 +189,83 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.1 - Load basic Python module"
+ "#### 1.1 - Load basic Python module"
]
},
{
"cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "The following modules were not unloaded:\n",
- " (Use \"module --force purge\" to unload all):\n",
- "\n",
- " 1) Stages/2022\n",
- "\n",
- "Currently Loaded Modules:\n",
- " 1) Stages/2022 (S) 16) libxslt/.1.1.34 (H)\n",
- " 2) GCCcore/.11.2.0 (H) 17) libyaml/.0.2.5 (H)\n",
- " 3) zlib/.1.2.11 (H) 18) PostgreSQL/13.4\n",
- " 4) binutils/.2.37 (H) 19) gflags/.2.2.2 (H)\n",
- " 5) GCC/11.2.0 20) libspatialindex/.1.9.3 (H)\n",
- " 6) bzip2/.1.0.8 (H) 21) NASM/.2.15.05 (H)\n",
- " 7) ncurses/.6.2 (H) 22) libjpeg-turbo/.2.1.1 (H)\n",
- " 8) libreadline/.8.1 (H) 23) Python/3.9.6\n",
- " 9) Tcl/8.6.11 24) numactl/2.0.14\n",
- " 10) SQLite/.3.36 (H) 25) nvidia-driver/.default (H,g)\n",
- " 11) XZ/.5.2.5 (H) 26) CUDA/11.5 (g)\n",
- " 12) GMP/6.2.1 27) UCX/default (g)\n",
- " 13) libffi/.3.4.2 (H) 28) pscom/.5.4-default (H)\n",
- " 14) OpenSSL/1.1 29) mpi-settings/UCX\n",
- " 15) libxml2/.2.9.10 (H) 30) ParaStationMPI/5.5.0-1 (g)\n",
- "\n",
- " Where:\n",
- " H: Hidden Module\n",
- " S: Module is Sticky, requires --force to unload or purge\n",
- " g: built for GPU\n",
- "\n",
- " \n",
- "\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
- "module purge \n",
- "module load Stages/2022 # any stage can be used\n",
- "\n",
- "module load GCC/11.2.0 \n",
- "module load Python/3.9.6 # only Python is required\n",
- "module load ParaStationMPI\n",
- "\n",
- "module list # double check"
+ "module -q purge \n",
+ "module -q load Stages/2024 # any stage can be used\n",
+ "module -q load GCC\n",
+ "module -q load Python # only Python is required\n",
+ "# module list # double check"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# get Python version\n",
+ "export PYV=$(python -c 'import sys; print(\".\".join(map(str, sys.version_info[:2])))')\n",
+ "echo $PYV"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.2 - Load extra modules you need for your kernel"
+ "#### 1.2 - Load extra modules you need for your kernel"
]
},
{
"cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "Currently Loaded Modules:\n",
- " 1) Stages/2022 (S) 70) MPFR/4.1.0\n",
- " 2) GCCcore/.11.2.0 (H) 71) MPC/1.2.1\n",
- " 3) zlib/.1.2.11 (H) 72) gmpy2/2.1.0b5\n",
- " 4) binutils/.2.37 (H) 73) sympy/1.8\n",
- " 5) GCC/11.2.0 74) xarray/0.20.1\n",
- " 6) bzip2/.1.0.8 (H) 75) SciPy-Stack/2021b\n",
- " 7) ncurses/.6.2 (H) 76) x264/.20210613 (H)\n",
- " 8) libreadline/.8.1 (H) 77) LAME/.3.100 (H)\n",
- " 9) Tcl/8.6.11 78) x265/.3.4 (H)\n",
- " 10) SQLite/.3.36 (H) 79) libvpx/1.11.0\n",
- " 11) XZ/.5.2.5 (H) 80) FriBidi/1.0.10\n",
- " 12) GMP/6.2.1 81) FFmpeg/.4.4.1 (H,g)\n",
- " 13) libffi/.3.4.2 (H) 82) nettle/.3.7.3 (H)\n",
- " 14) OpenSSL/1.1 83) libdrm/.2.4.108 (H)\n",
- " 15) libxml2/.2.9.10 (H) 84) LLVM/13.0.0\n",
- " 16) libxslt/.1.1.34 (H) 85) libunwind/.1.5.0 (H)\n",
- " 17) libyaml/.0.2.5 (H) 86) OpenGL/2021b (g)\n",
- " 18) PostgreSQL/13.4 87) freeglut/.3.2.1 (H)\n",
- " 19) gflags/.2.2.2 (H) 88) JasPer/.2.0.33 (H)\n",
- " 20) libspatialindex/.1.9.3 (H) 89) gettext/.0.21 (H)\n",
- " 21) NASM/.2.15.05 (H) 90) PCRE/.8.45 (H)\n",
- " 22) libjpeg-turbo/.2.1.1 (H) 91) GLib/.2.69.1 (H)\n",
- " 23) Python/3.9.6 92) ATK/.2.36.0 (H)\n",
- " 24) numactl/2.0.14 93) DBus/.1.13.18 (H)\n",
- " 25) nvidia-driver/.default (H,g) 94) DB/.18.1.40 (H)\n",
- " 26) CUDA/11.5 (g) 95) Perl/5.34.0\n",
- " 27) UCX/default (g) 96) intltool/.0.51.0 (H)\n",
- " 28) pscom/.5.4-default (H) 97) AT-SPI2-core/.2.40.3 (H)\n",
- " 29) mpi-settings/UCX 98) AT-SPI2-ATK/.2.38.0 (H)\n",
- " 30) ParaStationMPI/5.5.0-1 (g) 99) Gdk-Pixbuf/.2.42.6 (H)\n",
- " 31) imkl/.2021.4.0 (H) 100) pixman/.0.40.0 (H)\n",
- " 32) cuDNN/8.3.1.22-CUDA-11.5 (g) 101) cairo/.1.16.0 (H)\n",
- " 33) NCCL/2.12.7-1-CUDA-11.5 (g) 102) HarfBuzz/.2.8.2 (H)\n",
- " 34) pybind11/.2.7.1 (H) 103) Pango/.1.48.8 (H)\n",
- " 35) SciPy-bundle/2021.10 104) libepoxy/.1.5.9 (H)\n",
- " 36) Szip/.2.1.1 (H) 105) libcroco/.0.6.13 (H)\n",
- " 37) HDF5/1.12.1-serial 106) Rust/1.54.0\n",
- " 38) h5py/3.5.0-serial 107) librsvg/.2.51.2 (H)\n",
- " 39) cURL/7.78.0 108) GTK+/.3.24.23 (H)\n",
- " 40) double-conversion/3.1.6 109) Eigen/3.3.9\n",
- " 41) flatbuffers/.2.0.0 (H) 110) OpenEXR/3.1.1\n",
- " 42) giflib/.5.2.1 (H) 111) OpenCV/4.5.4\n",
- " 43) libpciaccess/.0.16 (H) 112) Pandoc/2.16.1\n",
- " 44) hwloc/2.5.0 (g) 113) texlive/20200406\n",
- " 45) ICU/.70.1 (H) 114) SWIG/.4.0.2 (H)\n",
- " 46) JsonCpp/.1.9.4 (H) 115) FFTW/.3.3.10-nompi (H)\n",
- " 47) LMDB/.0.9.29 (H) 116) tbb/2020.3\n",
- " 48) nsync/.1.24.0 (H) 117) PCRE2/10.37\n",
- " 49) protobuf/.3.17.3 (H) 118) libevent/.2.1.12 (H)\n",
- " 50) protobuf-python/.3.17.3 (H) 119) NSPR/4.32\n",
- " 51) flatbuffers-python/2.0 120) NSS/3.73\n",
- " 52) typing-extensions/3.10.0.0 121) Qt5/.5.15.2 (H)\n",
- " 53) libpng/.1.6.37 (H) 122) ITK/5.2.1-nompi\n",
- " 54) snappy/.1.1.9 (H) 123) netCDF/4.8.1-serial\n",
- " 55) TensorFlow/2.6.0-CUDA-11.5 (g) 124) netcdf4-python/1.5.7-serial\n",
- " 56) Brotli/1.0.9 125) git/2.33.1-nodocs\n",
- " 57) freetype/.2.11.0 (H) 126) scikit-build/0.11.1\n",
- " 58) expat/.2.4.1 (H) 127) scikit-learn/1.0.1\n",
- " 59) util-linux/.2.37 (H) 128) PyYAML/5.4.1\n",
- " 60) fontconfig/.2.13.94 (H) 129) bokeh/2.4.1\n",
- " 61) xorg-macros/.1.19.3 (H) 130) dask/2021.9.1\n",
- " 62) X11/20210802 131) scikit-image/0.18.3\n",
- " 63) Tk/.8.6.11 (H) 132) numba/0.55.1 (g)\n",
- " 64) Tkinter/3.9.6 133) GEOS/3.9.1\n",
- " 65) LibTIFF/.4.3.0 (H) 134) Shapely/1.8.0\n",
- " 66) Pillow-SIMD/9.0.1 135) nodejs/.16.13.0 (H)\n",
- " 67) Qhull/.2020.2 (H) 136) yarn/1.22.17\n",
- " 68) matplotlib/3.4.3 137) Jupyter/2022.3.4\n",
- " 69) Seaborn/0.11.2\n",
- "\n",
- " Where:\n",
- " H: Hidden Module\n",
- " S: Module is Sticky, requires --force to unload or purge\n",
- " g: built for GPU\n",
- "\n",
- " \n",
- "\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"# module load <module you need>\n",
- "module load TensorFlow\n",
- "module load OpenCV\n",
- "module load Jupyter\n",
+ "module load OpenCV # example\n",
"\n",
- "module list"
+ "# module list # double check"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.3 - Create and activate a virtual environment for the kernel \n",
+ "#### 1.3 - Create and activate a virtual environment for the kernel \n",
+ "\n",
"and ensure python packages installed in the virtual environment are always prefered"
]
},
{
"cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023\n",
- "(jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
- "if [ -d \"${KERNEL_VENVS_DIR}/${KERNEL_NAME}\" ]; then\n",
- " echo \"ERROR: Directory for virtual environment already ${KERNEL_VENVS_DIR}/${KERNEL_NAME}\"\n",
+ "export VIRTUAL_ENV=${KERNEL_VENVS_DIR}/${KERNEL_NAME}\n",
+ "if [ -d \"${VIRTUAL_ENV}\" ]; then\n",
+ " echo \"ERROR: Directory for virtual environment already ${VIRTUAL_ENV}\"\n",
" echo \" Rename kernel name or remove directory.\"\n",
"else\n",
- " python -m venv --system-site-packages ${KERNEL_VENVS_DIR}/${KERNEL_NAME}\n",
- " source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate\n",
- " export PYTHONPATH=${VIRTUAL_ENV}/lib/python3.9/site-packages:${PYTHONPATH}\n",
+ " python -m venv --system-site-packages ${VIRTUAL_ENV}\n",
+ " source ${VIRTUAL_ENV}/bin/activate\n",
+ " export PYTHONPATH=${VIRTUAL_ENV}/lib/python${PYV}/site-packages:${PYTHONPATH}\n",
" echo ${VIRTUAL_ENV} # double check\n",
"fi"
]
@@ -446,66 +274,23 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 1.4 - Install Python libraries required for communication with Jupyter"
+ "#### 1.4 - Install Python libraries required for communication with Jupyter"
]
},
{
"cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/bin/pip\n",
- "(jovyan_kernel_ecar2023) Requirement already satisfied: ipykernel in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (6.13.0)\n",
- "Requirement already satisfied: packaging in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipykernel) (20.9)\n",
- "Requirement already satisfied: psutil in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipykernel) (5.8.0)\n",
- "Requirement already satisfied: nest-asyncio in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from ipykernel) (1.5.5)\n",
- "Requirement already satisfied: tornado>=6.1 in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from ipykernel) (6.2)\n",
- "Requirement already satisfied: matplotlib-inline>=0.1 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipykernel) (0.1.3)\n",
- "Requirement already satisfied: ipython>=7.23.1 in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from ipykernel) (7.34.0)\n",
- "Requirement already satisfied: jupyter-client>=6.1.12 in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from ipykernel) (7.1.2)\n",
- "Requirement already satisfied: traitlets>=5.1.0 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipykernel) (5.1.0)\n",
- "Requirement already satisfied: debugpy>=1.0 in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from ipykernel) (1.6.0)\n",
- "Requirement already satisfied: pickleshare in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (0.7.5)\n",
- "Requirement already satisfied: setuptools>=18.5 in /home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (56.0.0)\n",
- "Requirement already satisfied: pygments in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (2.9.0)\n",
- "Requirement already satisfied: jedi>=0.16 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (0.18.0)\n",
- "Requirement already satisfied: backcall in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (0.2.0)\n",
- "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (3.0.20)\n",
- "Requirement already satisfied: decorator in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (5.0.9)\n",
- "Requirement already satisfied: pexpect>4.3 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from ipython>=7.23.1->ipykernel) (4.8.0)\n",
- "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel) (0.8.2)\n",
- "Requirement already satisfied: python-dateutil>=2.1 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel) (2.8.2)\n",
- "Requirement already satisfied: pyzmq>=13 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel) (22.3.0)\n",
- "Requirement already satisfied: entrypoints in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel) (0.4)\n",
- "Requirement already satisfied: jupyter-core>=4.6.0 in /p/software/hdfcloud/stages/2022/software/Jupyter/2022.3.4-gcccoremkl-11.2.0-2021.4.0/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel) (4.9.1)\n",
- "Requirement already satisfied: ptyprocess>=0.5 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel) (0.7.0)\n",
- "Requirement already satisfied: wcwidth in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=7.23.1->ipykernel) (0.2.5)\n",
- "Requirement already satisfied: six>=1.5 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from python-dateutil>=2.1->jupyter-client>=6.1.12->ipykernel) (1.16.0)\n",
- "Requirement already satisfied: pyparsing>=2.0.2 in /p/software/hdfcloud/stages/2022/software/Python/3.9.6-GCCcore-11.2.0/lib/python3.9/site-packages (from packaging->ipykernel) (2.4.7)\n",
- "_distutils_hack pip-21.1.3.dist-info setuptools-56.0.0.dist-info\n",
- "distutils-precedence.pth pkg_resources\n",
- "pip setuptools\n",
- "(jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"which pip\n",
"if [ -z \"${VIRTUAL_ENV}\" ]; then\n",
" echo \"ERROR: Virtual environment not successfully initialized.\"\n",
"else\n",
" pip install ipykernel\n",
- " ls ${VIRTUAL_ENV}/lib/python3.9/site-packages/ # double check\n",
+ " ls ${VIRTUAL_ENV}/lib/python${PYV}/site-packages/ # double check\n",
"fi"
]
},
@@ -519,7 +304,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"outputs": [],
"source": [
"#pip install <python-package you need>"
@@ -553,68 +340,29 @@
},
{
"cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) #!/bin/bash\n",
- "\n",
- "# Load basic Python module\n",
- "module purge \n",
- "module load Stages/2022\n",
- "\n",
- "module load GCC/11.2.0 \n",
- "module load Python/3.9.6\n",
- "module load ParaStationMPI\n",
- "\n",
- "# Load extra modules you need for your kernel (as you did in step 1.2)\n",
- "#module load <module you need>\n",
- "module load TensorFlow\n",
- "module load OpenCV\n",
- "module load Jupyter\n",
- " \n",
- "# Activate your Python virtual environment\n",
- "source /home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/bin/activate\n",
- " \n",
- "# Ensure python packages installed in the virtual environment are always prefered\n",
- "export PYTHONPATH=/home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/lib/python3.9/site-packages:${PYTHONPATH}\n",
- " \n",
- "exec python -m ipykernel $@\n",
- "(jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"echo '#!/bin/bash'\"\n",
"\n",
"# Load basic Python module\n",
"module purge \n",
- "module load Stages/2022\n",
- "\n",
- "module load GCC/11.2.0 \n",
- "module load Python/3.9.6\n",
- "module load ParaStationMPI\n",
+ "module load Stages/2024\n",
+ "module load GCC \n",
+ "module load Python\n",
"\n",
"# Load extra modules you need for your kernel (as you did in step 1.2)\n",
- "#module load <module you need>\n",
- "module load TensorFlow\n",
+ "# module load <module you need>\n",
"module load OpenCV\n",
- "module load Jupyter\n",
" \n",
"# Activate your Python virtual environment\n",
- "source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate\n",
+ "source ${VIRTUAL_ENV}/bin/activate\n",
" \n",
"# Ensure python packages installed in the virtual environment are always prefered\n",
- "export PYTHONPATH=${VIRTUAL_ENV}/lib/python3.9/site-packages:\"'${PYTHONPATH}'\"\n",
+ "export PYTHONPATH=${VIRTUAL_ENV}/lib/python${PYV}/site-packages:\"'${PYTHONPATH}'\"\n",
" \n",
"exec python -m ipykernel \"'$@' > ${VIRTUAL_ENV}/kernel.sh\n",
"chmod +x ${VIRTUAL_ENV}/kernel.sh\n",
@@ -640,29 +388,16 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.1 - Create Jupyter kernel configuration directory and files"
+ "#### 3.1 - Create Jupyter kernel configuration directory and files"
]
},
{
"cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Installed kernelspec jovyan_kernel_ecar2023 in /home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/share/jupyter/kernels/jovyan_kernel_ecar2023\n",
- "(jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"python -m ipykernel install --name=${KERNEL_NAME} --prefix ${VIRTUAL_ENV}\n",
"export VIRTUAL_ENV_KERNELS=${VIRTUAL_ENV}/share/jupyter/kernels"
@@ -672,42 +407,16 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.2 - Adjust kernel.json file"
+ "#### 3.2 - Adjust kernel.json file"
]
},
{
"cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) {\n",
- " \"argv\": [\n",
- " \"/home/jovyan/jupyter_kernel_venvs/jovyan_kernel_ecar2023/kernel.sh\",\n",
- " \"-m\",\n",
- " \"ipykernel_launcher\",\n",
- " \"-f\",\n",
- " \"{connection_file}\"\n",
- " ],\n",
- " \"display_name\": \"jovyan_kernel_ecar2023\",\n",
- " \"language\": \"python\",\n",
- " \"metadata\": {\n",
- " \"debugger\": true\n",
- " }\n",
- "}\n",
- "(jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"mv ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json.orig\n",
"\n",
@@ -733,33 +442,16 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "* 3.3 - Create link to kernel specs"
+ "#### 3.3 - Create link to kernel specs"
]
},
{
"cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "(jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) (jovyan_kernel_ecar2023) \n",
- "\n",
- "The new kernel 'jovyan_kernel_ecar2023' was added to your kernels in '/home/jovyan/.local/share/jupyter/kernels/'\n",
- "\n",
- "(jovyan_kernel_ecar2023) jovyan_kernel_ecar2023\n",
- "(jovyan_kernel_ecar2023) "
- ]
- },
- {
- "ename": "",
- "evalue": "1",
- "output_type": "error",
- "traceback": []
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
"source": [
"mkdir -p ${KERNEL_SPECS_DIR}\n",
"cd ${KERNEL_SPECS_DIR}\n",
@@ -786,7 +478,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"outputs": [],
"source": [
"deactivate"
@@ -805,8 +499,8 @@
"metadata": {},
"source": [
"#### Exercise 1:\n",
- "- Check for the kernel directory: `$HOME/.local/share/jupyter/kernels/jovyan_kernel_ecar/`\n",
- "- Check for the kernel definition file: `$HOME/.local/share/jupyter/kernels/jovyan_kernel_ecar/kernel.json`"
+ "- Check for the kernel directory\n",
+ "- Check for the kernel definition file"
]
},
{
@@ -827,13 +521,6 @@
"- Run `import cv2` which loads the Python interface to OpenCV.\n",
"- Double-check if this package comes from the loaded module by running `cv2`"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {