diff --git a/001-Jupyter/001-Tutorials/.ipynb_checkpoints/SanityCheck-Tutorials-checkpoint.ipynb b/001-Jupyter/001-Tutorials/.ipynb_checkpoints/SanityCheck-Tutorials-checkpoint.ipynb
deleted file mode 100644
index ad5e0d46b2da937e7f2ddbb96f13468e0b37bebf..0000000000000000000000000000000000000000
--- a/001-Jupyter/001-Tutorials/.ipynb_checkpoints/SanityCheck-Tutorials-checkpoint.ipynb
+++ /dev/null
@@ -1,655 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Sanity Check\n",
- "This notebook checks if the available general tutorials run without errors. \n",
- "\n",
- "This particulary tests the installed python packages and their interoperability for standard features and examples. Running these checks can take **a lot of time** and there for will not finish in the max. cpu-time we provide for a process on a login node. Hence the jupyter kernel for this notebook will eventually be killed by the system. These Sanity Checks are primarily usefull for system administrators. \n",
- "\n",
- "If you want to run them anyway - ensure your Jupyter is running as a **batch job** with far more compute time available."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Grab Tutorials from Git-Repo "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "from git import Repo\n",
- "#repo = Repo.clone_from(\"https://gitlab.version.fz-juelich.de/jupyter4jsc/j4j_notebooks.git\", './j4j_notebooks')\n",
- "#repo.git.checkout('integration')"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Define Funktions for later Sanity Check"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "from typing import Any, Tuple, Dict, Mapping\n",
- "from collections import defaultdict\n",
- "from nbformat import NotebookNode\n",
- "\n",
- "def count_source(source: str) -> Tuple[int, int, int]:\n",
- " \"\"\"Count number of non-blank lines, words, and non-whitespace characters.\n",
- "\n",
- " :param source: string to count\n",
- " :return: number of non-blank lines, words, and non-whitespace characters\n",
- " \"\"\"\n",
- " lines = [line for line in source.split('\\n') if line and not line.isspace()]\n",
- " words = source.split()\n",
- " chars = ''.join(words)\n",
- "\n",
- " return len(lines), len(words), len(chars)\n",
- "\n",
- "REQUIRED_NB_FIELDS = {\"metadata\", \"nbformat_minor\", \"nbformat\", \"cells\"}\n",
- "REQUIRED_NB_METADATA_FIELDS = {\"kernelspec\", \"language_info\"}\n",
- "CELL_TYPES = ('markdown', 'code', 'raw', )\n",
- "REQUIRED_CELL_FIELDS = {\n",
- " 'markdown': {\"cell_type\", \"metadata\", \"source\"},\n",
- " 'code': {\"cell_type\", \"metadata\", \"source\", \"execution_count\", \"outputs\"},\n",
- " 'raw': {\"cell_type\", \"metadata\", \"source\"}\n",
- "}\n",
- "OPTIONAL_CELL_FIELDS = {\n",
- " 'markdown': {\"attachments\"},\n",
- " 'code': set(),\n",
- " 'raw': {\"attachments\"}\n",
- "}\n",
- "OPTIONAL_OUTPUT_TYPES = {\n",
- " 'execute_result': {'data', 'metadata' ,'execution_count'},\n",
- " 'stream': {'name', 'text'},\n",
- " 'display_data': {'data', 'metadata', },\n",
- " 'error': {'ename', 'evalue', 'traceback'},\n",
- "}\n",
- "\n",
- "CELL_STATISTICS = (\n",
- " 'cell_types', #: cell type counts\n",
- " 'sources', #: cell sources counts\n",
- " 'cell_metadata', #: cell metadata counts, including separate ``tags``\n",
- " 'cell_attachments', #: cell attachment MIME type counts, and total\n",
- " 'code_execution', #: code cell execution count statistics\n",
- " 'code_outputs', #: code cell counts per output_type, subcounts per ``stream`` and ``error``, and total\n",
- " 'cell_extra', #: counts for extra (unknown) fields in cells\n",
- ")\n",
- "\n",
- "# dictionary keys for source statistics\n",
- "EMPTY_SOURCES = 'total empty sources'\n",
- "SOURCE_LINES = 'total source lines'\n",
- "SOURCE_WORDS = 'total source words'\n",
- "SOURCE_CHARS = 'total source chars'\n",
- "EMPTY_SOURCES_MD = 'markdown empty sources'\n",
- "SOURCE_LINES_MD = 'markdown source lines'\n",
- "SOURCE_WORDS_MD = 'markdown source words'\n",
- "SOURCE_CHARS_MD = 'markdown source chars'\n",
- "EMPTY_SOURCES_CODE = 'code empty sources'\n",
- "SOURCE_LINES_CODE = 'code source lines'\n",
- "SOURCE_WORDS_CODE = 'code source words'\n",
- "SOURCE_CHARS_CODE = 'code source chars'\n",
- "EMPTY_SOURCES_RAW = 'raw empty sources'\n",
- "SOURCE_LINES_RAW = 'raw source lines'\n",
- "SOURCE_WORDS_RAW = 'raw source words'\n",
- "SOURCE_CHARS_RAW = 'raw source chars'\n",
- "\n",
- "def nb_cell_stats(nb: NotebookNode) -> Dict[str, Dict[str, int]]:\n",
- " \"\"\"Count occurrences of various elements in notebook cells.\n",
- "\n",
- " :param nb: notebook to inspect\n",
- " :return: dictionary of dictionaries with counts per section;\n",
- " each section has its own key; see CELL_STATISTICS\n",
- " \"\"\"\n",
- " # process the notebook cells\n",
- " result = {key: defaultdict(int) for key in CELL_STATISTICS}\n",
- "\n",
- " # traverse all cells and gather statistics\n",
- " for index, cell in enumerate(nb.cells): # index can be used for debug output\n",
- " result['cell_types']['total cell count'] += 1 # count all cells\n",
- " ct = cell.cell_type\n",
- " result['cell_types'][ct] += 1 # count each cell type\n",
- "\n",
- " # compute source statistics\n",
- " lines, words, chars = count_source(cell.source) # cell.source should always be present\n",
- " empty_cell = chars == 0\n",
- " if empty_cell:\n",
- " result['sources'][EMPTY_SOURCES] += 1\n",
- " if ct == 'markdown':\n",
- " result['sources'][EMPTY_SOURCES_MD] += 1\n",
- " elif ct == 'code':\n",
- " result['sources'][EMPTY_SOURCES_CODE] += 1\n",
- " elif ct == 'raw':\n",
- " result['sources'][EMPTY_SOURCES_RAW] += 1\n",
- " if chars:\n",
- " result['sources'][SOURCE_LINES] += lines\n",
- " result['sources'][SOURCE_WORDS] += words\n",
- " result['sources'][SOURCE_CHARS] += chars\n",
- " if ct == 'markdown':\n",
- " result['sources'][SOURCE_LINES_MD] += lines\n",
- " result['sources'][SOURCE_WORDS_MD] += words\n",
- " result['sources'][SOURCE_CHARS_MD] += chars\n",
- " elif ct == 'code':\n",
- " result['sources'][SOURCE_LINES_CODE] += lines\n",
- " result['sources'][SOURCE_WORDS_CODE] += words\n",
- " result['sources'][SOURCE_CHARS_CODE] += chars\n",
- " elif ct == 'raw':\n",
- " result['sources'][SOURCE_LINES_RAW] += lines\n",
- " result['sources'][SOURCE_WORDS_RAW] += words\n",
- " result['sources'][SOURCE_CHARS_RAW] += chars\n",
- "\n",
- " # count each metadata key\n",
- " for attr in cell.metadata: # cell.metadata should always be present\n",
- " result['cell_metadata'][attr] += 1\n",
- "\n",
- " # count each tag in tags metadata\n",
- " if 'tags' in cell.metadata:\n",
- " for tag in cell.metadata.tags:\n",
- " result['cell_metadata']['tag ' + tag] += 1\n",
- "\n",
- " # count each attachment mime type\n",
- " if 'attachments' in cell:\n",
- " result['cell_attachments']['total count of cells with attachments'] += 1\n",
- " for attachment in cell.attachments.values():\n",
- " for key in attachment:\n",
- " result['cell_attachments']['total attachments count'] += 1\n",
- " result['cell_attachments'][key] += 1\n",
- "\n",
- " # count non-standard fields in cells\n",
- " for field in cell:\n",
- " if field not in REQUIRED_CELL_FIELDS[ct].union(OPTIONAL_CELL_FIELDS[ct]):\n",
- " result['cell_extra'][field] += 1\n",
- "\n",
- " return result\n",
- "\n",
- "from colorama import Fore, Back, Style\n",
- "DEFAULT_WIDTH = 10\n",
- "def print_dict(d: Dict[str, Any], header: str=None, width: int=DEFAULT_WIDTH) -> None:\n",
- " \"\"\"Print dictionary d with section header.\n",
- "\n",
- " :param d: dictionary to print\n",
- " :param header: header of the table\n",
- " :param width: width of the left column\n",
- " \"\"\"\n",
- " if d:\n",
- " if header:\n",
- " print('{}:'.format(header))\n",
- " for key in sorted(d):\n",
- " if key == 'raw':\n",
- " style = Fore.RED\n",
- " else:\n",
- " style = ''\n",
- " left = str(d[key])\n",
- " print(style + ' {:>{}} {}'.format(left, width, key) + Style.RESET_ALL)\n",
- " \n",
- "from pathlib import Path\n",
- "from nbformat import NotebookNode\n",
- "from typing import List, Union\n",
- "import nbformat\n",
- "import sys\n",
- "\n",
- "def read_nb(nb_path: Path) -> Union[None, NotebookNode]:\n",
- " \"\"\"Read notebook from given path, and return it.\n",
- " Uses ``args.debug``: in debug mode, a read error results in an exception, else it returns ``None``.\n",
- "\n",
- " :param nb_path: path to read from\n",
- " :param args: to check debug mode\n",
- " :return: notebook read from ``nb_path`` or None if reading failed``\n",
- " \"\"\"\n",
- " try:\n",
- " nb = nbformat.read(nb_path.open(encoding='utf-8'), as_version=4)\n",
- " except Exception as e:\n",
- " ename = type(e).__name__\n",
- " print('Reading of \"{}\" failed ({}):\\n {}'.format(nb_path.name, ename, e), file=sys.stderr)\n",
- " return None\n",
- "\n",
- " return nb"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Run the Sanity Check"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "ename": "ModuleNotFoundError",
- "evalue": "No module named 'papermill'",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<ipython-input-2-4fe21cbd60c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpapermill\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpapermill\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPapermillExecutionError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mfailed_notebooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'papermill'"
- ]
- }
- ],
- "source": [
- "import os\n",
- "import papermill as pm\n",
- "from papermill.exceptions import PapermillExecutionError\n",
- "\n",
- "failed_notebooks = list()\n",
- "dirbase=\"/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/\"\n",
- "for dirpath, dirs, files in os.walk(dirbase):\n",
- " dirs.sort()\n",
- " if os.path.basename(dirpath).startswith('.'):\n",
- " continue\n",
- " for filename in sorted(files):\n",
- " if filename.endswith('.ipynb') and not filename.startswith('papermill_'):\n",
- " print(os.path.join(dirpath,filename))\n",
- " if filename == \"SanityCheck-Tutorials.ipynb\":\n",
- " continue\n",
- " try:\n",
- " os.chdir(dirpath)\n",
- " nb_path = os.path.join(dirpath, filename)\n",
- " \n",
- " # get notebook statistics\n",
- " nb = read_nb(Path(nb_path))\n",
- " cell_stats = nb_cell_stats(nb)\n",
- " print_dict(cell_stats['cell_types'], \"Cell types\")\n",
- " #print_dict(cell_stats['sources'], \"Cell sources\")\n",
- " #print_dict(cell_stats['cell_metadata'], \"Cell metadata fields\")\n",
- " #print_dict(cell_stats['cell_attachments'], \"Cell attachments\") \n",
- " \n",
- " # execute notebook\n",
- " nb = pm.execute_notebook(\n",
- " nb_path,\n",
- " os.path.join(dirpath, 'papermill_' + filename),\n",
- " #kernel_name=\"Python3\"\n",
- " )\n",
- " \n",
- " os.chdir(dirbase)\n",
- " except PapermillExecutionError as e:\n",
- " failed_notebooks.append([os.path.join(dirpath, filename), e.evalue])\n",
- " print(e.evalue)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Check Results"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/Example - Beat Frequencies.ipynb',\n",
- " 'too many values to unpack (expected 2)'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/08_cuda.ipynb',\n",
- " 'Error at driver init: \\n[100] Call to cuInit results in CUDA_ERROR_NO_DEVICE:'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/09_ipyparallel.ipynb',\n",
- " \"Can't build targets without any engines\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/06_kde.ipynb',\n",
- " \"No module named 'cartopy'\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter09_numoptim/03_curvefitting.ipynb',\n",
- " 'Optimal parameters not found: Number of calls to function has reached maxfev = 1000.'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/05_faces.ipynb',\n",
- " \"No module named 'cv2'\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/02_airports.ipynb',\n",
- " \"No module named 'cartopy'\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/06_gis.ipynb',\n",
- " \"No module named 'cartopy'\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/07_gps.ipynb',\n",
- " 'read_shp requires OGR: http://www.gdal.org/'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/Animations Using clear_output.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Data Publication API.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Monitoring an MPI Simulation - 1.ipynb',\n",
- " 'You have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Monitoring an MPI Simulation - 2.ipynb',\n",
- " 'You have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Monte Carlo Options.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Parallel Decorator and map.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Parallel Magics.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Using Dill.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/Using MPI with IPython Parallel.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/rmt/rmt.ipynb',\n",
- " \"Connection file '~/.ipython/profile_default/security/ipcontroller-client.json' not found.\\nYou have attempted to connect to an IPython Cluster but no Controller could be found.\\nPlease double-check your configuration and ensure that a cluster is running.\"],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Notebook/Notebook Exercises.ipynb',\n",
- " 'HTTP Error 404: Not Found'],\n",
- " ['/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/Lecture-6B-HPC.ipynb',\n",
- " \"Can't build targets without any engines\"]]"
- ]
- },
- "execution_count": 7,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "failed_notebooks"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Clean Up"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/papermill_SanityCheck-Tutorials.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Beyond Plain Python.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Cell Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Working With External Code.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Custom Display Logic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Script Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Plotting in the Notebook with Matplotlib.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Animations Using clear_output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Rich Output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Raw Input in the Notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Capturing Output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Background Jobs.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_Cython Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/001-IPython-Kernel/papermill_SymPy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Example - Beat Frequencies.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Example - Factoring.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Example - Lorenz Differential Equations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Using Interact.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Widget Styling.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Widget Events.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Widget List.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/002-Interactive-Widgets/papermill_Widget Basics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_What is the IPython Notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_Notebook Basics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_Working With Markdown Cells.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_JupyterLab Interface.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_Typesetting Equations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/001-Basic-Tutorials/003-Notebook/papermill_Running Code.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_06_kde.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_03_bayesian.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_05_mlfit.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_01_pandas.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_02_z_test.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_04_correlation.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter07_stats/papermill_07_pymc.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_07_lotka.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_02_solvers.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_05_number_theory.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_01_sympy_intro.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_04_stats.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_03_function.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter15_symbolic/papermill_06_logic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_05_config.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_06_kernel.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_01_notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_04_magic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_02_pandas.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter01_basic/papermill_03_numpy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter06_viz/papermill_03_bokeh.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter06_viz/papermill_01_styles.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter06_viz/papermill_02_seaborn.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter06_viz/papermill_06_altair.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter06_viz/papermill_05_widgets.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter12_deterministic/papermill_01_bifurcation.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter12_deterministic/papermill_02_cellular.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter12_deterministic/papermill_04_turing.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter12_deterministic/papermill_03_ode.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_03_dag.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_04_connected.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_02_airports.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_05_voronoi.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_01_networkx.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_06_gis.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter14_graphgeo/papermill_07_gps.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_02_titanic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_03_digits.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_06_random_forest.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_04_text.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_05_svm.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_01_scikit.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_08_clustering.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter08_ml/papermill_07_pca.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_07_rolling_average.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_04_memprof.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_01_timeit.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_05_array_copies.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_02_profile.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_03_linebyline.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_08_memmap.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_09_hdf5_array.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter04_optimization/papermill_06_stride_tricks.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_12_julia.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_05_cython.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_08_cuda.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_06_ray.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_01_slow.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_07_openmp.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_03_numexpr.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_10_async.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_04_ctypes.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_02_numba.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_11_dask.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/papermill_09_ipyparallel.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_6.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_1.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_4.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_2.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_3.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_7.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter05_hpc/06_ray/papermill_06_ray_5.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter09_numoptim/papermill_04_energy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter09_numoptim/papermill_01_root.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter09_numoptim/papermill_02_minimize.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter09_numoptim/papermill_03_curvefitting.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter10_signal/papermill_03_autocorrelation.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter10_signal/papermill_02_filter.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter10_signal/papermill_01_fourier.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter13_stochastic/papermill_03_brownian.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter13_stochastic/papermill_01_markov.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter13_stochastic/papermill_04_sde.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter13_stochastic/papermill_02_poisson.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter03_notebook/papermill_02_nbformat.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter03_notebook/papermill_05_custom_notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter03_notebook/papermill_04_custom_widgets.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter03_notebook/papermill_03_widgets.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter03_notebook/papermill_01_blocks.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_02_filters.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_07_synth.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_04_interest.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_03_segmentation.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_06_speech.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_01_exposure.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/002-IPython-Cookbook/chapter11_image/papermill_05_faces.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/papermill_index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/quickstart/papermill_quickstart.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_00 - Introduction and Setup.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_08 - Graph and Network Plots.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_02 - Styling and Theming.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_06 - Linking and Interactions.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_A3 - High-Level Charting with Holoviews.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_05 - Presentation Layouts.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_A1 - Models and Primitives.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_A2 - Visualizing Big Data with Datashader.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_A4 - Additional Resources.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_04 - Data Sources and Transformations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_10 - Exporting and Embedding.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_07 - Bar and Categorical Data Plots.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_03 - Adding Annotations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_09 - Geographic Plots.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_11 - Running Bokeh Applictions.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/006-Bokeh/tutorial/papermill_01 - Basic Plotting.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_04.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_08.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_02.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_07.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_03.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_01.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_00.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_06.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_11.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/005-Python4Maths/papermill_05.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-4-Matplotlib.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-7-Revision-Control-Software.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-6B-HPC.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-0-Scientific-Computing-with-Python.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-6A-Fortran-and-C.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-3-Scipy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-2-Numpy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-5-Sympy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/004-Scientific-Python-Lectures/papermill_Lecture-1-Introduction-to-Python-Programming.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/IPython Kernel/papermill_Custom Display Logic Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/IPython Kernel/papermill_Rich Output Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/IPython Kernel/papermill_Background Jobs Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Customization/papermill_Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Customization/papermill_Condensed.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Customization/papermill_Configuration.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Customization/papermill__Sample.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Customization/papermill_Custom magic and cross language integration.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Interactive Widgets/papermill_Widget Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Interactive Widgets/papermill_Interact Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/exercises/Notebook/papermill_Notebook Exercises.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Embedding/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Terminal Usage.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Plotting in the Notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Beyond Plain Python.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Cell Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Trapezoid Rule.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Working With External Code.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Custom Display Logic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Script Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Importing Notebooks.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Animations Using clear_output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Rich Output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Raw Input in the Notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Capturing Output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Old Custom Display Logic.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Background Jobs.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Updating Displays.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_Third Party Rich Output.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/papermill_SymPy.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/nbpackage/papermill_mynotebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/IPython Kernel/nbpackage/nbs/papermill_other.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Monitoring an MPI Simulation - 2.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Data Publication API.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Parallel Decorator and map.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Using MPI with IPython Parallel.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Using Dill.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Parallel Magics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Monte Carlo Options.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/papermill_Monitoring an MPI Simulation - 1.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Parallel Computing/rmt/papermill_rmt.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Image Processing.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Export As (nbconvert).ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Beat Frequencies.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Nonblocking Console.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Using Interact.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Exploring Graphs.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Widget Styling.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Widget Events.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Factoring.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Image Browser.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Widget List.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Widget Basics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Interactive Widgets/papermill_Lorenz Differential Equations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_What is the IPython Notebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Running the Notebook Server.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Notebook Basics.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Working With Markdown Cells.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Importing Notebooks.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Connecting with the Qt Console.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Julia and Python Bridge.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Multiple Languages, Frontends.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Converting Notebooks With nbconvert.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Typesetting Equations.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/papermill_Running Code.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/nbpackage/papermill_mynotebook.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Notebook/nbpackage/nbs/papermill_other.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Magics/papermill_Index.ipynb\n",
- "/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/003-IPython-in-Depth/examples/Magics/papermill_Cython Magics.ipynb\n"
- ]
- }
- ],
- "source": [
- "import os\n",
- "import papermill as pm\n",
- "from papermill.exceptions import PapermillExecutionError\n",
- "\n",
- "failed_notebooks = list()\n",
- "dirbase=\"/p/home/jusers/goebbert1/jureca/j4j_notebooks_2/001-Tutorials/\"\n",
- "for dirpath, dirs, files in os.walk(dirbase):\n",
- " if os.path.basename(dirpath).startswith('.'):\n",
- " continue\n",
- " for filename in files:\n",
- " if filename.endswith('.ipynb') and filename.startswith('papermill_'):\n",
- " nb_path = os.path.join(dirpath,filename)\n",
- " print(nb_path)\n",
- " os.remove(nb_path)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.8"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/running-checkpoint.png b/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/running-checkpoint.png
deleted file mode 100644
index 4a1dc06e7ec0f1afae467345ace1043a16647d17..0000000000000000000000000000000000000000
Binary files a/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/running-checkpoint.png and /dev/null differ
diff --git a/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/tabs-checkpoint.png b/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/tabs-checkpoint.png
deleted file mode 100644
index bc9e09ace4edc30c4d2b3d5056c409564f98b514..0000000000000000000000000000000000000000
Binary files a/001-Jupyter/001-Tutorials/001-Basic-Tutorials/003-Notebook/images/.ipynb_checkpoints/tabs-checkpoint.png and /dev/null differ
diff --git a/002-Methods/001-Computing/.ipynb_checkpoints/Howto_Dask_onJUWELS-checkpoint.ipynb b/002-Methods/001-Computing/.ipynb_checkpoints/Howto_Dask_onJUWELS-checkpoint.ipynb
deleted file mode 100644
index b56c82bfc3037d5a7637e51e382cd8f4fc2069d2..0000000000000000000000000000000000000000
--- a/002-Methods/001-Computing/.ipynb_checkpoints/Howto_Dask_onJUWELS-checkpoint.ipynb
+++ /dev/null
@@ -1,346 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Dask Extension"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "---\n",
- "This notebook will give you a short introduction into the Dask Extension on JUWELS. It allows you to run Jobs on the compute nodes, even if your JupyterLab is running interactively on the login node. \n",
- "First you have to define on which project and partition it should be running."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "queue = \"batch\" # batch, gpus, develgpus, etc.\n",
- "project = \"training2005\" # your project: zam, training19xx, etc.\n",
- "port = 56755"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Monte-Carlo Estimate of $\\pi$\n",
- "\n",
- "We want to estimate the number $\\pi$ using a [Monte-Carlo method](https://en.wikipedia.org/wiki/Pi#Monte_Carlo_methods) exploiting that the area of a quarter circle of unit radius is $\\pi/4$ and that hence the probability of any randomly chosen point in a unit square to lie in a unit circle centerd at a corner of the unit square is $\\pi/4$ as well. So for N randomly chosen pairs $(x, y)$ with $x\\in[0, 1)$ and $y\\in[0, 1)$, we count the number $N_{circ}$ of pairs that also satisfy $(x^2 + y^2) < 1$ and estimage $\\pi \\approx 4 \\cdot N_{circ} / N$.\n",
- "\n",
- "[<img src=\"https://upload.wikimedia.org/wikipedia/commons/8/84/Pi_30K.gif\" \n",
- " width=\"50%\" \n",
- " align=top\n",
- " alt=\"PI monte-carlo estimate\">](https://en.wikipedia.org/wiki/Pi#Monte_Carlo_methods)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Core Lessons\n",
- "\n",
- "- setting up SLURM (and other jobqueue) clusters\n",
- "- Scaling clusters\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Set up a Slurm cluster\n",
- "\n",
- "We'll create a SLURM cluster and have a look at the job-script used to start workers on the HPC scheduler."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import dask\n",
- "from dask.distributed import Client\n",
- "from dask_jobqueue import SLURMCluster\n",
- "import os\n",
- "\n",
- "cluster = SLURMCluster(\n",
- " cores=96,\n",
- " processes=4,\n",
- " memory=\"70GB\",\n",
- " shebang=\"#!/usr/bin/env bash\",\n",
- " queue=queue,\n",
- " dashboard_address=\":\"+str(port),\n",
- " walltime=\"00:30:00\",\n",
- " local_directory=\"/tmp\",\n",
- " death_timeout=\"30s\",\n",
- " log_directory=f'{os.environ[\"HOME\"]}/dask_jobqueue_logs/',\n",
- " interface=\"ib1\",\n",
- " project=project,\n",
- " extra=['--host $SLURMD_NODENAME.ib.juwels.fzj.de'],\n",
- ")\n",
- "# optional: job_extra=[\"--reservation=reservation_name\"]\n",
- "# interface can be skipped if the master process runs on a comput node"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(cluster.job_script())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "cluster"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "client = Client(cluster)\n",
- "client"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## You can visit the Dask Dashboard at the following url: \n",
- "```\n",
- "https://jupyter-jsc.fz-juelich.de/user/<user_name>/<lab_name>/proxy/<port>/status\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## You can integrate it into your JupyterLab environment by putting the link into the Dask Extension"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "![\"Dask\"](https://zam10183.zam.kfa-juelich.de/hub/static/images/dask2.png \"dask\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Afterwards you can press on the orange buttons to open a new tab in your JupyterLab Environment."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Scale the cluster to two nodes\n",
- "\n",
- "A look at the Dashboard reveals that there are no workers in the clusetr. Let's start 4 workers (in 2 SLURM jobs).\n",
- "\n",
- "For the distiction between _workers_ and _jobs_, see [the Dask jobqueue docs](https://jobqueue.dask.org/en/latest/howitworks.html#workers-vs-jobs)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "cluster.scale(4) # scale to 4 _workers_"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## The Monte Carlo Method"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import dask.array as da\n",
- "import numpy as np\n",
- "\n",
- "\n",
- "def calc_pi_mc(size_in_bytes, chunksize_in_bytes=200e6):\n",
- " \"\"\"Calculate PI using a Monte Carlo estimate.\"\"\"\n",
- "\n",
- " size = int(size_in_bytes / 8)\n",
- " chunksize = int(chunksize_in_bytes / 8)\n",
- "\n",
- " xy = da.random.uniform(0, 1, size=(size / 2, 2), chunks=(chunksize / 2, 2))\n",
- "\n",
- " in_circle = (xy ** 2).sum(axis=-1) < 1\n",
- " pi = 4 * in_circle.mean()\n",
- "\n",
- " return pi\n",
- "\n",
- "\n",
- "def print_pi_stats(size, pi, time_delta, num_workers):\n",
- " \"\"\"Print pi, calculate offset from true value, and print some stats.\"\"\"\n",
- " print(\n",
- " f\"{size / 1e9} GB\\n\"\n",
- " f\"\\tMC pi: {pi : 13.11f}\"\n",
- " f\"\\tErr: {abs(pi - np.pi) : 10.3e}\\n\"\n",
- " f\"\\tWorkers: {num_workers}\"\n",
- " f\"\\t\\tTime: {time_delta : 7.3f}s\"\n",
- " )"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## The actual calculations\n",
- "\n",
- "We loop over different volumes of double-precision random numbers and estimate $\\pi$ as described above."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from time import time, sleep"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for size in (1e9 * n for n in (1, 10, 100)):\n",
- "\n",
- " start = time()\n",
- " pi = calc_pi_mc(size).compute()\n",
- " elaps = time() - start\n",
- "\n",
- " print_pi_stats(\n",
- " size, pi, time_delta=elaps, num_workers=len(cluster.scheduler.workers)\n",
- " )"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Is it running?"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "To check if something has been started for you just use the following command in a terminal: \n",
- "```\n",
- "squeue | grep ${USER}\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Scaling the Cluster to twice its size\n",
- "\n",
- "We increase the number of workers by 2 and the re-run the experiments."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "new_num_workers = 2 * len(cluster.scheduler.workers)\n",
- "\n",
- "print(f\"Scaling from {len(cluster.scheduler.workers)} to {new_num_workers} workers.\")\n",
- "\n",
- "cluster.scale(new_num_workers)\n",
- "\n",
- "sleep(10)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "client"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Re-run same experiments with doubled cluster"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for size in (1e9 * n for n in (1, 10, 100)):\n",
- "\n",
- " start = time()\n",
- " pi = calc_pi_mc(size).compute()\n",
- " elaps = time() - start\n",
- "\n",
- " print_pi_stats(\n",
- " size, pi, time_delta=elaps, num_workers=len(cluster.scheduler.workers)\n",
- " )"
- ]
- }
- ],
- "metadata": {
- "anaconda-cloud": {},
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.8"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}